A new non volatile storage system

doc/services/storage/index.rst

@@ -7,6 +7,7 @@ Storage
    :maxdepth: 1
 
    nvs/nvs.rst
+   zms/zms.rst
    disk/access.rst
    flash_map/flash_map.rst
    fcb/fcb.rst

include/zephyr/fs/zms.h

@@ -0,0 +1,215 @@
+/*  ZMS: Zephyr Memory Storage
+ *
+ * Copyright (c) 2024 BayLibre SAS
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+#ifndef ZEPHYR_INCLUDE_FS_ZMS_H_
+#define ZEPHYR_INCLUDE_FS_ZMS_H_
+
+#include <zephyr/drivers/flash.h>
+#include <sys/types.h>
+#include <zephyr/kernel.h>
+#include <zephyr/device.h>
+#include <zephyr/toolchain.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Zephyr Memory Storage (ZMS)
+ * @defgroup zms Zephyr Memory Storage (ZMS)
+ * @ingroup file_system_storage
+ * @{
+ * @}
+ */
+
+/**
+ * @brief Zephyr Memory Storage Data Structures
+ * @defgroup zms_data_structures Zephyr Memory Storage Data Structures
+ * @ingroup zms
+ * @{
+ */
+
+/**
+ * @brief Zephyr Memory Storage File system structure
+ */
+struct zms_fs {
+	/** File system offset in flash **/
+	off_t offset;
+	/** Allocation table entry write address.
+	 * Addresses are stored as uint64_t:
+	 * - high 4 bytes correspond to the sector
+	 * - low 4 bytes are the offset in the sector
+	 */
+	uint64_t ate_wra;
+	/** Data write address */
+	uint64_t data_wra;
+	/** Storage system is split into sectors, each sector size must be multiple of erase-blocks
+	 *  if the device has erase capabilities
+	 */
+	uint32_t sector_size;
+	/** Number of sectors in the file system */
+	uint32_t sector_count;
+	/** Current cycle counter of the active sector (pointed by ate_wra)*/
+	uint8_t sector_cycle;
+	/** Flag indicating if the file system is initialized */
+	bool ready;
+	/** Mutex */
+	struct k_mutex zms_lock;
+	/** Flash device runtime structure */
+	const struct device *flash_device;
+	/** Flash memory parameters structure */
+	const struct flash_parameters *flash_parameters;
+	/** Size of an Allocation Table Entry */
+	size_t ate_size;
+#if CONFIG_ZMS_LOOKUP_CACHE
+	/** Lookup table used to cache ATE address of a written ID */
+	uint64_t lookup_cache[CONFIG_ZMS_LOOKUP_CACHE_SIZE];
+#endif
+};
+
+/**
+ * @}
+ */
+
+/**
+ * @brief Zephyr Memory Storage APIs
+ * @defgroup zms_high_level_api Zephyr Memory Storage APIs
+ * @ingroup zms
+ * @{
+ */
+
+/**
+ * @brief Mount a ZMS file system onto the device specified in @p fs.
+ *
+ * @param fs Pointer to file system
+ * @retval 0 Success
+ * @retval -ERRNO errno code if error
+ */
+int zms_mount(struct zms_fs *fs);
+
+/**
+ * @brief Clear the ZMS file system from device.
+ *
+ * @param fs Pointer to file system
+ * @retval 0 Success
+ * @retval -ERRNO errno code if error
+ */
+int zms_clear(struct zms_fs *fs);
+
+/**
+ * @brief Write an entry to the file system.
+ *
+ * @note  When @p len parameter is equal to @p 0 then entry is effectively removed (it is
+ * equivalent to calling of zms_delete). It is not possible to distinguish between a deleted
+ * entry and an entry with data of length 0.
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be written
+ * @param data Pointer to the data to be written
+ * @param len Number of bytes to be written (maximum 64 KB)
+ *
+ * @return Number of bytes written. On success, it will be equal to the number of bytes requested
+ * to be written. When a rewrite of the same data already stored is attempted, nothing is written
+ * to flash, thus 0 is returned. On error, returns negative value of errno.h defined error codes.
+ */
+ssize_t zms_write(struct zms_fs *fs, uint32_t id, const void *data, size_t len);
+
+/**
+ * @brief Delete an entry from the file system
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be deleted
+ * @retval 0 Success
+ * @retval -ERRNO errno code if error
+ */
+int zms_delete(struct zms_fs *fs, uint32_t id);
+
+/**
+ * @brief Read an entry from the file system.
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be read
+ * @param data Pointer to data buffer
+ * @param len Number of bytes to be read (or size of the allocated read buffer)
+ *
+ * @return Number of bytes read. On success, it will be equal to the number of bytes requested
+ * to be read. When the return value is less than the number of bytes requested to read this
+ * indicates that ATE contain less data than requested. On error, returns negative value of
+ * errno.h defined error codes.
+ */
+ssize_t zms_read(struct zms_fs *fs, uint32_t id, void *data, size_t len);
+
+/**
+ * @brief Read a history entry from the file system.
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry to be read
+ * @param data Pointer to data buffer
+ * @param len Number of bytes to be read
+ * @param cnt History counter: 0: latest entry, 1: one before latest ...
+ *
+ * @return Number of bytes read. On success, it will be equal to the number of bytes requested
+ * to be read. When the return value is larger than the number of bytes requested to read this
+ * indicates not all bytes were read, and more data is available. On error, returns negative
+ * value of errno.h defined error codes.
+ */
+ssize_t zms_read_hist(struct zms_fs *fs, uint32_t id, void *data, size_t len, uint32_t cnt);
+
+/**
+ * @brief Gets the data size that is stored in an entry with a given id
+ *
+ * @param fs Pointer to file system
+ * @param id Id of the entry that we want to get its data length
+ *
+ * @return Data length contained in the ATE. On success, it will be equal to the number of bytes
+ * in the ATE. On error, returns negative value of errno.h defined error codes.
+ */
+ssize_t zms_get_data_length(struct zms_fs *fs, uint32_t id);
+/**
+ * @brief Calculate the available free space in the file system.
+ *
+ * @param fs Pointer to file system
+ *
+ * @return Number of bytes free. On success, it will be equal to the number of bytes that can
+ * still be written to the file system.
+ * Calculating the free space is a time consuming operation, especially on spi flash.
+ * On error, returns negative value of errno.h defined error codes.
+ */
+ssize_t zms_calc_free_space(struct zms_fs *fs);
+
+/**
+ * @brief Tell how many contiguous free space remains in the currently active ZMS sector.
+ *
+ * @param fs Pointer to the file system.
+ *
+ * @return Number of free bytes.
+ */
+size_t zms_sector_max_data_size(struct zms_fs *fs);
+
+/**
+ * @brief Close the currently active sector and switch to the next one.
+ *
+ * @note The garbage collector is called on the new sector.
+ *
+ * @warning This routine is made available for specific use cases.
+ * It collides with the ZMS goal of avoiding any unnecessary flash erase operations.
+ * Using this routine extensively can result in premature failure of the flash device.
+ *
+ * @param fs Pointer to the file system.
+ *
+ * @return 0 on success. On error, returns negative value of errno.h defined error codes.
+ */
+int zms_sector_use_next(struct zms_fs *fs);
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ZEPHYR_INCLUDE_FS_ZMS_H_ */

samples/subsys/fs/zms/CMakeLists.txt

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+
+cmake_minimum_required(VERSION 3.20.0)
+
+find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
+project(zms)
+
+
+target_sources(app PRIVATE src/main.c)
+target_include_directories(app PRIVATE ${ZEPHYR_BASE}/subsys/fs/zms)

samples/subsys/fs/zms/README.rst

@@ -0,0 +1,96 @@
+.. zephyr:code-sample:: zms
+   :name: Zephyr Memory Storage (ZMS)
+   :relevant-api: zms_high_level_api
+
+   Store and retrieve data from storage using the ZMS API.
+
+Overview
+********
+ The sample shows how to use ZMS to store ID/VALUE pairs and reads them back.
+ Deleting an ID/VALUE pair is also shown in this sample.
+
+ The sample stores the following items:
+
+ #. A string representing an IP address: stored at id=1, data="192.168.1.1"
+ #. A binary blob representing a key/value pair: stored at id=0xbeefdead,
+    data={0xDE, 0xAD, 0xBE, 0xEF, 0xDE, 0xAD, 0xBE, 0xEF}
+ #. A variable (32bit): stored at id=2, data=cnt
+ #. A long set of data (128 bytes)
+
+ A loop is executed where we mount the storage system, and then write all set
+ of data.
+
+ Each DELETE_ITERATION period, we delete all set of data and verify that it has been deleted.
+ We generate as well incremented ID/value pairs, we store them until storage is full, then we
+ delete them and verify that storage is empty.
+
+Requirements
+************
+
+* A board with flash support or native_sim target
+
+Building and Running
+********************
+
+This sample can be found under :zephyr_file:`samples/subsys/fs/zms` in the Zephyr tree.
+
+The sample can be built for several platforms, but for the moment it has been tested only
+on native_sim target
+
+.. zephyr-app-commands::
+   :zephyr-app: samples/subsys/fs/zms
+   :goals: build
+   :compact:
+
+After running the generated image on a native_sim target, the output on the console shows the
+multiple Iterations of read/write/delete exectuted.
+
+Sample Output
+=============
+
+.. code-block:: console
+
+   *** Booting Zephyr OS build v3.7.0-2383-g624f75400242 ***
+   [00:00:00.000,000] <inf> fs_zms: 3 Sectors of 4096 bytes
+   [00:00:00.000,000] <inf> fs_zms: alloc wra: 0, fc0
+   [00:00:00.000,000] <inf> fs_zms: data wra: 0, 0
+   ITERATION: 0
+   Adding IP_ADDRESS 172.16.254.1 at id 1
+   Adding key/value at id beefdead
+   Adding counter at id 2
+   Adding Longarray at id 3
+   [00:00:00.000,000] <inf> fs_zms: 3 Sectors of 4096 bytes
+   [00:00:00.000,000] <inf> fs_zms: alloc wra: 0, f80
+   [00:00:00.000,000] <inf> fs_zms: data wra: 0, 8c
+   ITERATION: 1
+   ID: 1, IP Address: 172.16.254.1
+   Adding IP_ADDRESS 172.16.254.1 at id 1
+   Id: beefdead, Key: de ad be ef de ad be ef
+   Adding key/value at id beefdead
+   Id: 2, loop_cnt: 0
+   Adding counter at id 2
+   Id: 3, Longarray: 0 1 2 3 4 5 6 7 8 9 a b c d e f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f 50 51 52 53 5
+   4 55 56 57 58 59 5a 5b 5c 5d 5e 5f 60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f 70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f
+   Adding Longarray at id 3
+   .
+   .
+   .
+   .
+   .
+   .
+   [00:00:00.000,000] <inf> fs_zms: 3 Sectors of 4096 bytes
+   [00:00:00.000,000] <inf> fs_zms: alloc wra: 0, f40
+   [00:00:00.000,000] <inf> fs_zms: data wra: 0, 80
+   ITERATION: 299
+   ID: 1, IP Address: 172.16.254.1
+   Adding IP_ADDRESS 172.16.254.1 at id 1
+   Id: beefdead, Key: de ad be ef de ad be ef
+   Adding key/value at id beefdead
+   Id: 2, loop_cnt: 298
+   Adding counter at id 2
+   Id: 3, Longarray: 0 1 2 3 4 5 6 7 8 9 a b c d e f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f 50 51 52 53 5
+   4 55 56 57 58 59 5a 5b 5c 5d 5e 5f 60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f 70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f
+   Adding Longarray at id 3
+   Memory is full let's delete all items
+   Free space in storage is 8064 bytes
+   Sample code finished Successfully

samples/subsys/fs/zms/prj.conf

@@ -0,0 +1,5 @@
+CONFIG_FLASH=y
+CONFIG_FLASH_MAP=y
+
+CONFIG_ZMS=y
+CONFIG_LOG=y

samples/subsys/fs/zms/sample.yaml

@@ -0,0 +1,10 @@
+sample:
+  name: ZMS Sample
+
+tests:
+  sample.zms.basic:
+    tags: zms
+    depends_on: zms
+    platform_allow:
+      - qemu_x86
+      - native_posix