diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index 17a4c46..2bf2375 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -35,6 +35,8 @@ #define SECTOR_MAGIC_WORD 0x30424446 /* magic word(`K`, `V`, `0`, `0`) */ #define KV_MAGIC_WORD 0x3030564B +/* GC minimum number of empty sectors. GC will using at least 1 empty sector. */ +#define GC_MIN_EMPTY_SEC_NUM 1 /* the sector remain threshold before full status */ #ifndef FDB_SEC_REMAIN_THRESHOLD @@ -135,9 +137,8 @@ struct alloc_kv_cb_args { struct gc_cb_args { fdb_kvdb_t db; - size_t cur_free_size; size_t setting_free_size; - uint32_t traversed_len; + size_t last_gc_sec_addr; }; static void gc_collect(fdb_kvdb_t db); @@ -1071,7 +1072,7 @@ static uint32_t new_kv(fdb_kvdb_t db, kv_sec_info_t sector, size_t kv_size) if ((empty_kv = alloc_kv(db, sector, kv_size)) == FAILED_ADDR) { if (db->gc_request && !already_gc) { - FDB_INFO("Warning: Alloc an KV (size %" PRIu32 ") failed when new KV. Now will GC then retry.\n", (uint32_t)kv_size); + FDB_DEBUG("Alloc an KV (size %" PRIu32 ") failed when new KV. Now will GC then retry.\n", (uint32_t)kv_size); gc_collect_by_free_size(db, kv_size); already_gc = true; goto __retry; @@ -1093,10 +1094,12 @@ static uint32_t new_kv_ex(fdb_kvdb_t db, kv_sec_info_t sector, size_t key_len, s static bool gc_check_cb(kv_sec_info_t sector, void *arg1, void *arg2) { - size_t *empty_sec = arg1; + size_t *empty_sec_num = arg1; + uint32_t *empty_sec_addr = arg2; if (sector->check_ok) { - *empty_sec = *empty_sec + 1; + *empty_sec_num = *empty_sec_num + 1; + *empty_sec_addr = sector->addr; } return false; @@ -1108,6 +1111,7 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) struct fdb_kv kv; struct gc_cb_args *gc = (struct gc_cb_args *)arg1; fdb_kvdb_t db = gc->db; + uint32_t last_gc_sec_addr = 0; if (sector->check_ok && (sector->status.dirty == FDB_SECTOR_DIRTY_TRUE || sector->status.dirty == FDB_SECTOR_DIRTY_GC)) { uint8_t status_table[FDB_DIRTY_STATUS_TABLE_SIZE]; @@ -1122,15 +1126,22 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) if (move_kv(db, &kv) != FDB_NO_ERR) { FDB_INFO("Error: Moved the KV (%.*s) for GC failed.\n", kv.name_len, kv.name); } + } else { + FDB_DEBUG("KV (%.*s) is garbage NOT need move, collect it.\n", kv.name_len, kv.name); } } while ((kv.addr.start = get_next_kv_addr(db, sector, &kv)) != FAILED_ADDR); format_sector(db, sector->addr, SECTOR_NOT_COMBINED); - gc->cur_free_size += db_sec_size(db) - SECTOR_HDR_DATA_SIZE; - FDB_DEBUG("Collect a sector @0x%08" PRIX32 "\n", sector->addr); + last_gc_sec_addr = gc->last_gc_sec_addr; + gc->last_gc_sec_addr = sector->addr; /* update oldest_addr for next GC sector format */ db_oldest_addr(db) = get_next_sector_addr(db, sector, 0); - if (gc->cur_free_size >= gc->setting_free_size) - return true; + FDB_DEBUG("Collect a sector @0x%08" PRIX32 "\n", sector->addr); + /* the collect new space is in last GC sector */ + struct kvdb_sec_info last_gc_sector; + if (read_sector_info(db, last_gc_sec_addr, &last_gc_sector, true) == FDB_NO_ERR) { + if (last_gc_sector.remain > gc->setting_free_size) + return true; + } } return false; @@ -1139,15 +1150,17 @@ static bool do_gc(kv_sec_info_t sector, void *arg1, void *arg2) static void gc_collect_by_free_size(fdb_kvdb_t db, size_t free_size) { struct kvdb_sec_info sector; - size_t empty_sec = 0; - struct gc_cb_args arg = { db, 0, free_size, 0 }; + size_t empty_sec_num = 0; + /* an empty sector address */ + uint32_t empty_sec_addr = 0; /* GC check the empty sector number */ - sector_iterator(db, §or, FDB_SECTOR_STORE_EMPTY, &empty_sec, NULL, gc_check_cb, false); + sector_iterator(db, §or, FDB_SECTOR_STORE_EMPTY, &empty_sec_num, &empty_sec_addr, gc_check_cb, false); /* do GC collect */ - FDB_DEBUG("The remain empty sector is %" PRIu32 ", GC threshold is %" PRIdLEAST16 ".\n", (uint32_t)empty_sec, FDB_GC_EMPTY_SEC_THRESHOLD); - if (empty_sec <= FDB_GC_EMPTY_SEC_THRESHOLD) { + FDB_DEBUG("The remain empty sector is %" PRIu32 ", GC threshold is %" PRIdLEAST16 ".\n", (uint32_t)empty_sec_num, FDB_GC_EMPTY_SEC_THRESHOLD); + if (empty_sec_num <= FDB_GC_EMPTY_SEC_THRESHOLD) { + struct gc_cb_args arg = { db, free_size, empty_sec_addr }; sector_iterator(db, §or, FDB_SECTOR_STORE_UNUSED, &arg, NULL, do_gc, false); } @@ -1580,7 +1593,7 @@ static bool check_sec_hdr_cb(kv_sec_info_t sector, void *arg1, void *arg2) if (db->parent.not_formatable) { return true; } else { - FDB_INFO("Sector header info is incorrect. Auto format this sector (0x%08" PRIX32 ").\n", sector->addr); + FDB_DEBUG("Sector header info is incorrect. Auto format this sector (0x%08" PRIX32 ").\n", sector->addr); format_sector(db, sector->addr, SECTOR_NOT_COMBINED); } } diff --git a/tests/fdb_kvdb_tc.c b/tests/fdb_kvdb_tc.c index 5eb9319..3db65d4 100644 --- a/tests/fdb_kvdb_tc.c +++ b/tests/fdb_kvdb_tc.c @@ -41,7 +41,8 @@ struct test_kv{ char name[32]; - uint8_t value[TEST_KV_VALUE_LEN]; + char *value; + size_t value_len; uint32_t addr; uint32_t saved_data_size; bool is_changed; @@ -311,10 +312,11 @@ static int iter_all_kv(fdb_kvdb_t db, struct test_kv *kv_tbl, size_t len) rt_strncpy(kv_tbl[index].name, cur_kv->name, 32); kv_tbl[index].saved_data_size = data_size; kv_tbl[index].addr = cur_kv->addr.start; + kv_tbl[index].value = (char *)rt_malloc(data_size); + if (kv_tbl[index].value == NULL) + RT_ASSERT(0 && "no memory for value"); /* read data */ - fdb_blob_read((fdb_db_t)db, fdb_kv_to_blob(cur_kv, fdb_blob_make(&fdb_blob, &kv_tbl[index].value, data_size))); - uassert_true(kv_tbl[index].saved_data_size <= TEST_KV_VALUE_LEN); - + fdb_blob_read((fdb_db_t)db, fdb_kv_to_blob(cur_kv, fdb_blob_make(&fdb_blob, kv_tbl[index].value, data_size))); index++; } @@ -323,13 +325,14 @@ static int iter_all_kv(fdb_kvdb_t db, struct test_kv *kv_tbl, size_t len) static void test_save_fdb_by_kvs(const struct test_kv *kv_tbl, size_t len) { - struct fdb_blob blob_obj, * blob = &blob_obj; + struct fdb_blob blob_obj, *blob = &blob_obj; for (size_t i = 0; i < len; i++) { if (kv_tbl[i].is_changed) { - fdb_kv_set_blob(&test_kvdb, kv_tbl[i].name, fdb_blob_make(blob, kv_tbl[i].value, TEST_KV_VALUE_LEN)); + int result = fdb_kv_set_blob(&test_kvdb, kv_tbl[i].name, fdb_blob_make(blob, kv_tbl[i].value, kv_tbl[i].value_len)); + uassert_true(result == FDB_NO_ERR); } } } @@ -338,6 +341,8 @@ static void test_check_fdb_by_kvs(const struct test_kv *kv_tbl, size_t len) { static struct test_kv saved_kv_tbl[TEST_KV_MAX_NUM] = { 0 }; + RT_ASSERT(len <= FDB_ARRAY_SIZE(saved_kv_tbl)); + iter_all_kv(&test_kvdb, saved_kv_tbl, FDB_ARRAY_SIZE(saved_kv_tbl)); for (size_t i = 0, j = 0; i < len; i++) @@ -395,10 +400,10 @@ static void test_fdb_gc(void) * +---------------------------------------------------------+ */ static const struct test_kv kv_tbl[] = { - {"kv0", "0", 0, 0, 1}, - {"kv1", "1", 0, 0, 1}, - {"kv2", "2", 0, 0, 1}, - {"kv3", "3", 1, 0, 1}, + {"kv0", "0", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv3", "3", TEST_KV_VALUE_LEN, 1, 0, 1}, }; test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); @@ -431,10 +436,10 @@ static void test_fdb_gc(void) * +--------------+-------------+--------------+-------------+ */ static const struct test_kv kv_tbl[] = { - {"kv1", "1", 0, 0, 0}, - {"kv2", "2", 0, 0, 0}, - {"kv0", "00", 1, 0, 1}, - {"kv3", "33", 1, 0, 1}, + {"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 0}, + {"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 0}, + {"kv0", "00", TEST_KV_VALUE_LEN, 1, 0, 1}, + {"kv3", "33", TEST_KV_VALUE_LEN, 1, 0, 1}, }; test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); @@ -511,10 +516,10 @@ static void test_fdb_gc(void) * */ static const struct test_kv kv_tbl[] = { - {"kv0", "000", 2, 0, 1}, - {"kv1", "111", 2, 0, 1}, - {"kv2", "222", 2, 0, 1}, - {"kv3", "333", 3, 0, 1}, + {"kv0", "000", TEST_KV_VALUE_LEN, 2, 0, 1}, + {"kv1", "111", TEST_KV_VALUE_LEN, 2, 0, 1}, + {"kv2", "222", TEST_KV_VALUE_LEN, 2, 0, 1}, + {"kv3", "333", TEST_KV_VALUE_LEN, 3, 0, 1}, }; test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); @@ -611,10 +616,10 @@ static void test_fdb_gc(void) * +--------------+-------------+--------------+-------------+ */ static const struct test_kv kv_tbl[] = { - {"kv0", "0000", 3, 0, 1}, - {"kv1", "1111", 3, 0, 1}, - {"kv2", "2222", 0, 0, 1}, - {"kv3", "3333", 0, 0, 1}, + {"kv0", "0000", TEST_KV_VALUE_LEN, 3, 0, 1}, + {"kv1", "1111", TEST_KV_VALUE_LEN, 3, 0, 1}, + {"kv2", "2222", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv3", "3333", TEST_KV_VALUE_LEN, 0, 0, 1}, }; test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); @@ -625,15 +630,209 @@ static void test_fdb_gc(void) } } +static void test_fdb_gc2(void) +{ + fdb_kv_set_default(&test_kvdb); + + { + /* + * prepare1: add 4 KVs + * + * +---------------------------------------------------------+ + * | sector0 | sector1 | sector2 | sector3 | + * | using | using | empty | empty | + * +---------------------------------------------------------+ + * | | | | | + * | kv0 | kv3 | | | + * | new | new | | | + * +----------------------------+ | | + * | | | | | + * | kv1 | | | | + * | new | | | | + * +--------------+ | | | + * | | | | | + * | kv2 | | | | + * | new | | | | + * +--------------+ | | | + * | | | | | + * +---------------------------------------------------------+ + */ + static const struct test_kv kv_tbl[] = { + {"kv0", "0", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv3", "3", TEST_KV_VALUE_LEN, 1, 0, 1}, + }; + + test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0); + fdb_reboot(); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0); + } + + { + /* + * prepare2: change kv0 and kv3 + * + * +--------------+-------------+--------------+-------------+ + * | sector0 | sector1 | sector2 | sector3 | + * | using | using | empty | empty | + * +---------------------------------------------------------+ + * | | | | | + * | kv0 | kv3 | | | + * | delete | delete | | | + * +----------------------------+ | | + * | | | | | + * | kv1 | kv0 | | | + * | new | new | | | + * +----------------------------+ | | + * | | | | | + * | kv2 | kv3 | | | + * | new | new | | | + * +----------------------------+ | | + * | | | | | + * +--------------+-------------+--------------+-------------+ + */ + static const struct test_kv kv_tbl[] = { + {"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 0}, + {"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 0}, + {"kv0", "00", TEST_KV_VALUE_LEN, 1, 0, 1}, + {"kv3", "33", TEST_KV_VALUE_LEN, 1, 0, 1}, + }; + + test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0); + fdb_reboot(); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0); + } + + { + /* + * prepare3: add big kv4 + * + * +--------------+-------------+--------------+-------------+ + * | sector0 | sector1 | sector2 | sector3 | + * | using | using | using | empty | + * +---------------------------------------------------------+ + * | | | | | + * | kv0 | kv3 | | | + * | delete | delete | | | + * +----------------------------+ kv4 | | + * | | | new | | + * | kv1 | kv0 | | | + * | new | new | | | + * +----------------------------+ | | + * | | | | | + * | kv2 | kv3 |--------------| | + * | new | new | | | + * +----------------------------+ | | + * | | | | | + * +--------------+-------------+--------------+-------------+ + */ + static const struct test_kv kv_tbl[] = { + {"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 0}, + {"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 0}, + {"kv0", "00", TEST_KV_VALUE_LEN, 1, 0, 0}, + {"kv3", "33", TEST_KV_VALUE_LEN, 1, 0, 0}, + {"kv4", "4", TEST_KV_VALUE_LEN * 3, 2, 0, 1}, + }; + + test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0); + fdb_reboot(); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 0); + } + + { + /* + * add kv5, trigger GC + + * step1: move kv1 and kv2 + * +--------------+-------------+--------------+-------------+ + * | sector0 | sector1 | sector2 | sector3 | + * | empty | using | using | using | + * +---------------------------------------------------------+ + * | | | | | + * | | kv3 | | kv1 | + * | | delete | | new | + * | +-------------+ kv4 |-------------| + * | | | new | | + * | | kv0 | | kv2 | + * | | new | | new | + * | +-------------+ |-------------| + * | | | | | + * | | kv3 |--------------| | + * | | new | | | + * | +-------------+ | | + * | | | | | + * +--------------+-------------+--------------+-------------+ + * + * step2: move kv0 and kv3 + * +--------------+-------------+--------------+-------------+ + * | sector0 | sector1 | sector2 | sector3 | + * | using | empty | using | using | + * +---------------------------------------------------------+ + * | | | | | + * | kv3 | | | kv1 | + * | new | | | new | + * +--------------+ | kv4 |-------------| + * | | | new | | + * | | | | kv2 | + * | | | | new | + * | | | |-------------| + * | | | | | + * | | |--------------| kv0 | + * | | | | new | + * | | | |-------------| + * | | | | | + * +--------------+-------------+--------------+-------------+ + * + * step3: add kv5 + * +--------------+-------------+--------------+-------------+ + * | sector0 | sector1 | sector2 | sector3 | + * | using | empty | using | using | + * +---------------------------------------------------------+ + * | | | | | + * | kv3 | | | kv1 | + * | new | | | new | + * +--------------+ | |-------------| + * | | | kv4 | | + * | | | new | kv2 | + * | kv5 | | | new | + * | new | | |-------------| + * | | | | | + * | | |--------------| kv0 | + * |--------------| | | new | + * | | | |-------------| + * | | | | | + * +--------------+-------------+--------------+-------------+ + * + */ + static const struct test_kv kv_tbl[] = { + {"kv3", "33", TEST_KV_VALUE_LEN, 0, 0, 0}, + {"kv5", "5", TEST_KV_VALUE_LEN * 2, 0, 0, 1}, + {"kv4", "4", TEST_KV_VALUE_LEN * 3, 2, 0, 0}, + {"kv1", "1", TEST_KV_VALUE_LEN, 3, 0, 0}, + {"kv2", "2", TEST_KV_VALUE_LEN, 3, 0, 0}, + {"kv0", "00", TEST_KV_VALUE_LEN, 3, 0, 0}, + }; + + test_fdb_by_kvs(kv_tbl, FDB_ARRAY_SIZE(kv_tbl)); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 2); + fdb_reboot(); + uassert_true(RT_ALIGN_DOWN(test_kvdb.parent.oldest_addr, TEST_KVDB_SECTOR_SIZE) == TEST_KVDB_SECTOR_SIZE * 2); + } +} + static void test_fdb_scale_up(void) { fdb_kv_set_default(&test_kvdb); static const struct test_kv old_kv_tbl[] = { - {"kv0", "0", 0, 0, 1}, - {"kv1", "1", 0, 0, 1}, - {"kv2", "2", 0, 0, 1}, - {"kv3", "3", 1, 0, 1}, + {"kv0", "0", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv1", "1", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv2", "2", TEST_KV_VALUE_LEN, 0, 0, 1}, + {"kv3", "3", TEST_KV_VALUE_LEN, 1, 0, 1}, }; /* save some data */ test_save_fdb_by_kvs(old_kv_tbl, FDB_ARRAY_SIZE(old_kv_tbl)); @@ -647,10 +846,10 @@ static void test_fdb_scale_up(void) /* save some new data */ static const struct test_kv new_kv_tbl[] = { - {"kv4", "4", 4, 0, 1}, - {"kv5", "5", 4, 0, 1}, - {"kv6", "6", 4, 0, 1}, - {"kv7", "7", 5, 0, 1}, + {"kv4", "4", TEST_KV_VALUE_LEN, 4, 0, 1}, + {"kv5", "5", TEST_KV_VALUE_LEN, 4, 0, 1}, + {"kv6", "6", TEST_KV_VALUE_LEN, 4, 0, 1}, + {"kv7", "7", TEST_KV_VALUE_LEN, 5, 0, 1}, }; /* kv4: sector1, kv5: sector1, kv6: sector2, kv7: sector2 */ test_save_fdb_by_kvs(new_kv_tbl, FDB_ARRAY_SIZE(new_kv_tbl)); @@ -698,6 +897,7 @@ static void testcase(void) UTEST_UNIT_RUN(test_fdb_change_kv); UTEST_UNIT_RUN(test_fdb_del_kv); UTEST_UNIT_RUN(test_fdb_gc); + UTEST_UNIT_RUN(test_fdb_gc2); UTEST_UNIT_RUN(test_fdb_scale_up); UTEST_UNIT_RUN(test_fdb_kvdb_set_default); UTEST_UNIT_RUN(test_fdb_kvdb_deinit);