PowLu/TencentOS-tiny
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kv gc bugfix

Browse Source 
if a power down or crash happened during gc process,  kv_init will fail with NO_WRITEABLE_BLOK next time.
This commit is contained in:
Arthur
2021-08-03 12:19:19 +08:00
parent fb7715a3c5
commit cb36ebae84
3 changed files with 215 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
components/fs/kv/include/tos_kv.h
View File

@@ -101,7 +101,21 @@ typedef uint64_t kv_dword_t; // double word
#define KV_ITEM_ADDR_OF_BODY(item) (item->pos + KV_ITEM_HDR_SIZE)
#define KV_BLK_HDR_MAGIC 0x1234ABCD4321DCBA
#define KV_BLK_HDR_GC_SRC 0x5678DCBA8765ABCD
#define KV_BLK_HDR_GC_DST 0x8765ABCD8765DCBA
#define KV_BLK_HDR_GC_DONE 0x4321DCBA1234ABCD
#define KV_BLK_IS_LEGAL(blk_hdr) ((blk_hdr)->magic == KV_BLK_HDR_MAGIC)
// DO NOT use gc_src == KV_BLK_HDR_GC_SRC here, in case of an incomplete write of src magic due to a power down
#define KV_BLK_IS_GC_SRC(blk_hdr) ((blk_hdr)->gc_src != (kv_dword_t)-1)
// DO NOT use gc_dst == KV_BLK_HDR_GC_DST here, in case of an incomplete write of dst magic due to a power down
#define KV_BLK_IS_GC_DST(blk_hdr) ((blk_hdr)->gc_dst != (kv_dword_t)-1)
// DO NOT use gc_done == KV_BLK_HDR_GC_DONE here, in case of an incomplete write of done magic due to a power down
#define KV_BLK_IS_GC_DONE(blk_hdr) ((blk_hdr)->gc_done != (kv_dword_t)-1)
#define KV_BLK_IS_GC_DST_NOT_DONE(blk_hdr) (KV_BLK_IS_GC_DST(blk_hdr) && !KV_BLK_IS_GC_DONE(blk_hdr))
#define KV_BLK_INVALID ((uint32_t)-1)
#define KV_BLK_HDR_SIZE KV_ALIGNED_SIZE(sizeof(kv_blk_hdr_t))
#define KV_BLK_SIZE (KV_FLASH_SECTOR_SIZE)
@@ -113,7 +127,7 @@ typedef uint64_t kv_dword_t; // double word
#define KV_BLK_NEXT(blk_start) (blk_start + KV_BLK_SIZE >= KV_FLASH_END ? KV_FLASH_START : blk_start + KV_BLK_SIZE)
#define KV_BLK_FOR_EACH_FROM(cur_blk, start_blk) \
for (cur_blk = KV_BLK_NEXT(start_blk); \
for (cur_blk = KV_BLK_NEXT(start_blk); \
cur_blk != start_blk; \
cur_blk = KV_BLK_NEXT(cur_blk))
@@ -159,6 +173,9 @@ typedef struct kv_control_st {
typedef struct kv_block_header_st {
kv_wunit_t magic; /*< is this block formatted? */
kv_wunit_t gc_src; /*< is this block gc-ing(as a source block)? */
kv_wunit_t gc_dst; /*< is this block gc-ing(as a destination block)? */
kv_wunit_t gc_done; /*< is this block gc done(as a destination block)? */
} __PACKED__ kv_blk_hdr_t;
typedef struct kv_item_header_st {

226
components/fs/kv/tos_kv.c
View File

@@ -213,6 +213,36 @@ __STATIC__ uint32_t kv_blk_search_suitable(uint32_t item_size)
} while (K_TRUE);
}
__STATIC__ kv_err_t kv_blk_mark_gc_src(uint32_t blk_start)
{
if (kv_flash_wunit_modify(KV_ADDR_OF_FIELD(blk_start, kv_blk_hdr_t, gc_src),
KV_BLK_HDR_GC_SRC) != KV_ERR_NONE) {
return KV_ERR_FLASH_WRITE_FAILED;
}
return KV_ERR_NONE;
}
__STATIC__ kv_err_t kv_blk_mark_gc_dst(uint32_t blk_start)
{
if (kv_flash_wunit_modify(KV_ADDR_OF_FIELD(blk_start, kv_blk_hdr_t, gc_dst),
KV_BLK_HDR_GC_DST) != KV_ERR_NONE) {
return KV_ERR_FLASH_WRITE_FAILED;
}
return KV_ERR_NONE;
}
__STATIC__ kv_err_t kv_blk_mark_gc_done(uint32_t blk_start)
{
if (kv_flash_wunit_modify(KV_ADDR_OF_FIELD(blk_start, kv_blk_hdr_t, gc_done),
KV_BLK_HDR_GC_DONE) != KV_ERR_NONE) {
return KV_ERR_FLASH_WRITE_FAILED;
}
return KV_ERR_NONE;
}
__STATIC__ kv_err_t kv_item_hdr_write(uint32_t item_start, kv_item_hdr_t *item_hdr)
{
if (kv_flash_write(KV_ADDR_OF_FIELD(item_start, kv_item_hdr_t, checksum),
@@ -337,11 +367,13 @@ __STATIC__ kv_err_t kv_item_try_delete(kv_item_t *item)
}
// key changes, means another turn of gc happened, the previous block is filled with new item.
if (memcmp(prev_key, (void *)KV_ITEM_ADDR_OF_BODY(item), k_len) != 0) {
if (memcmp(prev_key, (void *)item->body, k_len) != 0) {
tos_mmheap_free(prev_key);
return KV_ERR_NONE;
}
tos_mmheap_free(prev_key);
// the previous item is still there, delete it.
return kv_item_delete_aux(prev_pos);
}
@@ -436,23 +468,24 @@ __STATIC__ int kv_item_is_moved(kv_item_t *item)
return is_moved;
}
__STATIC__ kv_err_t kv_item_do_gc(kv_item_t *item, const void *dummy)
__STATIC__ kv_err_t kv_item_do_gc(kv_item_t *item, const void *p_blk_dst)
{
kv_err_t err;
uint32_t blk_dst = *(uint32_t *)p_blk_dst;
err = kv_item_body_read(item);
if (err != KV_ERR_NONE) {
return err;
}
if (kv_item_write(KV_BLK_USABLE_ADDR(KV_MGR_WORKSPACE),
if (kv_item_write(KV_BLK_USABLE_ADDR(blk_dst),
&item->hdr, item->body,
KV_ITEM_SIZE_OF_BODY(item)) != KV_ERR_NONE) {
return KV_ERR_FLASH_WRITE_FAILED;
}
// reduce the free_size
kv_blk_freesz_reduce(KV_MGR_WORKSPACE, KV_ITEM_SIZE_OF_ITEM(item));
kv_blk_freesz_reduce(blk_dst, KV_ITEM_SIZE_OF_ITEM(item));
return KV_ERR_NEXT_LOOP;
}
@@ -550,7 +583,7 @@ __STATIC__ kv_err_t kv_item_do_recovery(kv_item_t *item, const void *dummy)
*/
__STATIC__ kv_err_t kv_item_walkthru(uint32_t blk_start,
kv_item_walker_t walker,
const void *patten,
const void *arg,
kv_item_t **item_out)
{
kv_err_t err;
@@ -616,10 +649,12 @@ __STATIC__ kv_err_t kv_item_walkthru(uint32_t blk_start,
// tell the item where he is, he does not know yet.
item->pos = cur_item;
err = walker(item, patten);
err = walker(item, arg);
if (err == KV_ERR_NONE) {
if (item_out) {
*item_out = item;
} else {
kv_item_free(item);
}
return KV_ERR_NONE;
} else if (err != KV_ERR_NEXT_LOOP) {
@@ -937,11 +972,120 @@ __STATIC__ kv_err_t kv_mgr_workspace_locate(void)
return KV_ERR_NONE;
}
__STATIC__ void kv_mgr_ctl_build(void)
/*
* src dst
* 1. mark gc_src tag
* 2. mark gc_dst tag
* 3. copy data to dst
* 4. format src block
* 5. mark gc_done tag
*/
__STATIC__ kv_err_t kv_do_gc(uint32_t blk_src, uint32_t blk_dst, int is_handle_incomplete)
{
kv_err_t err;
// step 1
if (!is_handle_incomplete) {
// in handle incomplete case, gc_src already been marked before power down
err = kv_blk_mark_gc_src(blk_src);
if (err != KV_ERR_NONE) {
return err;
}
}
// step 2
err = kv_blk_mark_gc_dst(blk_dst);
if (err != KV_ERR_NONE) {
return err;
}
// step 3
err = kv_item_walkthru(blk_src, kv_item_do_gc, (const void *)&blk_dst, K_NULL);
if (err != KV_ERR_NONE) {
return err;
}
// step 4
kv_blk_reset_inuse(blk_src);
if (kv_blk_format(blk_src) != KV_ERR_NONE) {
kv_blk_set_bad(blk_src);
}
// step 5
err = kv_blk_mark_gc_done(blk_dst);
if (err != KV_ERR_NONE) {
return err;
}
kv_blk_reset_fresh(blk_dst);
if (!kv_blk_is_full(blk_dst)) {
kv_blk_set_inuse(blk_dst);
}
return KV_ERR_NONE;
}
struct blk_info {
int nr;
#define NR_ABNORMAL_BLK_MAX 3
uint32_t blks[NR_ABNORMAL_BLK_MAX];
};
/*
* a power down may happen during any process of a gc action, we should fix
* all incomplete gc when kv bootup.
*/
__STATIC__ int kv_handle_incomplete_gc(struct blk_info gc_src_blk, struct blk_info gc_dst_not_done_blk, uint32_t fresh_blk)
{
kv_err_t err;
uint32_t src_blk, dst_blk;
if (gc_src_blk.nr > 1 || gc_dst_not_done_blk.nr > 1) {
printf("warning: more than one gc_src[%d] or gc_dst_not_done[%d] block\n", gc_src_blk.nr, gc_dst_not_done_blk.nr);
return KV_ERR_BLK_STATUS_ERROR;
}
if (gc_src_blk.nr == 0) {
return KV_ERR_NONE;
}
src_blk = gc_src_blk.blks[0];
if (gc_dst_not_done_blk.nr == 0) {
if (fresh_blk == (uint32_t)-1) {
printf("warning: no gc_dst_not_done block, neither fresh block\n");
return KV_ERR_BLK_STATUS_ERROR;
} else {
printf("info: no gc_dst_not_done block, choose a fresh block[0x%x]\n", fresh_blk);
dst_blk = fresh_blk;
}
} else {
dst_blk = gc_dst_not_done_blk.blks[0];
/* format dst_blk first(make it fresh), for an in-complete gc happends in a very small
probability, for a clean code, we just do a totally retry here. */
err = kv_blk_format(dst_blk);
if (err != KV_ERR_NONE) {
kv_blk_set_bad(dst_blk);
return err;
}
}
return kv_do_gc(src_blk, dst_blk, K_TRUE);
}
__STATIC__ int kv_mgr_ctl_build(void)
{
kv_err_t err;
uint32_t cur_blk;
kv_blk_hdr_t blk_hdr;
/* theoretically, should at least only one gc_src/gc_dst_not_done block */
uint32_t fresh_blk = (uint32_t)-1;
struct blk_info gc_src_blk = { 0 }, gc_dst_not_done_blk = { 0 };
KV_BLK_FOR_EACH(cur_blk) {
if (kv_blk_hdr_read(cur_blk, &blk_hdr) != KV_ERR_NONE) {
// sth must be wrong seriously with this block
@@ -953,18 +1097,36 @@ __STATIC__ void kv_mgr_ctl_build(void)
if (kv_blk_format(cur_blk) != KV_ERR_NONE) {
// sth must be wrong seriously with this block
kv_blk_set_bad(cur_blk);
} else {
fresh_blk = cur_blk;
}
// we get a fresh block
continue;
}
if (KV_BLK_IS_GC_SRC(&blk_hdr)) {
gc_src_blk.blks[gc_src_blk.nr++] = cur_blk;
continue;
}
if (KV_BLK_IS_GC_DST_NOT_DONE(&blk_hdr)) {
gc_dst_not_done_blk.blks[gc_dst_not_done_blk.nr++] = cur_blk;
continue
}
// do index building
if (kv_mgr_index_build(cur_blk) != KV_ERR_NONE) {
// sth goes wrong while index building, we give it a mark
kv_blk_set_hanging(cur_blk);
continue;
}
if (kv_blk_is_fresh(cur_blk)) {
fresh_blk = cur_blk;
}
}
return kv_handle_incomplete_gc(gc_src_blk, gc_dst_not_done_blk, fresh_blk);
}
__STATIC__ kv_err_t kv_mgr_ctl_init(void)
@@ -997,11 +1159,6 @@ __STATIC__ void kv_mgr_ctl_deinit(void)
memset(&kv_ctl, 0, sizeof(kv_ctl));
}
__STATIC__ kv_err_t kv_do_gc(uint32_t dirty_blk)
{
return kv_item_walkthru(dirty_blk, kv_item_do_gc, K_NULL, K_NULL);
}
/* on each turn of gc, we free some discarded items in the dirty block.
so we get more space to save the new item.
gc should be done only when necessary, if there is no sitiation of an item too big to save,
@@ -1009,7 +1166,7 @@ __STATIC__ kv_err_t kv_do_gc(uint32_t dirty_blk)
*/
__STATIC__ kv_err_t kv_gc(void)
{
uint32_t cur_blk, workspace_backup;
uint32_t cur_blk, blk_dst;
int is_gc_done = K_FALSE, is_rebuild_done = K_FALSE;
/* we give blocks with KV_BLK_FLAG_HANGING a chance to rebuild index */
@@ -1017,39 +1174,25 @@ __STATIC__ kv_err_t kv_gc(void)
is_rebuild_done = kv_mgr_blk_index_rebuild();
}
workspace_backup = KV_MGR_WORKSPACE;
// there must be at least one fresh block left, make workspace pointer to the fresh one
KV_MGR_WORKSPACE = kv_blk_next_fresh();
blk_dst = kv_blk_next_fresh();
KV_BLK_FOR_EACH(cur_blk) {
if (kv_blk_is_dirty(cur_blk)) {
if (kv_do_gc(cur_blk) != KV_ERR_NONE) {
if (kv_do_gc(cur_blk, blk_dst, K_FALSE) != KV_ERR_NONE) {
// cannot do gc for this block, give others a try
continue;
}
kv_blk_reset_inuse(cur_blk);
if (kv_blk_format(cur_blk) != KV_ERR_NONE) {
kv_blk_set_bad(cur_blk);
}
kv_blk_reset_fresh(KV_MGR_WORKSPACE);
if (!kv_blk_is_full(KV_MGR_WORKSPACE)) {
kv_blk_set_inuse(KV_MGR_WORKSPACE);
}
is_gc_done = K_TRUE;
break;
}
}
if (!is_gc_done) {
if (is_gc_done) {
// if do nothing, should restore the workspace;
KV_MGR_WORKSPACE = workspace_backup;
KV_MGR_WORKSPACE = blk_dst;
}
return (is_gc_done || is_rebuild_done) ? KV_ERR_NONE : KV_ERR_GC_NOTHING;
@@ -1185,6 +1328,22 @@ __DEBUG__ kv_err_t tos_kv_walkthru(void)
continue;
}
if (KV_BLK_IS_GC_SRC(&blk_hdr)) {
printf("block is gc-src\n");
}
if (KV_BLK_IS_GC_DST(&blk_hdr)) {
printf("block is gc-dst\n");
}
if (KV_BLK_IS_GC_DONE(&blk_hdr)) {
printf("block is gc-done\n");
}
if (KV_BLK_IS_GC_DST_NOT_DONE(&blk_hdr)) {
printf("block is gc-dst but not done\n");
}
if (kv_block_walkthru(cur_blk) != KV_ERR_NONE) {
printf("block diagnosis failed\n");
continue;
@@ -1213,7 +1372,10 @@ __API__ kv_err_t tos_kv_init(uint32_t flash_start, uint32_t flash_end, kv_flash_
return err;
}
kv_mgr_ctl_build();
err = kv_mgr_ctl_build();
if (err != KV_ERR_NONE) {
return err;
}
return kv_mgr_workspace_locate();
}

5
kernel/core/tos_mmheap.c
View File

@@ -496,7 +496,7 @@ __STATIC__ size_t adjust_request_size(size_t size, size_t align)
}
adjust_size = align_up(size, align);
if ((adjust_size > K_MMHEAP_BLK_SIZE_MAX)||(!adjust_size)) {
if (!adjust_size || adjust_size > K_MMHEAP_BLK_SIZE_MAX) {
return 0;
}
@@ -590,9 +590,10 @@ __API__ void *tos_mmheap_alloc(size_t size)
size_t adjust_size;
mmheap_blk_t *blk;
if (size>K_MMHEAP_BLK_SIZE_MAX) {
if (size > K_MMHEAP_BLK_SIZE_MAX) {
return K_NULL;
}
adjust_size = adjust_request_size(size, K_MMHEAP_ALIGN_SIZE);
blk = blk_locate_free(adjust_size);
if (!blk) {