From a2e2f2378a86066567e44f6afbb991ab92684d78 Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Fri, 6 Dec 2024 15:58:25 +0800 Subject: [PATCH 1/9] wip(map): map iterator Signed-off-by: Haobo Gu --- Cargo.toml | 2 ++ src/map.rs | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 7c704e7..0eec6ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,8 @@ std = [] arrayvec = ["dep:arrayvec"] alloc = [] heapless = ["dep:heapless"] +# Enable the raw access of items. This feature is NOT bounded by the semver. +raw_access = [] _test = ["dep:futures", "dep:approx", "std", "arrayvec", "alloc", "heapless"] [lints.rust] diff --git a/src/map.rs b/src/map.rs index bf0c751..d2f0903 100644 --- a/src/map.rs +++ b/src/map.rs @@ -97,8 +97,12 @@ //! For your convenience there are premade implementations for the [Key] and [Value] traits. //! -use core::mem::{size_of, MaybeUninit}; +use core::{ + marker::PhantomData, + mem::{size_of, MaybeUninit}, +}; +use cache::NoCache; use embedded_storage_async::nor_flash::MultiwriteNorFlash; use crate::item::{find_next_free_item_spot, Item, ItemHeader, ItemIter}; @@ -110,6 +114,90 @@ use self::{ use super::*; +/// TODO: 1. gated by feature, 2. remove unwraps, 3. documentations +pub struct MapItemIter<'d, K: Key + 'd, V: Value<'d> + 'd, S: NorFlash> { + flash: &'d mut S, + flash_range: Range, + data_buffer: &'d mut [u8], + current_page_index: usize, + current_iter: ItemIter, + p: PhantomData<(K, V)>, +} + +impl<'d, K: Key + 'd, V: Value<'d> + 'd, S: NorFlash> MapItemIter<'d, K, V, S> { + /// TODO: doc + pub async fn next( + &mut self, + data_buffer: &'d mut [u8], + ) -> Result, Error> { + loop { + let item_result = self.current_iter.next(self.flash, self.data_buffer).await?; + match item_result { + Some((item, _)) => { + let (key, _) = K::deserialize_from(item.data())?; + let (_, buf) = item.destruct(); + // FIXME: Have to use a different data buffer, because the item is borrowed + data_buffer.copy_from_slice(buf); + return Ok(Some((key, data_buffer))); + } + None => { + // Current iter ends, try to create next iter for next page + self.current_page_index += 1; + + // FIXME: The last page, does it mean the iterator ends? + if self.current_page_index >= self.flash_range.len() / S::ERASE_SIZE { + return Ok(None); + } + + // Find next valid page iter + match get_page_state::( + self.flash, + self.flash_range.clone(), + &mut NoCache::new(), + self.current_page_index, + ) + .await + { + Ok(PageState::Closed) | Ok(PageState::PartialOpen) => { + self.current_iter = ItemIter::new( + calculate_page_address::( + self.flash_range.clone(), + self.current_page_index, + ), + calculate_page_end_address::( + self.flash_range.clone(), + self.current_page_index, + ), + ); + continue; + } + _ => return Ok(None), + } + } + } + } + } +} + +/// Get an iterator that iterates over all non-erased & non-corrupted items in the flash. +pub async fn get_items<'d, K: Key + 'd, V: Value<'d> + 'd, S: NorFlash>( + flash: &'d mut S, + flash_range: Range, + data_buffer: &'d mut [u8], +) -> Result, ()> { + Ok(MapItemIter { + flash, + flash_range: flash_range.clone(), + data_buffer, + current_page_index: 0, + current_iter: ItemIter::new( + calculate_page_address::(flash_range.clone(), 0), + calculate_page_end_address::(flash_range.clone(), 0), + ), + p: PhantomData, + }) +} + /// Get the last stored value from the flash that is associated with the given key. /// If no value with the key is found, None is returned. /// From a05daa20fcb99c1696b40a22325079ddf419338b Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Wed, 11 Dec 2024 16:11:44 +0800 Subject: [PATCH 2/9] feat(map): finish the map iterator Signed-off-by: Haobo Gu --- src/map.rs | 131 +++++++++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/src/map.rs b/src/map.rs index d2f0903..96010a0 100644 --- a/src/map.rs +++ b/src/map.rs @@ -97,10 +97,7 @@ //! For your convenience there are premade implementations for the [Key] and [Value] traits. //! -use core::{ - marker::PhantomData, - mem::{size_of, MaybeUninit}, -}; +use core::mem::{size_of, MaybeUninit}; use cache::NoCache; use embedded_storage_async::nor_flash::MultiwriteNorFlash; @@ -114,87 +111,101 @@ use self::{ use super::*; -/// TODO: 1. gated by feature, 2. remove unwraps, 3. documentations -pub struct MapItemIter<'d, K: Key + 'd, V: Value<'d> + 'd, S: NorFlash> { +/// Iterator which iterates all valid items +pub struct MapItemIter<'d, S: NorFlash> { flash: &'d mut S, flash_range: Range, - data_buffer: &'d mut [u8], current_page_index: usize, current_iter: ItemIter, - p: PhantomData<(K, V)>, } -impl<'d, K: Key + 'd, V: Value<'d> + 'd, S: NorFlash> MapItemIter<'d, K, V, S> { - /// TODO: doc - pub async fn next( +impl<'d, S: NorFlash> MapItemIter<'d, S> { + /// Get the next item in the iterator. Be careful that the given `data_buffer` should large enough to contain the serialized key and value. + pub async fn next<'a, K: Key, V: Value<'a>>( &mut self, - data_buffer: &'d mut [u8], - ) -> Result, Error> { - loop { - let item_result = self.current_iter.next(self.flash, self.data_buffer).await?; - match item_result { - Some((item, _)) => { - let (key, _) = K::deserialize_from(item.data())?; - let (_, buf) = item.destruct(); - // FIXME: Have to use a different data buffer, because the item is borrowed - data_buffer.copy_from_slice(buf); - return Ok(Some((key, data_buffer))); - } - None => { - // Current iter ends, try to create next iter for next page - self.current_page_index += 1; + data_buffer: &'a mut [u8], + ) -> Result, Error> { + // Find the next item + let item = loop { + if let Some((item, _address)) = self.current_iter.next(self.flash, data_buffer).await? { + // We've found the next item, quit the loop + break item; + } - // FIXME: The last page, does it mean the iterator ends? - if self.current_page_index >= self.flash_range.len() / S::ERASE_SIZE { - return Ok(None); - } + // The current page is done, we need to find the next page + // Find next page which is not open, update `self.current_iter` + loop { + self.current_page_index = + next_page::(self.flash_range.clone(), self.current_page_index); + + // All pages are checked, there's nothing left so we return None + if self.current_page_index >= self.flash_range.len() / S::ERASE_SIZE + || self.current_page_index == 0 + { + return Ok(None); + } - // Find next valid page iter - match get_page_state::( - self.flash, - self.flash_range.clone(), - &mut NoCache::new(), - self.current_page_index, - ) - .await - { - Ok(PageState::Closed) | Ok(PageState::PartialOpen) => { - self.current_iter = ItemIter::new( - calculate_page_address::( - self.flash_range.clone(), - self.current_page_index, - ), - calculate_page_end_address::( - self.flash_range.clone(), - self.current_page_index, - ), - ); - continue; - } - _ => return Ok(None), + match get_page_state::( + self.flash, + self.flash_range.clone(), + &mut NoCache::new(), + self.current_page_index, + ) + .await + { + Ok(PageState::Closed) | Ok(PageState::PartialOpen) => { + self.current_iter = ItemIter::new( + calculate_page_address::( + self.flash_range.clone(), + self.current_page_index, + ), + calculate_page_end_address::( + self.flash_range.clone(), + self.current_page_index, + ), + ); + break; } + _ => continue, } } - } + }; + + let data_len = item.header.length as usize; + let (key, key_len) = K::deserialize_from(item.data())?; + + Ok(Some(( + key, + V::deserialize_from(&data_buffer[key_len..][..data_len - key_len]) + .map_err(Error::SerializationError)?, + ))) } } -/// Get an iterator that iterates over all non-erased & non-corrupted items in the flash. -pub async fn get_items<'d, K: Key + 'd, V: Value<'d> + 'd, S: NorFlash>( +/// Get an iterator that iterates over all non-erased & non-corrupted items in the map. +pub async fn get_next_item<'d, 'a, K: Key, V: Value<'a>, S: NorFlash>( + data_buffer: &'a mut [u8], + iter: &mut MapItemIter<'d, S>, +) -> Result, Error> +where + 'd: 'a, +{ + iter.next(data_buffer).await +} + +/// Get an iterator that iterates over all non-erased & non-corrupted items in the map. +pub fn get_item_iter<'d, S: NorFlash>( flash: &'d mut S, flash_range: Range, - data_buffer: &'d mut [u8], -) -> Result, ()> { +) -> Result, ()> { Ok(MapItemIter { flash, flash_range: flash_range.clone(), - data_buffer, current_page_index: 0, current_iter: ItemIter::new( calculate_page_address::(flash_range.clone(), 0), calculate_page_end_address::(flash_range.clone(), 0), ), - p: PhantomData, }) } From 0b0dc63a77b5531f1266742f4bdddf18d5d48846 Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Wed, 11 Dec 2024 16:22:59 +0800 Subject: [PATCH 3/9] refactor(map): clean map iterator api, add docs Signed-off-by: Haobo Gu --- Cargo.toml | 2 -- src/map.rs | 39 ++++++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0eec6ec..7c704e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,8 +31,6 @@ std = [] arrayvec = ["dep:arrayvec"] alloc = [] heapless = ["dep:heapless"] -# Enable the raw access of items. This feature is NOT bounded by the semver. -raw_access = [] _test = ["dep:futures", "dep:approx", "std", "arrayvec", "alloc", "heapless"] [lints.rust] diff --git a/src/map.rs b/src/map.rs index 96010a0..d426fc3 100644 --- a/src/map.rs +++ b/src/map.rs @@ -111,7 +111,27 @@ use self::{ use super::*; -/// Iterator which iterates all valid items +/// Iterator which iterates all non-erased & non-corrupted items in the map. +/// +/// The iterator will return the (Key, Value) tuple when calling `next()`. +/// If the iterator ends, it will return `None`. +/// +/// The following is a simple example of how to use the iterator: +/// ```rust +/// // Create the iterator +/// let mut iterator = get_item_iter(&mut flash, flash_range.clone()); +/// +/// // Iterate through all items +/// loop { +/// // Suppose the Key and Value types are u8, u32 +/// if let Ok(Some(item)) = iterator.next::(&mut buffer).await { +/// // Do something with the item +/// } else { +/// // Iterator ends +/// break; +/// } +/// } +/// ``` pub struct MapItemIter<'d, S: NorFlash> { flash: &'d mut S, flash_range: Range, @@ -182,23 +202,12 @@ impl<'d, S: NorFlash> MapItemIter<'d, S> { } } -/// Get an iterator that iterates over all non-erased & non-corrupted items in the map. -pub async fn get_next_item<'d, 'a, K: Key, V: Value<'a>, S: NorFlash>( - data_buffer: &'a mut [u8], - iter: &mut MapItemIter<'d, S>, -) -> Result, Error> -where - 'd: 'a, -{ - iter.next(data_buffer).await -} - /// Get an iterator that iterates over all non-erased & non-corrupted items in the map. pub fn get_item_iter<'d, S: NorFlash>( flash: &'d mut S, flash_range: Range, -) -> Result, ()> { - Ok(MapItemIter { +) -> MapItemIter<'d, S> { + MapItemIter { flash, flash_range: flash_range.clone(), current_page_index: 0, @@ -206,7 +215,7 @@ pub fn get_item_iter<'d, S: NorFlash>( calculate_page_address::(flash_range.clone(), 0), calculate_page_end_address::(flash_range.clone(), 0), ), - }) + } } /// Get the last stored value from the flash that is associated with the given key. From c09208cee37a0bcd4167da24a3691c08219a1373 Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Thu, 12 Dec 2024 14:08:56 +0800 Subject: [PATCH 4/9] feat(map): fix PR comments, add test Signed-off-by: Haobo Gu --- src/map.rs | 122 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 103 insertions(+), 19 deletions(-) diff --git a/src/map.rs b/src/map.rs index d426fc3..2d0c270 100644 --- a/src/map.rs +++ b/src/map.rs @@ -99,7 +99,7 @@ use core::mem::{size_of, MaybeUninit}; -use cache::NoCache; +use cache::CacheImpl; use embedded_storage_async::nor_flash::MultiwriteNorFlash; use crate::item::{find_next_free_item_spot, Item, ItemHeader, ItemIter}; @@ -119,7 +119,7 @@ use super::*; /// The following is a simple example of how to use the iterator: /// ```rust /// // Create the iterator -/// let mut iterator = get_item_iter(&mut flash, flash_range.clone()); +/// let mut iterator = get_item_iter(&mut flash, flash_range.clone(), &mut cache).await; /// /// // Iterate through all items /// loop { @@ -132,14 +132,16 @@ use super::*; /// } /// } /// ``` -pub struct MapItemIter<'d, S: NorFlash> { +pub struct MapItemIter<'d, 'c, S: NorFlash, CI: CacheImpl> { flash: &'d mut S, flash_range: Range, + first_page: usize, + cache: &'c mut CI, current_page_index: usize, - current_iter: ItemIter, + pub(crate) current_iter: ItemIter, } -impl<'d, S: NorFlash> MapItemIter<'d, S> { +impl<'d, 'c, S: NorFlash, CI: CacheImpl> MapItemIter<'d, 'c, S, CI> { /// Get the next item in the iterator. Be careful that the given `data_buffer` should large enough to contain the serialized key and value. pub async fn next<'a, K: Key, V: Value<'a>>( &mut self, @@ -149,6 +151,7 @@ impl<'d, S: NorFlash> MapItemIter<'d, S> { let item = loop { if let Some((item, _address)) = self.current_iter.next(self.flash, data_buffer).await? { // We've found the next item, quit the loop + break item; } @@ -158,17 +161,15 @@ impl<'d, S: NorFlash> MapItemIter<'d, S> { self.current_page_index = next_page::(self.flash_range.clone(), self.current_page_index); - // All pages are checked, there's nothing left so we return None - if self.current_page_index >= self.flash_range.len() / S::ERASE_SIZE - || self.current_page_index == 0 - { + // We've looped back to the first page, which means all pages are checked, there's nothing left so we return None + if self.current_page_index == self.first_page { return Ok(None); } match get_page_state::( self.flash, self.flash_range.clone(), - &mut NoCache::new(), + self.cache, self.current_page_index, ) .await @@ -178,11 +179,11 @@ impl<'d, S: NorFlash> MapItemIter<'d, S> { calculate_page_address::( self.flash_range.clone(), self.current_page_index, - ), + ) + S::WORD_SIZE as u32, calculate_page_end_address::( self.flash_range.clone(), self.current_page_index, - ), + ) - S::WORD_SIZE as u32, ); break; } @@ -203,19 +204,66 @@ impl<'d, S: NorFlash> MapItemIter<'d, S> { } /// Get an iterator that iterates over all non-erased & non-corrupted items in the map. -pub fn get_item_iter<'d, S: NorFlash>( +/// +/// Note that the returned iterator might yield items in a different order than they were stored, +/// and the only the last active item with a particular key will be yielded. +/// If the iterator returns `Ok(None)`, the iterator has ended. +pub async fn fetch_item_stream<'d, 'c, S: NorFlash, CI: CacheImpl>( flash: &'d mut S, flash_range: Range, -) -> MapItemIter<'d, S> { - MapItemIter { + cache: &'c mut CI, +) -> Result, Error> { + // Get the first page index + let first_page = match find_first_page( + flash, + flash_range.clone(), + cache, + 0, + PageState::PartialOpen, + ) + .await? + { + Some(first_page) => first_page, + None => { + // In the event that all pages are still open or the last used page was just closed, we search for the first open page. + // If the page one before that is closed, then that's the last used page. + if let Some(first_open_page) = + find_first_page(flash, flash_range.clone(), cache, 0, PageState::Open).await? + { + let previous_page = previous_page::(flash_range.clone(), first_open_page); + if get_page_state(flash, flash_range.clone(), cache, previous_page) + .await? + .is_closed() + { + previous_page + } else { + // The page before the open page is not closed, so it must be open. + // This means that all pages are open and that we don't have any items yet. + cache.unmark_dirty(); + 0 + } + } else { + // There are no open pages, so everything must be closed. + // Something is up and this should never happen. + // To recover, we will just erase all the flash. + return Err(Error::Corrupted { + #[cfg(feature = "_test")] + backtrace: std::backtrace::Backtrace::capture(), + }); + } + } + }; + Ok(MapItemIter { flash, flash_range: flash_range.clone(), - current_page_index: 0, + first_page, + cache, + current_page_index: first_page, current_iter: ItemIter::new( - calculate_page_address::(flash_range.clone(), 0), - calculate_page_end_address::(flash_range.clone(), 0), + calculate_page_address::(flash_range.clone(), first_page) + S::WORD_SIZE as u32, + calculate_page_end_address::(flash_range.clone(), first_page) - S::WORD_SIZE as u32, ), - } + }) } /// Get the last stored value from the flash that is associated with the given key. @@ -1639,4 +1687,40 @@ mod tests { Err(Error::ItemTooBig) ); } + + #[test] + async fn item_iterator() { + const UPPER_BOUND: u8 = 64; + let mut flash = MockFlashBig::default(); + let flash_range = 0x000..0x1000; + let mut cache = cache::NoCache::new(); + + let mut data_buffer = AlignedBuf([0; 128]); + + for i in 0..UPPER_BOUND { + store_item( + &mut flash, + flash_range.clone(), + &mut cache, + &mut data_buffer, + &i, + &vec![i; i as usize].as_slice(), + ) + .await + .unwrap(); + } + + let mut map_iter = fetch_item_stream(&mut flash, flash_range.clone(), &mut cache) + .await + .unwrap(); + + let mut count = 0; + while let Ok(Some((key, value))) = map_iter.next::(&mut data_buffer).await { + assert_eq!(value, vec![key; key as usize]); + count += 1; + } + + // Check total number of fetched items + assert_eq!(count, UPPER_BOUND); + } } From 037c1880c091ef6167ac16f2bf0fa284ddefd11e Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Fri, 13 Dec 2024 17:44:26 +0800 Subject: [PATCH 5/9] fix(map): use the correct first page of the map for iterator Signed-off-by: Haobo Gu --- src/map.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/map.rs b/src/map.rs index 2d0c270..47ef8a2 100644 --- a/src/map.rs +++ b/src/map.rs @@ -208,12 +208,13 @@ impl<'d, 'c, S: NorFlash, CI: CacheImpl> MapItemIter<'d, 'c, S, CI> { /// Note that the returned iterator might yield items in a different order than they were stored, /// and the only the last active item with a particular key will be yielded. /// If the iterator returns `Ok(None)`, the iterator has ended. -pub async fn fetch_item_stream<'d, 'c, S: NorFlash, CI: CacheImpl>( +pub async fn fetch_all_items<'d, 'c, S: NorFlash, CI: CacheImpl>( flash: &'d mut S, flash_range: Range, cache: &'c mut CI, ) -> Result, Error> { - // Get the first page index + // Get the first page index. + // The first page used by the map is the next page of the `PartialOpen` page or the last `Closed` page let first_page = match find_first_page( flash, flash_range.clone(), @@ -223,7 +224,10 @@ pub async fn fetch_item_stream<'d, 'c, S: NorFlash, CI: CacheImpl>( ) .await? { - Some(first_page) => first_page, + Some(last_used_page) => { + // The next page of the `PartialOpen` page is the first page + next_page::(flash_range.clone(), last_used_page) + }, None => { // In the event that all pages are still open or the last used page was just closed, we search for the first open page. // If the page one before that is closed, then that's the last used page. @@ -235,7 +239,8 @@ pub async fn fetch_item_stream<'d, 'c, S: NorFlash, CI: CacheImpl>( .await? .is_closed() { - previous_page + // The previous page is closed, so the first_open_page is what we want + first_open_page } else { // The page before the open page is not closed, so it must be open. // This means that all pages are open and that we don't have any items yet. @@ -1710,12 +1715,13 @@ mod tests { .unwrap(); } - let mut map_iter = fetch_item_stream(&mut flash, flash_range.clone(), &mut cache) + let mut map_iter = fetch_all_items(&mut flash, flash_range.clone(), &mut cache) .await .unwrap(); let mut count = 0; while let Ok(Some((key, value))) = map_iter.next::(&mut data_buffer).await { + println!("{}: {:?}", key, value); assert_eq!(value, vec![key; key as usize]); count += 1; } From 3fea43a4c905e77c9bb4338b5cd2541d1b6f57f6 Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Fri, 13 Dec 2024 17:48:39 +0800 Subject: [PATCH 6/9] doc(map): update the reminder Signed-off-by: Haobo Gu --- src/map.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/map.rs b/src/map.rs index 47ef8a2..e57d757 100644 --- a/src/map.rs +++ b/src/map.rs @@ -205,8 +205,11 @@ impl<'d, 'c, S: NorFlash, CI: CacheImpl> MapItemIter<'d, 'c, S, CI> { /// Get an iterator that iterates over all non-erased & non-corrupted items in the map. /// -/// Note that the returned iterator might yield items in a different order than they were stored, -/// and the only the last active item with a particular key will be yielded. +/// Note: Because map doesn't erase the items when you insert a new one with the same key, +/// so it's possible that the iterator returns items with the same key multiple times. +/// Generally the last returned one is the `active` one. +/// You should be very careful when using the map item iterator. +/// /// If the iterator returns `Ok(None)`, the iterator has ended. pub async fn fetch_all_items<'d, 'c, S: NorFlash, CI: CacheImpl>( flash: &'d mut S, From c8fe5f0cf831b5419fe24c6e7487981d5ec1357d Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Fri, 13 Dec 2024 19:39:08 +0800 Subject: [PATCH 7/9] feat(map): add auto repair to iterator, update test Signed-off-by: Haobo Gu --- src/map.rs | 133 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 49 deletions(-) diff --git a/src/map.rs b/src/map.rs index e57d757..612acd5 100644 --- a/src/map.rs +++ b/src/map.rs @@ -151,7 +151,6 @@ impl<'d, 'c, S: NorFlash, CI: CacheImpl> MapItemIter<'d, 'c, S, CI> { let item = loop { if let Some((item, _address)) = self.current_iter.next(self.flash, data_buffer).await? { // We've found the next item, quit the loop - break item; } @@ -209,58 +208,61 @@ impl<'d, 'c, S: NorFlash, CI: CacheImpl> MapItemIter<'d, 'c, S, CI> { /// so it's possible that the iterator returns items with the same key multiple times. /// Generally the last returned one is the `active` one. /// You should be very careful when using the map item iterator. -/// +/// /// If the iterator returns `Ok(None)`, the iterator has ended. -pub async fn fetch_all_items<'d, 'c, S: NorFlash, CI: CacheImpl>( +pub async fn fetch_all_items<'d, 'c, K: Key, S: NorFlash, CI: KeyCacheImpl>( flash: &'d mut S, flash_range: Range, cache: &'c mut CI, + data_buffer: &mut [u8], ) -> Result, Error> { // Get the first page index. // The first page used by the map is the next page of the `PartialOpen` page or the last `Closed` page - let first_page = match find_first_page( - flash, - flash_range.clone(), - cache, - 0, - PageState::PartialOpen, - ) - .await? - { - Some(last_used_page) => { - // The next page of the `PartialOpen` page is the first page - next_page::(flash_range.clone(), last_used_page) - }, - None => { - // In the event that all pages are still open or the last used page was just closed, we search for the first open page. - // If the page one before that is closed, then that's the last used page. - if let Some(first_open_page) = - find_first_page(flash, flash_range.clone(), cache, 0, PageState::Open).await? + let first_page = run_with_auto_repair!( + function = { + match find_first_page(flash, flash_range.clone(), cache, 0, PageState::PartialOpen) + .await? { - let previous_page = previous_page::(flash_range.clone(), first_open_page); - if get_page_state(flash, flash_range.clone(), cache, previous_page) - .await? - .is_closed() - { - // The previous page is closed, so the first_open_page is what we want - first_open_page - } else { - // The page before the open page is not closed, so it must be open. - // This means that all pages are open and that we don't have any items yet. - cache.unmark_dirty(); - 0 + Some(last_used_page) => { + // The next page of the `PartialOpen` page is the first page + Ok(next_page::(flash_range.clone(), last_used_page)) + } + None => { + // In the event that all pages are still open or the last used page was just closed, we search for the first open page. + // If the page one before that is closed, then that's the last used page. + if let Some(first_open_page) = + find_first_page(flash, flash_range.clone(), cache, 0, PageState::Open) + .await? + { + let previous_page = + previous_page::(flash_range.clone(), first_open_page); + if get_page_state(flash, flash_range.clone(), cache, previous_page) + .await? + .is_closed() + { + // The previous page is closed, so the first_open_page is what we want + Ok(first_open_page) + } else { + // The page before the open page is not closed, so it must be open. + // This means that all pages are open and that we don't have any items yet. + cache.unmark_dirty(); + Ok(0) + } + } else { + // There are no open pages, so everything must be closed. + // Something is up and this should never happen. + // To recover, we will just erase all the flash. + Err(Error::Corrupted { + #[cfg(feature = "_test")] + backtrace: std::backtrace::Backtrace::capture(), + }) + } } - } else { - // There are no open pages, so everything must be closed. - // Something is up and this should never happen. - // To recover, we will just erase all the flash. - return Err(Error::Corrupted { - #[cfg(feature = "_test")] - backtrace: std::backtrace::Backtrace::capture(), - }); } - } - }; + }, + repair = try_repair::(flash, flash_range.clone(), cache, data_buffer).await? + )?; + Ok(MapItemIter { flash, flash_range: flash_range.clone(), @@ -1718,18 +1720,51 @@ mod tests { .unwrap(); } - let mut map_iter = fetch_all_items(&mut flash, flash_range.clone(), &mut cache) + // Save 10 times for key 1 + for i in 0..10 { + store_item( + &mut flash, + flash_range.clone(), + &mut cache, + &mut data_buffer, + &1u8, + &vec![i; i as usize].as_slice(), + ) .await .unwrap(); + } + + let mut map_iter = fetch_all_items::( + &mut flash, + flash_range.clone(), + &mut cache, + &mut data_buffer, + ) + .await + .unwrap(); let mut count = 0; + let mut last_value_buffer = [0u8; 64]; + let mut last_value_length = 0; while let Ok(Some((key, value))) = map_iter.next::(&mut data_buffer).await { - println!("{}: {:?}", key, value); - assert_eq!(value, vec![key; key as usize]); - count += 1; + if key == 1 { + // This is the key we stored multiple times, record the last value + last_value_length = value.len(); + last_value_buffer[..value.len()].copy_from_slice(value); + } else { + assert_eq!(value, vec![key; key as usize]); + count += 1; + } } - // Check total number of fetched items - assert_eq!(count, UPPER_BOUND); + // Check the + assert_eq!(last_value_length, 9); + assert_eq!( + &last_value_buffer[..last_value_length], + vec![9u8; 9].as_slice() + ); + + // Check total number of fetched items, +1 since we didn't count key 1 + assert_eq!(count + 1, UPPER_BOUND); } } From c897734291aeb0b930d5134fba2601262dc438f9 Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Thu, 19 Dec 2024 14:26:16 +0800 Subject: [PATCH 8/9] doc(map): improve map iterator doc, add a warning Signed-off-by: Haobo Gu --- src/map.rs | 70 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/src/map.rs b/src/map.rs index 612acd5..8863e7a 100644 --- a/src/map.rs +++ b/src/map.rs @@ -92,6 +92,7 @@ //! If done incorrectly, the deserialize function of requested value type will see //! data it doesn't expect. In the best case it'll return an error, in a bad case it'll //! give bad invalid data and in the worst case the deserialization code panics. +//! It is worth mentioning that `fetch_all_items` also requires that all items have the same type. //! So be careful. //! //! For your convenience there are premade implementations for the [Key] and [Value] traits. @@ -114,22 +115,28 @@ use super::*; /// Iterator which iterates all non-erased & non-corrupted items in the map. /// /// The iterator will return the (Key, Value) tuple when calling `next()`. -/// If the iterator ends, it will return `None`. +/// If the iterator ends, it will return `Ok(None)`. /// /// The following is a simple example of how to use the iterator: /// ```rust -/// // Create the iterator -/// let mut iterator = get_item_iter(&mut flash, flash_range.clone(), &mut cache).await; +/// // Create the iterator of map items +/// let mut iterator = fetch_all_item::( +/// &mut flash, +/// flash_range.clone(), +/// &mut cache, +/// &mut buffer +/// ) +/// .await +/// .unwrap(); /// -/// // Iterate through all items -/// loop { -/// // Suppose the Key and Value types are u8, u32 -/// if let Ok(Some(item)) = iterator.next::(&mut buffer).await { -/// // Do something with the item -/// } else { -/// // Iterator ends -/// break; -/// } +/// // Iterate through all items, suppose the Key and Value types are u8, u32 +/// while let Ok(Some((key, value))) = iterator +/// .next::(&mut buffer) +/// .await +/// { +/// // Do somethinmg with the item. +/// // Please note that for the same key there might be multiple items returned, +/// // the last one is the current active one. /// } /// ``` pub struct MapItemIter<'d, 'c, S: NorFlash, CI: CacheImpl> { @@ -204,12 +211,45 @@ impl<'d, 'c, S: NorFlash, CI: CacheImpl> MapItemIter<'d, 'c, S, CI> { /// Get an iterator that iterates over all non-erased & non-corrupted items in the map. /// -/// Note: Because map doesn't erase the items when you insert a new one with the same key, +///
+/// You should be very careful when using the map item iterator: +///
    +///
  • +/// Because map doesn't erase the items when you insert a new one with the same key, /// so it's possible that the iterator returns items with the same key multiple times. /// Generally the last returned one is the `active` one. -/// You should be very careful when using the map item iterator. +///
    • +///
  • +/// The iterator requires ALL items in the storage have the SAME type. +/// If you have different types of items in your map, the iterator might return incorrect data or error. +///
    • +///
+///
+/// +/// The following is a simple example of how to use the iterator: +/// ```rust +/// // Create the iterator of map items +/// let mut iterator = fetch_all_item::( +/// &mut flash, +/// flash_range.clone(), +/// &mut cache, +/// &mut buffer +/// ) +/// .await +/// .unwrap(); /// -/// If the iterator returns `Ok(None)`, the iterator has ended. +/// // Iterate through all items, suppose the Key and Value types are u8, u32 +/// while let Ok(Some((key, value))) = iterator +/// .next::(&mut buffer) +/// .await +/// { +/// // Do somethinmg with the item. +/// // Please note that for the same key there might be multiple items returned, +/// // the last one is the current active one. +/// } +/// ``` +/// + pub async fn fetch_all_items<'d, 'c, K: Key, S: NorFlash, CI: KeyCacheImpl>( flash: &'d mut S, flash_range: Range, From 4ed90344a5a2f39a2e0177c650637dee27eb5b83 Mon Sep 17 00:00:00 2001 From: Haobo Gu Date: Thu, 19 Dec 2024 14:26:26 +0800 Subject: [PATCH 9/9] chore: update changelog Signed-off-by: Haobo Gu --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 43686a0..242f275 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ## Unreleased +- Added item iterator for map. - *Breaking:* Added `Value` impls for `bool`, `Option`, and `[T: Value; N]`. *This can break existing code because it changes type inference, be mindfull of that!*