Map iterator

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ## Unreleased
 
+- Added item iterator for map.
 - *Breaking:* Added `Value` impls for `bool`, `Option<T: Value>`, and `[T: Value; N]`.
   *This can break existing code because it changes type inference, be mindfull of that!*
 

src/map.rs

@@ -92,13 +92,15 @@
 //! If done incorrectly, the deserialize function of requested value type will see
 //! data it doesn't expect. In the best case it'll return an error, in a bad case it'll
 //! give bad invalid data and in the worst case the deserialization code panics.
+//! It is worth mentioning that `fetch_all_items` also requires that all items have the same type.
 //! So be careful.
 //!
 //! For your convenience there are premade implementations for the [Key] and [Value] traits.
 //!
 
 use core::mem::{size_of, MaybeUninit};
 
+use cache::CacheImpl;
 use embedded_storage_async::nor_flash::MultiwriteNorFlash;
 
 use crate::item::{find_next_free_item_spot, Item, ItemHeader, ItemIter};
@@ -110,6 +112,210 @@ use self::{
 
 use super::*;
 
+/// Iterator which iterates all non-erased & non-corrupted items in the map.
+///
+/// The iterator will return the (Key, Value) tuple when calling `next()`.
+/// If the iterator ends, it will return `Ok(None)`.
+///
+/// The following is a simple example of how to use the iterator:
+/// ```rust
+/// // Create the iterator of map items
+/// let mut iterator = fetch_all_item::<u8, _, _>(
+///     &mut flash,
+///     flash_range.clone(),
+///     &mut cache,
+///     &mut buffer
+/// )
+/// .await
+/// .unwrap();
+///
+/// // Iterate through all items, suppose the Key and Value types are u8, u32
+/// while let Ok(Some((key, value))) = iterator
+///     .next::<u8, u32>(&mut buffer)
+///     .await
+/// {
+///     // Do somethinmg with the item.
+///     // Please note that for the same key there might be multiple items returned,
+///     // the last one is the current active one.
+/// }
+/// ```
+pub struct MapItemIter<'d, 'c, S: NorFlash, CI: CacheImpl> {
+    flash: &'d mut S,
+    flash_range: Range<u32>,
+    first_page: usize,
+    cache: &'c mut CI,
+    current_page_index: usize,
+    pub(crate) current_iter: ItemIter,
+}
+
+impl<'d, 'c, S: NorFlash, CI: CacheImpl> MapItemIter<'d, 'c, S, CI> {
+    /// Get the next item in the iterator. Be careful that the given `data_buffer` should large enough to contain the serialized key and value.
+    pub async fn next<'a, K: Key, V: Value<'a>>(
+        &mut self,
+        data_buffer: &'a mut [u8],
+    ) -> Result<Option<(K, V)>, Error<S::Error>> {
+        // Find the next item
+        let item = loop {
+            if let Some((item, _address)) = self.current_iter.next(self.flash, data_buffer).await? {
+                // We've found the next item, quit the loop
+                break item;
+            }
+
+            // The current page is done, we need to find the next page
+            // Find next page which is not open, update `self.current_iter`
+            loop {
+                self.current_page_index =
+                    next_page::<S>(self.flash_range.clone(), self.current_page_index);
+
+                // We've looped back to the first page, which means all pages are checked, there's nothing left so we return None
+                if self.current_page_index == self.first_page {
+                    return Ok(None);
+                }
+
+                match get_page_state::<S>(
+                    self.flash,
+                    self.flash_range.clone(),
+                    self.cache,
+                    self.current_page_index,
+                )
+                .await
+                {
+                    Ok(PageState::Closed) | Ok(PageState::PartialOpen) => {
+                        self.current_iter = ItemIter::new(
+                            calculate_page_address::<S>(
+                                self.flash_range.clone(),
+                                self.current_page_index,
+                            ) + S::WORD_SIZE as u32,
+                            calculate_page_end_address::<S>(
+                                self.flash_range.clone(),
+                                self.current_page_index,
+                            ) - S::WORD_SIZE as u32,
+                        );
+                        break;
+                    }
+                    _ => continue,
+                }
+            }
+        };
+
+        let data_len = item.header.length as usize;
+        let (key, key_len) = K::deserialize_from(item.data())?;
+
+        Ok(Some((
+            key,
+            V::deserialize_from(&data_buffer[key_len..][..data_len - key_len])
+                .map_err(Error::SerializationError)?,
+        )))
+    }
+}
+
+/// Get an iterator that iterates over all non-erased & non-corrupted items in the map.
+///
+/// <div class="warning">
+/// You should be very careful when using the map item iterator:
+/// <ul>
+/// <li>
+/// Because map doesn't erase the items when you insert a new one with the same key,
+/// so it's possible that the iterator returns items with the same key multiple times.
+/// Generally the last returned one is the `active` one.
+/// </li>
+/// <li>
+/// The iterator requires ALL items in the storage have the SAME type.
+/// If you have different types of items in your map, the iterator might return incorrect data or error.
+/// </li>
+/// </ul>
+/// </div>
+///
+/// The following is a simple example of how to use the iterator:
+/// ```rust
+/// // Create the iterator of map items
+/// let mut iterator = fetch_all_item::<u8, _, _>(
+///     &mut flash,
+///     flash_range.clone(),
+///     &mut cache,
+///     &mut buffer
+/// )
+/// .await
+/// .unwrap();
+///
+/// // Iterate through all items, suppose the Key and Value types are u8, u32
+/// while let Ok(Some((key, value))) = iterator
+///     .next::<u8, u32>(&mut buffer)
+///     .await
+/// {
+///     // Do somethinmg with the item.
+///     // Please note that for the same key there might be multiple items returned,
+///     // the last one is the current active one.
+/// }
+/// ```
+///
+
+pub async fn fetch_all_items<'d, 'c, K: Key, S: NorFlash, CI: KeyCacheImpl<K>>(
+    flash: &'d mut S,
+    flash_range: Range<u32>,
+    cache: &'c mut CI,
+    data_buffer: &mut [u8],
+) -> Result<MapItemIter<'d, 'c, S, CI>, Error<S::Error>> {
+    // Get the first page index.
+    // The first page used by the map is the next page of the `PartialOpen` page or the last `Closed` page
+    let first_page = run_with_auto_repair!(
+        function = {
+            match find_first_page(flash, flash_range.clone(), cache, 0, PageState::PartialOpen)
+                .await?
+            {
+                Some(last_used_page) => {
+                    // The next page of the `PartialOpen` page is the first page
+                    Ok(next_page::<S>(flash_range.clone(), last_used_page))
+                }
+                None => {
+                    // In the event that all pages are still open or the last used page was just closed, we search for the first open page.
+                    // If the page one before that is closed, then that's the last used page.
+                    if let Some(first_open_page) =
+                        find_first_page(flash, flash_range.clone(), cache, 0, PageState::Open)
+                            .await?
+                    {
+                        let previous_page =
+                            previous_page::<S>(flash_range.clone(), first_open_page);
+                        if get_page_state(flash, flash_range.clone(), cache, previous_page)
+                            .await?
+                            .is_closed()
+                        {
+                            // The previous page is closed, so the first_open_page is what we want
+                            Ok(first_open_page)
+                        } else {
+                            // The page before the open page is not closed, so it must be open.
+                            // This means that all pages are open and that we don't have any items yet.
+                            cache.unmark_dirty();
+                            Ok(0)
+                        }
+                    } else {
+                        // There are no open pages, so everything must be closed.
+                        // Something is up and this should never happen.
+                        // To recover, we will just erase all the flash.
+                        Err(Error::Corrupted {
+                            #[cfg(feature = "_test")]
+                            backtrace: std::backtrace::Backtrace::capture(),
+                        })
+                    }
+                }
+            }
+        },
+        repair = try_repair::<K, _>(flash, flash_range.clone(), cache, data_buffer).await?
+    )?;
+
+    Ok(MapItemIter {
+        flash,
+        flash_range: flash_range.clone(),
+        first_page,
+        cache,
+        current_page_index: first_page,
+        current_iter: ItemIter::new(
+            calculate_page_address::<S>(flash_range.clone(), first_page) + S::WORD_SIZE as u32,
+            calculate_page_end_address::<S>(flash_range.clone(), first_page) - S::WORD_SIZE as u32,
+        ),
+    })
+}
+
 /// Get the last stored value from the flash that is associated with the given key.
 /// If no value with the key is found, None is returned.
 ///
@@ -1531,4 +1737,74 @@ mod tests {
             Err(Error::ItemTooBig)
         );
     }
+
+    #[test]
+    async fn item_iterator() {
+        const UPPER_BOUND: u8 = 64;
+        let mut flash = MockFlashBig::default();
+        let flash_range = 0x000..0x1000;
+        let mut cache = cache::NoCache::new();
+
+        let mut data_buffer = AlignedBuf([0; 128]);
+
+        for i in 0..UPPER_BOUND {
+            store_item(
+                &mut flash,
+                flash_range.clone(),
+                &mut cache,
+                &mut data_buffer,
+                &i,
+                &vec![i; i as usize].as_slice(),
+            )
+            .await
+            .unwrap();
+        }
+
+        // Save 10 times for key 1
+        for i in 0..10 {
+            store_item(
+                &mut flash,
+                flash_range.clone(),
+                &mut cache,
+                &mut data_buffer,
+                &1u8,
+                &vec![i; i as usize].as_slice(),
+            )
+            .await
+            .unwrap();
+        }
+
+        let mut map_iter = fetch_all_items::<u8, _, _>(
+            &mut flash,
+            flash_range.clone(),
+            &mut cache,
+            &mut data_buffer,
+        )
+        .await
+        .unwrap();
+
+        let mut count = 0;
+        let mut last_value_buffer = [0u8; 64];
+        let mut last_value_length = 0;
+        while let Ok(Some((key, value))) = map_iter.next::<u8, &[u8]>(&mut data_buffer).await {
+            if key == 1 {
+                // This is the key we stored multiple times, record the last value
+                last_value_length = value.len();
+                last_value_buffer[..value.len()].copy_from_slice(value);
+            } else {
+                assert_eq!(value, vec![key; key as usize]);
+                count += 1;
+            }
+        }
+
+        // Check the
+        assert_eq!(last_value_length, 9);
+        assert_eq!(
+            &last_value_buffer[..last_value_length],
+            vec![9u8; 9].as_slice()
+        );
+
+        // Check total number of fetched items, +1 since we didn't count key 1
+        assert_eq!(count + 1, UPPER_BOUND);
+    }
 }