From 6b17c72685aa0fae0d0227596d8e4b734388c79d Mon Sep 17 00:00:00 2001 From: Chris Beck Date: Thu, 3 Nov 2022 15:25:48 -0500 Subject: [PATCH] Add mc-util-vec-map which exposes a heapless map type (#2810) * Add mc-util-vec-map which exposes a heapless map type A heapless map type is needed for things like hardware wallets. This was pulled out of #2725 to ease review. * Update util/vec-map/src/lib.rs Co-authored-by: Nick Santana * Update util/vec-map/README.md Co-authored-by: Nick Santana * fixup trait bounds per suggestions * address more review and test comments * Update util/vec-map/src/lib.rs Co-authored-by: Remoun Metyas * more review comments * fix test build Co-authored-by: Nick Santana Co-authored-by: Remoun Metyas --- Cargo.lock | 175 ++++++++++++++++++++++- Cargo.toml | 2 + util/vec-map/Cargo.toml | 11 ++ util/vec-map/README.md | 16 +++ util/vec-map/src/lib.rs | 302 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 503 insertions(+), 3 deletions(-) create mode 100644 util/vec-map/Cargo.toml create mode 100644 util/vec-map/README.md create mode 100644 util/vec-map/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 4f7c29c5c8..350a1e9581 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,6 +256,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "atomic-polyfill" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c041a8d9751a520ee19656232a18971f18946a7900f1520ee4400002244dd89" +dependencies = [ + "critical-section", +] + [[package]] name = "atty" version = "0.2.14" @@ -298,6 +307,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "bare-metal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5deb64efa5bd81e31fcd1938615a6d98c82eafcbcd787162b6f63b91d6bac5b3" +dependencies = [ + "rustc_version 0.2.3", +] + +[[package]] +name = "bare-metal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fe8f5a8a398345e52358e18ff07cc17a568fbca5c6f73873d3a62056309603" + [[package]] name = "base64" version = "0.13.1" @@ -386,6 +410,18 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f0dc55f2d8a1a85650ac47858bb001b4c0dd73d79e3c455a842925e68d29cd3" +[[package]] +name = "bit_field" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" + +[[package]] +name = "bitfield" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719" + [[package]] name = "bitflags" version = "1.2.1" @@ -831,6 +867,18 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "cortex-m" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70858629a458fdfd39f9675c4dc309411f2a3f83bede76988d81bf1a0ecee9e0" +dependencies = [ + "bare-metal 0.2.5", + "bitfield", + "embedded-hal", + "volatile-register", +] + [[package]] name = "cpufeatures" version = "0.2.1" @@ -900,6 +948,18 @@ dependencies = [ "itertools", ] +[[package]] +name = "critical-section" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95da181745b56d4bd339530ec393508910c909c784e8962d15d722bacf0bcbcd" +dependencies = [ + "bare-metal 1.0.0", + "cfg-if 1.0.0", + "cortex-m", + "riscv", +] + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -1289,6 +1349,16 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "embedded-hal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" +dependencies = [ + "nb 0.1.3", + "void", +] + [[package]] name = "encoding_rs" version = "0.8.22" @@ -1749,6 +1819,15 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -1764,6 +1843,19 @@ dependencies = [ "serde", ] +[[package]] +name = "heapless" +version = "0.7.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version 0.4.0", + "spin 0.9.3", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.0" @@ -5642,6 +5734,14 @@ dependencies = [ "url", ] +[[package]] +name = "mc-util-vec-map" +version = "2.1.0-pre0" +dependencies = [ + "displaydoc", + "heapless", +] + [[package]] name = "mc-util-zip-exact" version = "2.1.0-pre0" @@ -5881,6 +5981,21 @@ dependencies = [ "version_check", ] +[[package]] +name = "nb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" +dependencies = [ + "nb 1.0.0", +] + +[[package]] +name = "nb" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae" + [[package]] name = "nom" version = "5.1.2" @@ -6883,6 +6998,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "riscv" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6907ccdd7a31012b70faf2af85cd9e5ba97657cc3987c4f13f8e4d2c2a088aba" +dependencies = [ + "bare-metal 1.0.0", + "bit_field", + "riscv-target", +] + +[[package]] +name = "riscv-target" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88aa938cda42a0cf62a20cfe8d139ff1af20c2e681212b5b34adb5a58333f222" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "rjson" version = "0.3.1" @@ -6998,7 +7134,7 @@ dependencies = [ "log", "rusoto_credential", "rusoto_signature", - "rustc_version", + "rustc_version 0.4.0", "serde", "serde_json", "tokio", @@ -7056,7 +7192,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rusoto_credential", - "rustc_version", + "rustc_version 0.4.0", "serde", "sha2 0.9.8", "tokio", @@ -7080,6 +7216,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.0" @@ -7312,7 +7457,7 @@ checksum = "c63317c4051889e73f0b00ce4024cae3e6a225f2e18a27d2c1522eb9ce2743da" dependencies = [ "hostname", "libc", - "rustc_version", + "rustc_version 0.4.0", "sentry-core", "uname", ] @@ -7767,6 +7912,9 @@ name = "spin" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" +dependencies = [ + "lock_api", +] [[package]] name = "stable-pattern" @@ -7777,6 +7925,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "state" version = "0.5.3" @@ -8345,6 +8499,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcell" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002" + [[package]] name = "vcpkg" version = "0.2.8" @@ -8369,6 +8529,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "volatile-register" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ee8f19f9d74293faf70901bc20ad067dc1ad390d2cbf1e3f75f721ffee908b6" +dependencies = [ + "vcell", +] + [[package]] name = "wait-timeout" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 23eda3c4fe..ccb1273132 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,6 +160,8 @@ members = [ "util/test-helper", "util/test-vector", "util/uri", + "util/vec-map", + "util/zip-exact", "wasm-test", "watcher", "watcher/api", diff --git a/util/vec-map/Cargo.toml b/util/vec-map/Cargo.toml new file mode 100644 index 0000000000..3913ad8cfe --- /dev/null +++ b/util/vec-map/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "mc-util-vec-map" +version = "2.1.0-pre0" +authors = ["MobileCoin"] +edition = "2021" +description = "A map object based on heapless Vec" +readme = "README.md" + +[dependencies] +displaydoc = "0.2" +heapless = { version = "0.7", default-features = false } diff --git a/util/vec-map/README.md b/util/vec-map/README.md new file mode 100644 index 0000000000..3cea38df30 --- /dev/null +++ b/util/vec-map/README.md @@ -0,0 +1,16 @@ +mc-util-vec-map +=============== + +This is a map (container of key-value pairs) whose storage is arranged as two +`Heapless::Vec` objects, one for keys and one for values. + +The motivation is somewhat connected to crates.io crates: +* https://crates.io/crates/vector-map +* https://docs.rs/vec_map/latest/vec_map/struct.VecMap.html +* https://docs.rs/vec-collections/latest/vec_collections/ + +However, these crates use `std::vec::Vec` rather than a `no_std` friendly object. + +We are using `heapless` because we want this to be friendly for hardware wallets. + +This will be much smaller on the stack than some kind of hash table. diff --git a/util/vec-map/src/lib.rs b/util/vec-map/src/lib.rs new file mode 100644 index 0000000000..e755806aca --- /dev/null +++ b/util/vec-map/src/lib.rs @@ -0,0 +1,302 @@ +// Copyright (c) 2018-2022 The MobileCoin Foundation + +//! Provides a map (key-value store) interface backed by `heapless::Vec` + +#![no_std] +#![deny(missing_docs)] + +use displaydoc::Display; +use heapless::Vec; + +/// An error which can occur when using VecMap +#[derive(Clone, Debug, Display, Eq, PartialEq)] +pub enum Error { + /// VecMap capacity exceeded + CapacityExceeded, +} + +/// This is a mini version of VecMap that is no_std compatible and uses +/// heapless::Vec instead alloc::Vec. +/// +/// It may be better to patch upstream for no_std compatibility and use that, +/// but that crate has other issues -- it relies on an +/// experimental "contracts" crate that causes a dependency on rand crate. +/// Porting to Heapless would be a breaking change to the API. +/// +/// TBD what the best path is: https://github.com/p-avital/vec-map-rs/blob/master/src/lib.rs +#[derive(Clone, Debug)] +pub struct VecMap { + keys: Vec, + values: Vec, +} + +impl Default for VecMap { + fn default() -> Self { + Self { + keys: Default::default(), + values: Default::default(), + } + } +} + +impl VecMap { + /// Check if the map is empty + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Get the number of items in the map + #[inline] + pub fn len(&self) -> usize { + debug_assert!(self.keys.len() == self.values.len()); + self.keys.len() + } +} + +impl VecMap { + /// Get the value associated to a key, if present + pub fn get(&self, key: &K) -> Option<&V> { + self.keys + .iter() + .position(|k| k == key) + .map(|idx| &self.values[idx]) + } + + /// Get a mutable reference to the value associated to a key, if present + pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { + self.keys + .iter() + .position(|k| k == key) + .map(|idx| &mut self.values[idx]) + } +} + +impl VecMap { + /// Get a mutable reference to the value associated to a key, if present, + /// or else insert such a value produced by given callback, + /// and then return a mutable reference + /// + /// Returns an error if the heapless::Vec capacity was exceeded + pub fn get_mut_or_insert_with( + &mut self, + key: &K, + val_fn: impl FnOnce() -> V, + ) -> Result<&mut V, Error> { + if let Some(idx) = self.keys.iter().position(|k| k == key) { + Ok(&mut self.values[idx]) + } else { + let idx = self.keys.len(); + debug_assert_eq!(idx, self.values.len()); + self.keys + .push(key.clone()) + .map_err(|_| Error::CapacityExceeded)?; + self.values + .push(val_fn()) + .map_err(|_| panic!("Length invariant violated"))?; + Ok(&mut self.values[idx]) + } + } + + /// Get an iterator over the pairs in the VecMap + pub fn iter(&self) -> IterVecMap { + IterVecMap::new(self) + } +} + +// Sorting is possible when keys are ordered, and keys and values are cloneable +impl VecMap { + /// Sort the key-value pairs of the VecMap + pub fn sort(&mut self) { + // First compute the order that would sort the set of keys + let mut indices: Vec = (0..self.keys.len()).collect(); + indices.sort_by_key(|&i| &self.keys[i]); + // Make new key and val sets + let mut new_keys = Vec::::default(); + let mut new_vals = Vec::::default(); + // Push items into the new sets in appropriate order + for idx in indices { + // Safety: This is okay because indices + // has length at most n, so we are pushing at most n + // things into new_keys and new_vals. + unsafe { new_keys.push_unchecked(self.keys[idx].clone()) }; + unsafe { new_vals.push_unchecked(self.values[idx].clone()) }; + } + // Overwrite old sets + self.keys = new_keys; + self.values = new_vals; + } +} + +/// An iterator over a VecMap +pub struct IterVecMap<'a, K: Clone + Eq, V, const N: usize> { + src: &'a VecMap, + idx: usize, +} + +impl<'a, K: Clone + Eq, V, const N: usize> IterVecMap<'a, K, V, N> { + fn new(src: &'a VecMap) -> Self { + Self { src, idx: 0 } + } +} + +impl<'a, K: Clone + Eq, V, const N: usize> Iterator for IterVecMap<'a, K, V, N> { + type Item = (&'a K, &'a V); + + fn next(&mut self) -> Option { + let result = self.src.keys.get(self.idx).and_then(|key_ref| { + self.src + .values + .get(self.idx) + .map(|value_ref| (key_ref, value_ref)) + }); + self.idx += 1; + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate alloc; + use alloc::vec; + + #[test] + fn test_get_mut_or_insert_with() { + let mut vec_map = VecMap::::default(); + assert_eq!(*vec_map.get_mut_or_insert_with(&3, || 5).unwrap(), 5); + assert_eq!(*vec_map.get_mut_or_insert_with(&4, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&7, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&8, || 10).unwrap(), 10); + assert_eq!( + vec_map.get_mut_or_insert_with(&9, || 11), + Err(Error::CapacityExceeded) + ); + assert_eq!(vec_map.len(), 4); + assert_eq!(*vec_map.get_mut_or_insert_with(&7, || 10).unwrap(), 6); + assert_eq!(vec_map.len(), 4); + } + + #[test] + fn test_get() { + let mut vec_map = VecMap::::default(); + assert_eq!(*vec_map.get_mut_or_insert_with(&3, || 5).unwrap(), 5); + assert_eq!(*vec_map.get_mut_or_insert_with(&4, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&7, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&8, || 10).unwrap(), 10); + + assert_eq!(vec_map.get(&3), Some(&5)); + assert_eq!(vec_map.get(&4), Some(&6)); + assert_eq!(vec_map.get(&5), None); + assert_eq!(vec_map.get(&6), None); + assert_eq!(vec_map.get(&7), Some(&6)); + assert_eq!(vec_map.get(&8), Some(&10)); + } + + #[test] + fn test_get_mut() { + let mut vec_map = VecMap::::default(); + assert_eq!(*vec_map.get_mut_or_insert_with(&3, || 5).unwrap(), 5); + assert_eq!(*vec_map.get_mut_or_insert_with(&4, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&7, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&8, || 10).unwrap(), 10); + + assert_eq!(vec_map.get_mut(&3), Some(5).as_mut()); + assert_eq!(vec_map.get_mut(&4), Some(6).as_mut()); + assert_eq!(vec_map.get_mut(&5), None); + assert_eq!(vec_map.get_mut(&6), None); + assert_eq!(vec_map.get_mut(&7), Some(6).as_mut()); + assert_eq!(vec_map.get_mut(&8), Some(10).as_mut()); + } + + #[test] + fn test_iter() { + let mut vec_map = VecMap::::default(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![]); + + assert_eq!(*vec_map.get_mut_or_insert_with(&3, || 5).unwrap(), 5); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&3, &5)]); + + assert_eq!(*vec_map.get_mut_or_insert_with(&4, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&7, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&8, || 10).unwrap(), 10); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&3, &5), (&4, &6), (&7, &6), (&8, &10)]); + } + + #[test] + fn test_sort_empty() { + let mut vec_map = VecMap::::default(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![]); + + vec_map.sort(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![]); + } + + #[test] + fn test_sort_one() { + let mut vec_map = VecMap::::default(); + + assert_eq!(*vec_map.get_mut_or_insert_with(&9, || 5).unwrap(), 5); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&9, &5)]); + + vec_map.sort(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&9, &5)]); + } + + #[test] + fn test_sort_two() { + let mut vec_map = VecMap::::default(); + + assert_eq!(*vec_map.get_mut_or_insert_with(&9, || 5).unwrap(), 5); + assert_eq!(*vec_map.get_mut_or_insert_with(&3, || 6).unwrap(), 6); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&9, &5), (&3, &6)]); + + vec_map.sort(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&3, &6), (&9, &5)]); + + vec_map.sort(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&3, &6), (&9, &5)]); + } + + #[test] + fn test_sort_at_capacity() { + let mut vec_map = VecMap::::default(); + assert_eq!(*vec_map.get_mut_or_insert_with(&9, || 5).unwrap(), 5); + assert_eq!(*vec_map.get_mut_or_insert_with(&3, || 6).unwrap(), 6); + assert_eq!(*vec_map.get_mut_or_insert_with(&7, || 7).unwrap(), 7); + assert_eq!(*vec_map.get_mut_or_insert_with(&1, || 10).unwrap(), 10); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&9, &5), (&3, &6), (&7, &7), (&1, &10)]); + + vec_map.sort(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&1, &10), (&3, &6), (&7, &7), (&9, &5)]); + + vec_map.sort(); + + let seq: alloc::vec::Vec<_> = vec_map.iter().collect(); + assert_eq!(seq, vec![(&1, &10), (&3, &6), (&7, &7), (&9, &5)]); + } +}