From 74d519baf69483ede37bd2f5d2beae6443a7c5da Mon Sep 17 00:00:00 2001 From: Evan Chan Date: Tue, 14 Jun 2022 23:21:34 -0700 Subject: [PATCH] Event Store Initial Implementation (#2507) --- Cargo.lock | 316 ++++++++- crates/sui-storage/Cargo.toml | 5 + .../{event_store.rs => event_store/mod.rs} | 68 +- crates/sui-storage/src/event_store/sql.rs | 658 ++++++++++++++++++ crates/sui-types/Cargo.toml | 1 + crates/sui-types/src/base_types.rs | 5 + crates/sui-types/src/event.rs | 55 +- crates/workspace-hack/Cargo.toml | 71 +- 8 files changed, 1116 insertions(+), 63 deletions(-) rename crates/sui-storage/src/{event_store.rs => event_store/mod.rs} (69%) create mode 100644 crates/sui-storage/src/event_store/sql.rs diff --git a/Cargo.lock b/Cargo.lock index 49de8ad5e32a6..a6fc2f9e074f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -305,6 +305,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atoi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" +dependencies = [ + "num-traits 0.2.15", +] + [[package]] name = "atomicwrites" version = "0.3.1" @@ -1076,6 +1085,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + [[package]] name = "crc32fast" version = "1.3.2" @@ -1648,6 +1672,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c97b9233581d84b8e1e689cdd3a47b6f69770084fc246e86a7f78b0d9c1d4a5" +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dyn-clone" version = "1.0.5" @@ -1893,6 +1923,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flume" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843c03199d0c0ca54bc1ea90ac0d507274c28abcc4f691ae8b4eaa375087c76a" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project", + "spin 0.9.3", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1966,6 +2008,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62007592ac46aa7c2b6416f7deb9a8a8f63a01e0f1d6e1787d5630170db2b63e" +dependencies = [ + "futures-core", + "lock_api 0.4.7", + "parking_lot 0.11.2", +] + [[package]] name = "futures-io" version = "0.3.21" @@ -2292,6 +2345,15 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown 0.11.2", +] + [[package]] name = "hdrhistogram" version = "7.5.0" @@ -2320,6 +2382,9 @@ name = "heck" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -2448,11 +2513,11 @@ dependencies = [ "http", "hyper", "log", - "rustls", + "rustls 0.20.6", "rustls-native-certs", "tokio", - "tokio-rustls", - "webpki-roots", + "tokio-rustls 0.23.4", + "webpki-roots 0.22.3", ] [[package]] @@ -2693,10 +2758,10 @@ dependencies = [ "soketto", "thiserror", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util", "tracing", - "webpki-roots", + "webpki-roots 0.22.3", ] [[package]] @@ -2907,6 +2972,17 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "libsqlite3-sys" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libtest-mimic" version = "0.4.0" @@ -5100,7 +5176,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -5157,6 +5233,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct 0.6.1", + "webpki 0.21.4", +] + [[package]] name = "rustls" version = "0.20.6" @@ -5165,8 +5254,8 @@ checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ "log", "ring", - "sct", - "webpki", + "sct 0.7.0", + "webpki 0.22.0", ] [[package]] @@ -5286,6 +5375,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sct" version = "0.7.0" @@ -5717,6 +5816,111 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" +dependencies = [ + "lock_api 0.4.7", +] + +[[package]] +name = "sqlformat" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4b7922be017ee70900be125523f38bdd644f4f06a1b16e8fa5a8ee8c34bffd4" +dependencies = [ + "itertools", + "nom 7.1.1", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "551873805652ba0d912fec5bbb0f8b4cdd96baf8e2ebf5970e5671092966019b" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48c61941ccf5ddcada342cd59e3e5173b007c509e1e8e990dafc830294d9dc5" +dependencies = [ + "ahash", + "atoi", + "bitflags", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "hashlink", + "hex", + "indexmap", + "itoa 1.0.2", + "libc", + "libsqlite3-sys", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls 0.19.1", + "sha2 0.10.2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "tokio-stream", + "url", + "webpki 0.21.4", + "webpki-roots 0.21.1", +] + +[[package]] +name = "sqlx-macros" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0fba2b0cae21fc00fe6046f8baa4c7fcb49e379f0f592b04696607f69ed2e1" +dependencies = [ + "dotenv", + "either", + "heck 0.4.0", + "once_cell", + "proc-macro2", + "quote", + "sha2 0.10.2", + "sqlx-core", + "sqlx-rt", + "syn", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4db708cd3e459078f85f39f96a00960bd841f66ee2a669e90bf36907f5a79aae" +dependencies = [ + "once_cell", + "tokio", + "tokio-rustls 0.22.0", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -5735,6 +5939,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strip-ansi-escapes" version = "0.1.1" @@ -6258,6 +6472,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "bcs", "fdlimit", "flexstr", "futures", @@ -6267,6 +6482,10 @@ dependencies = [ "rand 0.7.3", "rocksdb", "serde 1.0.137", + "serde_json", + "sqlx", + "strum", + "strum_macros", "sui-types", "telemetry-subscribers", "tempfile", @@ -6370,6 +6589,7 @@ dependencies = [ "sha3 0.10.1", "signature", "static_assertions", + "strum", "strum_macros", "thiserror", "tonic", @@ -6758,9 +6978,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.19.1" +version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95eec79ea28c00a365f539f1961e9278fbcaf81c0ff6aaf0e93c181352446948" +checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ "bytes", "libc", @@ -6808,15 +7028,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.1", + "tokio", + "webpki 0.21.4", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.6", "tokio", - "webpki", + "webpki 0.22.0", ] [[package]] @@ -6892,7 +7123,7 @@ dependencies = [ "prost-derive", "rustls-pemfile", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-stream", "tokio-util", "tower", @@ -7356,6 +7587,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unsigned-varint" version = "0.7.1" @@ -7569,6 +7806,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki" version = "0.22.0" @@ -7579,13 +7826,22 @@ dependencies = [ "untrusted", ] +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki 0.21.4", +] + [[package]] name = "webpki-roots" version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" dependencies = [ - "webpki", + "webpki 0.22.0", ] [[package]] @@ -7794,6 +8050,7 @@ dependencies = [ "async-stream", "async-stream-impl", "async-trait", + "atoi", "atomicwrites", "atty", "autocfg", @@ -7858,6 +8115,8 @@ dependencies = [ "console-subscriber", "constant_time_eq", "core2", + "crc", + "crc-catalog", "crc32fast", "crossbeam", "crossbeam-channel 0.4.4", @@ -7906,6 +8165,7 @@ dependencies = [ "dirs-sys", "dirs-sys-next", "dissimilar", + "dotenv", "dyn-clone", "ed25519", "ed25519-dalek", @@ -7926,12 +8186,14 @@ dependencies = [ "fixedbitset 0.4.1", "flate2", "flexstr", + "flume", "fnv", "form_urlencoded", "futures", "futures-channel", "futures-core", "futures-executor", + "futures-intrusive", "futures-io", "futures-macro", "futures-sink", @@ -7956,6 +8218,7 @@ dependencies = [ "hakari", "hashbrown 0.11.2", "hashbrown 0.12.1", + "hashlink", "hdrhistogram", "heck 0.3.3", "heck 0.4.0", @@ -8010,6 +8273,7 @@ dependencies = [ "libc", "libloading", "librocksdb-sys", + "libsqlite3-sys", "libtest-mimic", "libz-sys", "linked-hash-map", @@ -8173,7 +8437,8 @@ dependencies = [ "rustc-demangle", "rustc-hash", "rustc_version", - "rustls", + "rustls 0.19.1", + "rustls 0.20.6", "rustls-native-certs", "rustls-pemfile", "rustversion", @@ -8184,7 +8449,8 @@ dependencies = [ "schemars", "schemars_derive", "scopeguard", - "sct", + "sct 0.6.1", + "sct 0.7.0", "semver 0.11.0", "semver 1.0.9", "semver-parser", @@ -8221,8 +8487,15 @@ dependencies = [ "smallvec", "socket2", "soketto", + "spin 0.9.3", + "sqlformat", + "sqlx", + "sqlx-core", + "sqlx-macros", + "sqlx-rt", "stable_deref_trait", "static_assertions", + "stringprep", "strip-ansi-escapes", "strsim 0.10.0", "strsim 0.8.0", @@ -8263,7 +8536,8 @@ dependencies = [ "tokio", "tokio-io-timeout", "tokio-macros", - "tokio-rustls", + "tokio-rustls 0.22.0", + "tokio-rustls 0.23.4", "tokio-stream", "tokio-util", "toml", @@ -8312,12 +8586,14 @@ dependencies = [ "unicode-segmentation", "unicode-width", "unicode-xid", + "unicode_categories", "unsigned-varint", "untrusted", "unzip-n", "url", "utf8parse", "variant_count", + "vcpkg", "vec_map", "version_check", "vte", @@ -8331,8 +8607,10 @@ dependencies = [ "wasm-bindgen-macro-support", "wasm-bindgen-shared", "web-sys", - "webpki", - "webpki-roots", + "webpki 0.21.4", + "webpki 0.22.0", + "webpki-roots 0.21.1", + "webpki-roots 0.22.3", "which", "worker", "yaml-rust", diff --git a/crates/sui-storage/Cargo.toml b/crates/sui-storage/Cargo.toml index 3663544148757..72e955fc7a695 100644 --- a/crates/sui-storage/Cargo.toml +++ b/crates/sui-storage/Cargo.toml @@ -14,9 +14,13 @@ futures = "0.3.21" flexstr = "^0.9" rand = "0.7.3" serde = { version = "1.0.136", features = ["derive"] } +serde_json = "1.0.80" tokio = { version = "1.17.0", features = ["full", "tracing"] } rocksdb = "0.18.0" tracing = "0.1.34" +sqlx = { version = "0.5", features = [ "runtime-tokio-rustls", "sqlite" ] } +strum = "^0.24" +strum_macros = "^0.24" sui-types = { path = "../sui-types" } @@ -26,6 +30,7 @@ workspace-hack = { path = "../workspace-hack"} move-core-types = { git = "https://github.com/move-language/move", rev = "c2949bc7967de5b93f0850ce4987fc06c529f9f2", features = ["address20"] } [dev-dependencies] +bcs = "0.1.3" tempfile = "3.3.0" num_cpus = "1.13.1" pretty_assertions = "1.2.0" diff --git a/crates/sui-storage/src/event_store.rs b/crates/sui-storage/src/event_store/mod.rs similarity index 69% rename from crates/sui-storage/src/event_store.rs rename to crates/sui-storage/src/event_store/mod.rs index 55aad18790720..3c0bf62d0b6be 100644 --- a/crates/sui-storage/src/event_store.rs +++ b/crates/sui-storage/src/event_store/mod.rs @@ -11,15 +11,20 @@ //! Events are also archived into checkpoints so this API should support that as well. //! +use async_trait::async_trait; use move_core_types::language_storage::ModuleId; use move_core_types::value::MoveValue; +use serde_json::Value; use sui_types::base_types::{ObjectID, TransactionDigest}; use sui_types::event::{EventEnvelope, EventType}; +pub mod sql; + use flexstr::SharedStr; /// One event pulled out from the EventStore #[allow(unused)] +#[derive(Clone, Debug, PartialEq)] pub struct StoredEvent { /// UTC timestamp in milliseconds timestamp: u64, @@ -28,42 +33,51 @@ pub struct StoredEvent { tx_digest: Option, /// The variant name from SuiEvent, eg MoveEvent, Publish, etc. event_type: SharedStr, - /// Object ID of the Move package generating the event + /// Package ID if available package_id: Option, /// Module name of the Move package generating the event module_name: Option, + /// Function name that produced the event, for Move Events + function_name: Option, + /// Object ID of NewObject, DeleteObject, package being published, or object being transferred + object_id: Option, /// Individual event fields. As much as possible these should be deconstructed and flattened, /// ie `{'obj': {'fieldA': 'A', 'fieldB': 'B'}}` should really be broken down to /// `[('obj.fieldA', 'A'), ('obj.fieldB', 'B')] /// + /// There is no guarantee of ordering in the fields. + /// /// ## Common field names - /// * `object_id` - used by TransferObject, DeleteObject /// * `version` - used by TransferObject + /// * `destination` - address, in hex bytes, used by TransferObject + /// * `type` - used by TransferObject (TransferType - Coin, ToAddress, ToObject) fields: Vec<(SharedStr, EventValue)>, // Change this to something based on CBOR for binary values, or our own value types for efficiency } /// Enum for different types of values returnable from events in the EventStore // This is distinct from MoveValue because we want to explicitly represent (and translate) // blobs and strings, allowing us to use more efficient representations. +#[derive(Clone, Debug, PartialEq)] pub enum EventValue { Move(MoveValue), /// Efficient string representation, no allocation for small strings String(SharedStr), /// Arbitrary-length blob. Please use MoveValue::Address for ObjectIDs and similar things. BinaryBlob(Vec), + Json(Value), } /// An EventStore supports event ingestion and flexible event querying /// One can think of events as logs. They represent a log of what is happening to Sui. /// Thus, all different kinds of events fit on a timeline, and one should be able to query for /// different types of events that happen over that timeline. -trait EventStore -where - EventIt: Iterator, -{ +#[async_trait] +trait EventStore { + type EventIt: IntoIterator; + /// Adds events to the EventStore. /// Semantics: events are appended, no deduplication is done. - fn add_events( + async fn add_events( &self, events: &[EventEnvelope], checkpoint_num: u64, @@ -71,41 +85,59 @@ where /// Queries for events emitted by a given transaction, returned in order emitted /// NOTE: Not all events come from transactions - fn events_for_transaction(&self, digest: TransactionDigest) - -> Result; + async fn events_for_transaction( + &self, + digest: TransactionDigest, + ) -> Result; /// Queries for all events of a certain EventType within a given time window. - /// Will return at most limit of the most recent events within the window, sorted in ascending time. - fn events_by_type( + /// Will return at most limit of the most recent events within the window, sorted in descending time. + async fn events_by_type( &self, start_time: u64, end_time: u64, event_type: EventType, limit: usize, - ) -> Result; + ) -> Result; /// Generic event iteration bounded by time. Return in ingestion order. - fn event_iterator(&self, start_time: u64, end_time: u64) -> Result; + /// start_time is inclusive and end_time is exclusive. + async fn event_iterator( + &self, + start_time: u64, + end_time: u64, + limit: usize, + ) -> Result; /// Generic event iteration bounded by checkpoint number. Return in ingestion order. /// Checkpoint numbers are inclusive on both ends. - fn events_by_checkpoint( + async fn events_by_checkpoint( &self, start_checkpoint: u64, end_checkpoint: u64, - ) -> Result; + limit: usize, + ) -> Result; /// Queries all Move events belonging to a certain Module ID within a given time window. - /// Will return at most limit of the most recent events within the window, sorted in ascending time. - fn events_by_module_id( + /// Will return at most limit of the most recent events within the window, sorted in descending time. + async fn events_by_module_id( &self, start_time: u64, end_time: u64, module: ModuleId, limit: usize, - ) -> Result; + ) -> Result; } +#[derive(Debug)] pub enum EventStoreError { GenericError(Box), + SqlError(sqlx::Error), + LimitTooHigh(usize), +} + +impl From for EventStoreError { + fn from(err: sqlx::Error) -> Self { + EventStoreError::SqlError(err) + } } diff --git a/crates/sui-storage/src/event_store/sql.rs b/crates/sui-storage/src/event_store/sql.rs new file mode 100644 index 0000000000000..248a2d20c462e --- /dev/null +++ b/crates/sui-storage/src/event_store/sql.rs @@ -0,0 +1,658 @@ +// Copyright (c) 2022, Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//! SQL and SQLite-based Event Store + +use super::*; + +use async_trait::async_trait; +use serde_json::{json, Value}; +use strum::{EnumMessage, IntoEnumIterator}; + +use sqlx::{sqlite::SqliteRow, Executor, Row, SqlitePool}; +use sui_types::event::Event; +use tracing::{info, warn}; + +/// Maximum number of events one can ask for right now +const MAX_LIMIT: usize = 5000; + +/// Sqlite-based Event Store +/// +/// ## Data Model +/// - Main columns hold most common fields +/// - object_id is used for multiple purposes, including the Publish package ID +/// - event_type is an integer in order to save space and corresponds to EventType discriminant +/// - fields is JSON for now (for easy JSON filtering) and contains all fields not in main columns +pub struct SqlEventStore { + pool: SqlitePool, +} + +// OK this is some strum macros magic so we can programmatically get the column number / position, +// and generate consistent tables as well. +// Put the SQL CREATE TABLE line for each field in the comments +#[derive(strum_macros::EnumMessage, strum_macros::EnumIter)] +#[repr(u8)] +enum EventsTableColumns { + /// timestamp INTEGER NOT NULL + Timestamp = 0, + /// checkpoint INTEGER + Checkpoint, + /// tx_digest BLOB + TxDigest, + /// event_type INTEGER + EventType, + /// package_id BLOB + PackageId, + /// module_name TEXT + ModuleName, + /// function TEXT + Function, + /// object_id BLOB + ObjectId, + /// fields TEXT + Fields, +} + +const INDEXED_COLUMNS: &[&str] = &[ + "timestamp", + "tx_digest", + "event_type", + "package_id", + "module_name", +]; + +impl SqlEventStore { + /// Creates a new SQLite database for event storage + /// db_path may be a regular path starting with "/" or ":memory:" for in-memory database. + pub async fn new_sqlite(db_path: &str) -> Result { + let pool = SqlitePool::connect(format!("sqlite:{}", db_path).as_str()).await?; + info!(db_path, "Created new SQLite EventStore"); + Ok(Self { pool }) + } + + /// Initializes the database, creating tables and indexes as needed + /// It should be safe to call this every time after new_sqlite() as IF NOT EXISTS are used. + pub async fn initialize(&self) -> Result<(), EventStoreError> { + // First create the table if needed... make the create out of the enum for consistency + // NOTE: If the below line errors, docstring might be missing for a field + let table_columns: Vec<_> = EventsTableColumns::iter() + .map(|c| c.get_documentation().unwrap()) + .collect(); + let create_sql = format!( + "CREATE TABLE IF NOT EXISTS events({});", + table_columns.join(", ") + ); + self.pool.execute(create_sql.as_str()).await?; + info!("SQLite events table is initialized"); + + // Then, create indexes + for column in INDEXED_COLUMNS { + // NOTE: Cannot prepare CREATE INDEX statements. + // Also, this may take a long time if we add fields to index, at startup. TODO + self.pool + .execute( + format!( + "CREATE INDEX IF NOT EXISTS {}_idx on events ({})", + column, column + ) + .as_str(), + ) + .await?; + info!(column, "Index is ready"); + } + + Ok(()) + } + + /// Returns total size of table. Should really only be used for testing. + #[allow(unused)] + async fn total_event_count(&self) -> Result { + let result = sqlx::query("SELECT COUNT(*) FROM events") + .fetch_one(&self.pool) + .await?; + let num_rows: i64 = result.get(0); + Ok(num_rows as usize) + } +} + +fn try_extract_object_id(row: &SqliteRow, col: usize) -> Result, EventStoreError> { + let raw_bytes: Option> = row.get(col); + match raw_bytes { + Some(bytes) => Ok(Some( + ObjectID::try_from(bytes).map_err(|e| EventStoreError::GenericError(e.into()))?, + )), + None => Ok(None), + } +} + +// Translate a Row into StoredEvent +// TODO: convert to use FromRow trait so query_as() could be used? +fn sql_row_to_event(row: SqliteRow) -> StoredEvent { + let timestamp: i64 = row.get(EventsTableColumns::Timestamp as usize); + let checkpoint: i64 = row.get(EventsTableColumns::Checkpoint as usize); + let digest_raw: Option> = row.get(EventsTableColumns::TxDigest as usize); + let tx_digest = digest_raw.map(|bytes| { + TransactionDigest::new( + bytes + .try_into() + .expect("Cannot convert digest bytes to TxDigest"), + ) + }); + let event_type: u16 = row.get(EventsTableColumns::EventType as usize); + let package_id = try_extract_object_id(&row, EventsTableColumns::PackageId as usize) + .expect("Error converting package ID bytes"); + let object_id = try_extract_object_id(&row, EventsTableColumns::ObjectId as usize) + .expect("Error converting object ID bytes"); + let module_name: Option = row.get(EventsTableColumns::ModuleName as usize); + let function: Option = row.get(EventsTableColumns::Function as usize); + let fields_text: &str = row.get(EventsTableColumns::Fields as usize); + let fields: Vec<_> = if fields_text.is_empty() { + Vec::new() + } else { + let fields_json = serde_json::from_str(fields_text) + .unwrap_or_else(|e| panic!("Could not parse [{}] as JSON: {}", fields_text, e)); + if let Value::Object(map) = fields_json { + map.into_iter() + .map(|(k, v)| (flexstr::SharedStr::from(k), EventValue::Json(v))) + .collect() + } else { + warn!( + ?fields_json, + "Could not parse JSON as object, should not happen" + ); + Vec::new() + } + }; + + StoredEvent { + timestamp: timestamp as u64, + checkpoint_num: checkpoint as u64, + tx_digest, + event_type: SharedStr::from(Event::name_from_ordinal(event_type as usize)), + package_id, + module_name: module_name.map(|s| s.into()), + function_name: function.map(SharedStr::from), + object_id, + fields, + } +} + +// Adds JSON fields for items not in any of the standard columns in table definition, eg for MOVE events. +fn event_to_json(event: &EventEnvelope) -> String { + if let Some(json_value) = &event.move_struct_json_value { + json_value.to_string() + } else { + let maybe_json = match &event.event { + Event::TransferObject { + version, + destination_addr, + type_, + .. + } => Some(json!({"destination": destination_addr.to_string(), + "version": version.value(), + "type": type_.to_string() })), + // TODO: for other event types eg EpochChange + _ => None, + }; + maybe_json.map(|j| j.to_string()).unwrap_or_default() + } +} + +const SQL_INSERT_TX: &str = "INSERT INTO events (timestamp, checkpoint, tx_digest, event_type, \ + package_id, module_name, function, object_id, fields) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + +const TS_QUERY: &str = "SELECT * FROM events WHERE timestamp >= ? AND timestamp < ? LIMIT ?"; + +const TX_QUERY: &str = "SELECT * FROM events WHERE tx_digest = ?"; + +const QUERY_BY_TYPE: &str = "SELECT * FROM events WHERE timestamp >= ? AND \ + timestamp < ? AND event_type = ? ORDER BY timestamp DESC LIMIT ?"; + +const QUERY_BY_MODULE: &str = "SELECT * FROM events WHERE timestamp >= ? AND \ + timestamp < ? AND package_id = ? AND module_name = ? ORDER BY timestamp DESC LIMIT ?"; + +const QUERY_BY_CHECKPOINT: &str = + "SELECT * FROM events WHERE checkpoint >= ? AND checkpoint <= ? LIMIT ?"; + +fn check_limit(limit: usize) -> Result<(), EventStoreError> { + if limit <= MAX_LIMIT { + Ok(()) + } else { + Err(EventStoreError::LimitTooHigh(limit)) + } +} + +#[async_trait] +impl EventStore for SqlEventStore { + type EventIt = std::vec::IntoIter; + + async fn add_events( + &self, + events: &[EventEnvelope], + checkpoint_num: u64, + ) -> Result<(), EventStoreError> { + // TODO: benchmark + // TODO: use techniques in https://docs.rs/sqlx-core/0.5.13/sqlx_core/query_builder/struct.QueryBuilder.html#method.push_values + // to execute all inserts in a single statement? + // TODO: See https://kerkour.com/high-performance-rust-with-sqlite + for event in events { + // If batching, turn off persistent to avoid caching as we may fill up the prepared statement cache + let insert_tx_q = sqlx::query(SQL_INSERT_TX).persistent(true); + let event_type = EventType::from(&event.event); + // TODO: use batched API? + insert_tx_q + .bind(event.timestamp as i64) + .bind(checkpoint_num as i64) + .bind(event.tx_digest.map(|txd| txd.to_bytes())) + .bind(event_type as u16) + .bind(event.event.package_id().map(|pid| pid.to_vec())) + .bind(event.event.module_name()) + .bind(event.event.function_name()) + .bind(event.event.object_id().map(|id| id.to_vec())) + .bind(event_to_json(event)) + .execute(&self.pool) + .await?; + } + Ok(()) + } + + async fn events_for_transaction( + &self, + digest: TransactionDigest, + ) -> Result { + let rows = sqlx::query(TX_QUERY) + .persistent(true) + .bind(digest.to_bytes()) + .map(sql_row_to_event) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter()) + } + + async fn events_by_type( + &self, + start_time: u64, + end_time: u64, + event_type: EventType, + limit: usize, + ) -> Result { + check_limit(limit)?; + let rows = sqlx::query(QUERY_BY_TYPE) + .persistent(true) + .bind(start_time as i64) + .bind(end_time as i64) + .bind(event_type as u16) + .bind(limit as i64) + .map(sql_row_to_event) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter()) + } + + async fn event_iterator( + &self, + start_time: u64, + end_time: u64, + limit: usize, + ) -> Result { + check_limit(limit)?; + let rows = sqlx::query(TS_QUERY) + .bind(start_time as i64) + .bind(end_time as i64) + .bind(limit as i64) + .map(sql_row_to_event) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter()) + } + + async fn events_by_checkpoint( + &self, + start_checkpoint: u64, + end_checkpoint: u64, + limit: usize, + ) -> Result { + // TODO: a limit maybe doesn't make sense here. May change to unbounded iterator? + check_limit(limit)?; + let rows = sqlx::query(QUERY_BY_CHECKPOINT) + .bind(start_checkpoint as i64) + .bind(end_checkpoint as i64) + .bind(limit as i64) + .map(sql_row_to_event) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter()) + } + + async fn events_by_module_id( + &self, + start_time: u64, + end_time: u64, + module: ModuleId, + limit: usize, + ) -> Result { + check_limit(limit)?; + let rows = sqlx::query(QUERY_BY_MODULE) + .persistent(true) + .bind(start_time as i64) + .bind(end_time as i64) + .bind(module.address().to_vec()) + .bind(module.name().to_string()) + .bind(limit as i64) + .map(sql_row_to_event) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use flexstr::shared_str; + use move_core_types::{ + account_address::AccountAddress, + ident_str, + identifier::Identifier, + language_storage::{StructTag, TypeTag}, + value::MoveStruct, + }; + use serde::{Deserialize, Serialize}; + use serde_json::json; + use std::collections::BTreeMap; + + use sui_types::{ + base_types::SuiAddress, + event::{Event, EventEnvelope, TransferType}, + SUI_FRAMEWORK_ADDRESS, + }; + + #[derive(Debug, Serialize, Deserialize)] + struct TestEvent { + creator: AccountAddress, + name: String, + } + + impl TestEvent { + fn struct_tag() -> StructTag { + StructTag { + address: SUI_FRAMEWORK_ADDRESS, + module: ident_str!("SUI").to_owned(), + name: ident_str!("new_foobar").to_owned(), + type_params: vec![TypeTag::Address, TypeTag::Vector(Box::new(TypeTag::U8))], + } + } + + fn move_struct(&self) -> MoveStruct { + let move_bytes: Vec<_> = self + .name + .as_bytes() + .iter() + .map(|b| MoveValue::U8(*b)) + .collect(); + MoveStruct::WithFields(vec![ + ( + ident_str!("creator").to_owned(), + MoveValue::Address(self.creator), + ), + (ident_str!("name").to_owned(), MoveValue::Vector(move_bytes)), + ]) + } + } + + fn new_test_publish_event() -> Event { + Event::Publish { + package_id: ObjectID::random(), + } + } + + fn new_test_newobj_event() -> Event { + Event::NewObject(ObjectID::random()) + } + + fn new_test_deleteobj_event() -> Event { + Event::DeleteObject(ObjectID::random()) + } + + fn new_test_transfer_event(typ: TransferType) -> Event { + Event::TransferObject { + object_id: ObjectID::random(), + version: 1.into(), + destination_addr: SuiAddress::random_for_testing_only(), + type_: typ, + } + } + + fn new_test_move_event() -> (Event, MoveStruct) { + let move_event = TestEvent { + creator: AccountAddress::random(), + name: "foobar_buz".to_string(), + }; + let event_bytes = bcs::to_bytes(&move_event).unwrap(); + ( + Event::MoveEvent { + type_: TestEvent::struct_tag(), + contents: event_bytes, + }, + move_event.move_struct(), + ) + } + + fn test_events() -> Vec { + let (move_event, move_struct) = new_test_move_event(); + let json = + serde_json::to_value(&move_struct).expect("Cannot serialize move struct to JSON"); + vec![ + EventEnvelope::new( + 1_000_000, + Some(TransactionDigest::random()), + new_test_newobj_event(), + None, + ), + EventEnvelope::new(1_001_000, None, new_test_publish_event(), None), + EventEnvelope::new( + 1_002_000, + Some(TransactionDigest::random()), + new_test_transfer_event(TransferType::Coin), + None, + ), + EventEnvelope::new( + 1_003_000, + Some(TransactionDigest::random()), + new_test_deleteobj_event(), + None, + ), + EventEnvelope::new( + 1_004_000, + Some(TransactionDigest::random()), + new_test_transfer_event(TransferType::ToAddress), + None, + ), + EventEnvelope::new( + 1_005_000, + Some(TransactionDigest::random()), + move_event, + Some(json), + ), + ] + } + + fn test_queried_event_vs_test_envelope(queried: &StoredEvent, orig: &EventEnvelope) { + assert_eq!(queried.timestamp, orig.timestamp); + assert_eq!(queried.checkpoint_num, 1); + assert_eq!(queried.tx_digest, orig.tx_digest); + assert_eq!(queried.event_type, shared_str!(orig.event_type())); + assert_eq!(queried.package_id, orig.event.package_id()); + assert_eq!( + queried.module_name, + orig.event.module_name().map(SharedStr::from) + ); + assert_eq!( + queried.function_name, + orig.event.function_name().map(SharedStr::from) + ); + assert_eq!(queried.object_id, orig.event.object_id()); + } + + #[tokio::test] + async fn test_eventstore_basic_insert_read() -> Result<(), EventStoreError> { + telemetry_subscribers::init_for_testing(); + + // Initialize store + let db = SqlEventStore::new_sqlite(":memory:").await?; + db.initialize().await?; + + // Insert some records + info!("Inserting records!"); + let to_insert = test_events(); + db.add_events(&to_insert, 1).await?; + info!("Done inserting"); + + assert_eq!(db.total_event_count().await?, 6); + + // Query for records in time range, end should be exclusive - should get 2 + let event_it = db.event_iterator(1_000_000, 1_002_000, 20).await?; + let queried_events: Vec<_> = event_it.collect(); + + assert_eq!(queried_events.len(), 2); + for i in 0..2 { + test_queried_event_vs_test_envelope(&queried_events[i], &to_insert[i]); + } + + Ok(()) + } + + #[tokio::test] + async fn test_eventstore_transfers_tx_read() -> Result<(), EventStoreError> { + telemetry_subscribers::init_for_testing(); + + // Initialize store + let db = SqlEventStore::new_sqlite(":memory:").await?; + db.initialize().await?; + + // Insert some records + info!("Inserting records!"); + let to_insert = test_events(); + db.add_events(&to_insert, 1).await?; + info!("Done inserting"); + + // Query for transfer event + let mut event_it = db + .events_for_transaction(to_insert[2].tx_digest.unwrap()) + .await?; + let transfer_event = event_it.next().expect("No transfer events in result!!"); + assert_eq!(event_it.next(), None); // Should be no more events, just that one + + test_queried_event_vs_test_envelope(&transfer_event, &to_insert[2]); + + // Now test for fields + assert_eq!(transfer_event.fields.len(), 3); + let field_map: BTreeMap<_, _> = transfer_event.fields.into_iter().collect(); + let keys: Vec<_> = field_map.keys().collect(); + assert_eq!( + keys, + vec![ + shared_str!("destination"), + shared_str!("type"), + shared_str!("version") + ] + ); + + let type_str = field_map.get(&shared_str!("type")).unwrap(); + assert_eq!(type_str, &EventValue::Json(json!("Coin"))); + + Ok(()) + } + + // Test for reads by event type, plus returning events in desc timestamp and limit + #[tokio::test] + async fn test_eventstore_query_by_type() -> Result<(), EventStoreError> { + telemetry_subscribers::init_for_testing(); + + // Initialize store + let db = SqlEventStore::new_sqlite(":memory:").await?; + db.initialize().await?; + + // Insert some records + info!("Inserting records!"); + let to_insert = test_events(); + db.add_events(&to_insert, 1).await?; + info!("Done inserting"); + + let event_it = db + .events_by_type(1_000_000, 1_005_000, EventType::TransferObject, 2) + .await?; + let queried_events: Vec<_> = event_it.collect(); + assert_eq!(queried_events.len(), 2); + + // Desc timestamp order, so the last transfer event should be first + test_queried_event_vs_test_envelope(&queried_events[0], &to_insert[4]); + test_queried_event_vs_test_envelope(&queried_events[1], &to_insert[2]); + + // Query again with limit of 1, it should return only the last transfer event + let event_it = db + .events_by_type(1_000_000, 1_005_000, EventType::TransferObject, 1) + .await?; + let queried_events: Vec<_> = event_it.collect(); + assert_eq!(queried_events.len(), 1); + test_queried_event_vs_test_envelope(&queried_events[0], &to_insert[4]); + + Ok(()) + } + + // Test for reads by move event + #[tokio::test] + async fn test_eventstore_move_events() -> Result<(), EventStoreError> { + telemetry_subscribers::init_for_testing(); + + // Initialize store + let db = SqlEventStore::new_sqlite(":memory:").await?; + db.initialize().await?; + + // Insert some records + info!("Inserting records!"); + let to_insert = test_events(); + db.add_events(&to_insert, 1).await?; + info!("Done inserting"); + + // Query for the Move event and validate basic fields + let mut event_it = db + .events_for_transaction(to_insert[5].tx_digest.unwrap()) + .await?; + let move_event = event_it.next().expect("No move events in result!!"); + assert_eq!(event_it.next(), None); // Should be no more events, just that one + + test_queried_event_vs_test_envelope(&move_event, &to_insert[5]); + assert_eq!(move_event.fields.len(), 2); + + // Query by module ID + let mod_id = ModuleId::new( + *to_insert[5].event.package_id().unwrap(), + Identifier::new(to_insert[5].event.module_name().unwrap()).unwrap(), + ); + let event_it = db + .events_by_module_id(1_000_000, 1_005_001, mod_id, 2) + .await?; + let queried_events: Vec<_> = event_it.collect(); + assert_eq!(queried_events.len(), 1); + + test_queried_event_vs_test_envelope(&queried_events[0], &to_insert[5]); + assert_eq!(queried_events[0].fields.len(), 2); + + Ok(()) + } + + #[tokio::test] + async fn test_eventstore_max_limit() -> Result<(), EventStoreError> { + telemetry_subscribers::init_for_testing(); + + // Initialize store + let db = SqlEventStore::new_sqlite(":memory:").await?; + db.initialize().await?; + + let res = db.event_iterator(1_000_000, 1_002_000, 100_000).await; + assert!(matches!(res, Err(EventStoreError::LimitTooHigh(100_000)))); + + Ok(()) + } +} diff --git a/crates/sui-types/Cargo.toml b/crates/sui-types/Cargo.toml index 9bfff238945a9..75e7f744d28f0 100644 --- a/crates/sui-types/Cargo.toml +++ b/crates/sui-types/Cargo.toml @@ -34,6 +34,7 @@ hkdf = "0.12.3" digest = "0.10.3" schemars ="0.8.10" tonic = "0.7" +strum = "^0.24" strum_macros = "^0.24" name-variant = { git = "https://github.com/MystenLabs/mysten-infra", rev = "ff5c1d69057fe93be658377462ca2875a57a0223" } diff --git a/crates/sui-types/src/base_types.rs b/crates/sui-types/src/base_types.rs index 0261fa53a47a8..481f9d86ec2e0 100644 --- a/crates/sui-types/src/base_types.rs +++ b/crates/sui-types/src/base_types.rs @@ -405,6 +405,11 @@ impl TransactionDigest { let random_bytes = rand::thread_rng().gen::<[u8; 32]>(); Self::new(random_bytes) } + + /// Translates digest into a Vec of bytes + pub fn to_bytes(&self) -> Vec { + self.0.to_vec() + } } impl AsRef<[u8]> for TransactionDigest { diff --git a/crates/sui-types/src/event.rs b/crates/sui-types/src/event.rs index ab86023edb400..4980aef271d37 100644 --- a/crates/sui-types/src/event.rs +++ b/crates/sui-types/src/event.rs @@ -3,14 +3,15 @@ use move_bytecode_utils::{layout::TypeLayoutBuilder, module_cache::GetModule}; use move_core_types::{ - language_storage::{ModuleId, StructTag, TypeTag}, + language_storage::{StructTag, TypeTag}, value::{MoveStruct, MoveTypeLayout}, }; use name_variant::NamedVariant; use serde::{Deserialize, Serialize}; use serde_json::Value; use serde_with::{serde_as, Bytes}; -use strum_macros::EnumDiscriminants; +use strum::VariantNames; +use strum_macros::{EnumDiscriminants, EnumVariantNames}; use crate::{ base_types::{ObjectID, SequenceNumber, SuiAddress, TransactionDigest}, @@ -23,9 +24,9 @@ use crate::{ #[derive(Debug, Clone, PartialEq, Eq)] pub struct EventEnvelope { /// UTC timestamp in milliseconds since epoch (1/1/1970) - timestamp: u64, + pub timestamp: u64, /// Transaction digest of associated transaction, if any - tx_digest: Option, + pub tx_digest: Option, /// Specific event type pub event: Event, /// json value for MoveStruct (for MoveEvent only) @@ -52,7 +53,7 @@ impl EventEnvelope { } } -#[derive(Eq, Debug, Clone, PartialEq, Deserialize, Serialize, Hash)] +#[derive(Eq, Debug, strum_macros::Display, Clone, PartialEq, Deserialize, Serialize, Hash)] pub enum TransferType { Coin, ToAddress, @@ -62,9 +63,19 @@ pub enum TransferType { /// Specific type of event #[serde_as] #[derive( - Eq, Debug, Clone, PartialEq, NamedVariant, Deserialize, Serialize, Hash, EnumDiscriminants, + Eq, + Debug, + Clone, + PartialEq, + NamedVariant, + Deserialize, + Serialize, + Hash, + EnumDiscriminants, + EnumVariantNames, )] #[strum_discriminants(name(EventType))] +// Developer note: PLEASE only append new entries, do not modify existing entries (binary compat) pub enum Event { /// Move-specific event MoveEvent { @@ -96,6 +107,10 @@ impl Event { Event::MoveEvent { type_, contents } } + pub fn name_from_ordinal(ordinal: usize) -> &'static str { + Event::VARIANTS[ordinal] + } + /// Returns the EventType associated with an Event pub fn event_type(&self) -> EventType { self.into() @@ -103,11 +118,9 @@ impl Event { /// Returns the object or package ID associated with the event, if available. Specifically: /// - For TransferObject: the object ID being transferred (eg moving child from parent, its the child) - /// - for Publish, the package ID (which is the object ID of the module) /// - for DeleteObject and NewObject, the Object ID pub fn object_id(&self) -> Option { match self { - Event::Publish { package_id } => Some(*package_id), Event::TransferObject { object_id, .. } => Some(*object_id), Event::DeleteObject(obj_id) => Some(*obj_id), Event::NewObject(obj_id) => Some(*obj_id), @@ -115,12 +128,32 @@ impl Event { } } - /// Extract a module ID, if available, from a SuiEvent - pub fn module_id(&self) -> Option { + /// Extracts the Move package ID associated with the event, or the package published. + pub fn package_id(&self) -> Option { + match self { + Event::MoveEvent { type_, .. } => Some(type_.address.into()), + Event::Publish { package_id } => Some(*package_id), + _ => None, + } + } + + /// Extract a module name, if available, from a SuiEvent + // TODO: should we switch to IdentStr or &str? These are more complicated to make work due to lifetimes + pub fn module_name(&self) -> Option<&str> { + match self { + Event::MoveEvent { + type_: struct_tag, .. + } => Some(struct_tag.module.as_ident_str().as_str()), + _ => None, + } + } + + /// Extracts the function name from a SuiEvent, if available + pub fn function_name(&self) -> Option { match self { Event::MoveEvent { type_: struct_tag, .. - } => Some(struct_tag.module_id()), + } => Some(struct_tag.name.to_string()), _ => None, } } diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml index 3917e54e2fa9e..43c904dec37a0 100644 --- a/crates/workspace-hack/Cargo.toml +++ b/crates/workspace-hack/Cargo.toml @@ -38,6 +38,7 @@ arrayvec-ca01ad9e24f5d932 = { package = "arrayvec", version = "0.7", features = ascii = { version = "1", features = ["std"] } async-lock = { version = "2", default-features = false } async-stream = { version = "0.3", default-features = false } +atoi = { version = "0.4", default-features = false } atomicwrites = { version = "0.3", default-features = false } atty = { version = "0.2", default-features = false } axum = { version = "0.5", features = ["form", "http1", "json", "matched-path", "original-uri", "query", "serde_json", "serde_urlencoded", "tower-log"] } @@ -92,13 +93,15 @@ console-api = { version = "0.3", default-features = false, features = ["transpor console-subscriber = { version = "0.1", features = ["env-filter"] } constant_time_eq = { version = "0.1", default-features = false } core2 = { version = "0.4", default-features = false, features = ["alloc"] } +crc = { version = "2", default-features = false } +crc-catalog = { version = "1", default-features = false } crc32fast = { version = "1", features = ["std"] } crossbeam = { version = "0.8", features = ["alloc", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", "crossbeam-queue", "std"] } crossbeam-channel-9fbad63c4bcf4a8f = { package = "crossbeam-channel", version = "0.4", default-features = false } crossbeam-channel-d8f496e17d97b5cb = { package = "crossbeam-channel", version = "0.5", features = ["crossbeam-utils", "std"] } crossbeam-deque = { version = "0.8", features = ["crossbeam-epoch", "crossbeam-utils", "std"] } crossbeam-epoch = { version = "0.9", default-features = false, features = ["alloc", "lazy_static", "std"] } -crossbeam-queue = { version = "0.3", default-features = false, features = ["alloc", "std"] } +crossbeam-queue = { version = "0.3", features = ["alloc", "std"] } crossbeam-utils-ca01ad9e24f5d932 = { package = "crossbeam-utils", version = "0.7", features = ["lazy_static", "std"] } crossbeam-utils-c38e5c1d305a1b54 = { package = "crossbeam-utils", version = "0.8", features = ["lazy_static", "std"] } crossterm-647d43efb71741da = { package = "crossterm", version = "0.21" } @@ -150,12 +153,14 @@ fixedbitset-6f8ce4dd05d13bba = { package = "fixedbitset", version = "0.2", defau fixedbitset-9fbad63c4bcf4a8f = { package = "fixedbitset", version = "0.4", default-features = false } flate2 = { version = "1", features = ["miniz_oxide", "rust_backend"] } flexstr = { version = "0.9", features = ["std"] } +flume = { version = "0.10", default-features = false, features = ["async", "futures-core", "futures-sink", "pin-project"] } fnv = { version = "1", features = ["std"] } form_urlencoded = { version = "1", default-features = false } futures = { version = "0.3", features = ["alloc", "async-await", "bilock", "executor", "futures-executor", "std", "unstable"] } futures-channel = { version = "0.3", features = ["alloc", "futures-sink", "sink", "std", "unstable"] } futures-core = { version = "0.3", features = ["alloc", "std", "unstable"] } futures-executor = { version = "0.3", features = ["std"] } +futures-intrusive = { version = "0.4", features = ["alloc", "parking_lot", "std"] } futures-io = { version = "0.3", features = ["std", "unstable"] } futures-sink = { version = "0.3", features = ["alloc", "std"] } futures-task = { version = "0.3", default-features = false, features = ["alloc", "std", "unstable"] } @@ -179,6 +184,7 @@ h2 = { version = "0.3", default-features = false } hakari = { version = "0.10", default-features = false, features = ["cli-support", "include_dir", "owo-colors", "serde", "tabular", "toml"] } hashbrown-a6292c17cd707f01 = { package = "hashbrown", version = "0.11", features = ["ahash", "inline-more", "raw"] } hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["ahash", "inline-more"] } +hashlink = { version = "0.7", default-features = false } hdrhistogram = { version = "7", features = ["base64", "crossbeam-channel", "flate2", "nom", "serialization", "sync"] } heck-468e82937335b1c9 = { package = "heck", version = "0.3", default-features = false } hex = { version = "0.4", features = ["alloc", "std"] } @@ -224,6 +230,7 @@ lazy_static-dff4ba8e3ae991db = { package = "lazy_static", version = "1", default lexical-core = { version = "0.7", features = ["arrayvec", "correct", "ryu", "static_assertions", "std", "table"] } libc = { version = "0.2", features = ["std"] } librocksdb-sys = { version = "0.6", features = ["bzip2", "bzip2-sys", "libz-sys", "lz4", "snappy", "static", "zlib", "zstd", "zstd-sys"] } +libsqlite3-sys = { version = "0.24", default-features = false, features = ["bundled", "bundled_bindings", "cc", "pkg-config", "unlock_notify", "vcpkg"] } libtest-mimic = { version = "0.4", default-features = false } libz-sys = { version = "1", default-features = false, features = ["static"] } linked-hash-map = { version = "0.5", default-features = false } @@ -362,7 +369,8 @@ rocksdb = { version = "0.18", features = ["bzip2", "lz4", "snappy", "zlib", "zst rust-ini = { version = "0.13", default-features = false } rustc-demangle = { version = "0.1", default-features = false } rustc-hash = { version = "1", features = ["std"] } -rustls = { version = "0.20", default-features = false, features = ["log", "logging", "tls12"] } +rustls-cdcf2f9584511fe6 = { package = "rustls", version = "0.19", features = ["dangerous_configuration", "log", "logging"] } +rustls-56bd22fc3884b12 = { package = "rustls", version = "0.20", default-features = false, features = ["log", "logging", "tls12"] } rustls-native-certs = { version = "0.6", default-features = false } rustls-pemfile = { version = "1", default-features = false } rustyline = { version = "9", features = ["dirs-next", "with-dirs"] } @@ -370,7 +378,8 @@ ryu = { version = "1", default-features = false } same-file = { version = "1", default-features = false } schemars = { version = "0.8", features = ["derive", "either", "schemars_derive"] } scopeguard = { version = "1", features = ["use_std"] } -sct = { version = "0.7", default-features = false } +sct-3b31131e45eafb45 = { package = "sct", version = "0.6", default-features = false } +sct-ca01ad9e24f5d932 = { package = "sct", version = "0.7", default-features = false } semver-dff4ba8e3ae991db = { package = "semver", version = "1", features = ["serde", "std"] } send_wrapper = { version = "0.4", default-features = false } serde-c38e5c1d305a1b54 = { package = "serde", version = "0.8", features = ["std"] } @@ -401,8 +410,14 @@ slug = { version = "0.1", default-features = false } smallvec = { version = "1", default-features = false } socket2 = { version = "0.4", default-features = false, features = ["all"] } soketto = { version = "0.7" } +spin = { version = "0.9", features = ["barrier", "lazy", "lock_api", "lock_api_crate", "mutex", "once", "rwlock", "spin_mutex"] } +sqlformat = { version = "0.1", default-features = false } +sqlx = { version = "0.5", features = ["_rt-tokio", "macros", "migrate", "runtime-tokio-rustls", "sqlite", "sqlx-macros"] } +sqlx-core = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-rustls", "crc", "flume", "futures-executor", "libsqlite3-sys", "migrate", "runtime-tokio-rustls", "rustls", "sha2", "sqlite", "tokio-stream", "webpki", "webpki-roots"] } +sqlx-rt = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-rustls", "once_cell", "runtime-tokio-rustls", "tokio", "tokio-rustls"] } stable_deref_trait = { version = "1", features = ["alloc", "std"] } static_assertions = { version = "1", default-features = false } +stringprep = { version = "0.1", default-features = false } strip-ansi-escapes = { version = "0.1", default-features = false } strsim-93f6ce9d446188ac = { package = "strsim", version = "0.10", default-features = false } strsim-c38e5c1d305a1b54 = { package = "strsim", version = "0.8", default-features = false } @@ -435,8 +450,9 @@ tinyvec = { version = "1", features = ["alloc", "tinyvec_macros"] } tinyvec_macros = { version = "0.1", default-features = false } tokio = { version = "1", features = ["bytes", "fs", "full", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "test-util", "time", "tokio-macros", "tracing"] } tokio-io-timeout = { version = "1", default-features = false } -tokio-rustls = { version = "0.23", features = ["logging", "tls12"] } -tokio-stream = { version = "0.1", features = ["net", "sync", "time", "tokio-util"] } +tokio-rustls-3c51e837cfc5589a = { package = "tokio-rustls", version = "0.22", default-features = false } +tokio-rustls-2b5c6dc72f624058 = { package = "tokio-rustls", version = "0.23", features = ["logging", "tls12"] } +tokio-stream = { version = "0.1", features = ["fs", "net", "sync", "time", "tokio-util"] } tokio-util = { version = "0.7", features = ["codec", "compat", "futures-io", "tracing"] } toml = { version = "0.5", features = ["indexmap", "preserve_order"] } toml_edit = { version = "0.13" } @@ -479,6 +495,7 @@ unicode-ident = { version = "1", default-features = false } unicode-normalization = { version = "0.1", features = ["std"] } unicode-segmentation = { version = "1", default-features = false } unicode-width = { version = "0.1" } +unicode_categories = { version = "0.1", default-features = false } unsigned-varint = { version = "0.7", default-features = false, features = ["std"] } untrusted = { version = "0.7", default-features = false } url = { version = "2", default-features = false } @@ -490,8 +507,10 @@ want = { version = "0.3", default-features = false } wasm-bindgen = { version = "0.2", features = ["serde", "serde-serialize", "serde_json", "spans", "std"] } wasm-bindgen-futures = { version = "0.4", default-features = false } web-sys = { version = "0.3", default-features = false, features = ["BinaryType", "Blob", "CloseEvent", "Document", "Element", "ErrorEvent", "Event", "EventTarget", "FileReader", "History", "HtmlElement", "HtmlHeadElement", "Location", "MessageEvent", "Node", "ProgressEvent", "WebSocket", "Window"] } -webpki = { version = "0.22", default-features = false, features = ["alloc", "std"] } -webpki-roots = { version = "0.22", default-features = false } +webpki-647d43efb71741da = { package = "webpki", version = "0.21", features = ["std", "trust_anchor_util"] } +webpki-3c51e837cfc5589a = { package = "webpki", version = "0.22", default-features = false, features = ["alloc", "std"] } +webpki-roots-647d43efb71741da = { package = "webpki-roots", version = "0.21", default-features = false } +webpki-roots-3c51e837cfc5589a = { package = "webpki-roots", version = "0.22", default-features = false } worker = { git = "https://github.com/MystenLabs/narwhal", rev = "2c5e8236c0702a3ff47dd769c2bbc94b029bf4a9", default-features = false, features = ["benchmark"] } yaml-rust = { version = "0.4", default-features = false } zeroize = { version = "1", features = ["alloc", "zeroize_derive"] } @@ -528,6 +547,7 @@ async-recursion = { version = "1", default-features = false } async-stream = { version = "0.3", default-features = false } async-stream-impl = { version = "0.3", default-features = false } async-trait = { version = "0.1", default-features = false } +atoi = { version = "0.4", default-features = false } atomicwrites = { version = "0.3", default-features = false } atty = { version = "0.2", default-features = false } autocfg = { version = "1", default-features = false } @@ -592,13 +612,15 @@ console-api = { version = "0.3", default-features = false, features = ["transpor console-subscriber = { version = "0.1", features = ["env-filter"] } constant_time_eq = { version = "0.1", default-features = false } core2 = { version = "0.4", default-features = false, features = ["alloc"] } +crc = { version = "2", default-features = false } +crc-catalog = { version = "1", default-features = false } crc32fast = { version = "1", features = ["std"] } crossbeam = { version = "0.8", features = ["alloc", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", "crossbeam-queue", "std"] } crossbeam-channel-9fbad63c4bcf4a8f = { package = "crossbeam-channel", version = "0.4", default-features = false } crossbeam-channel-d8f496e17d97b5cb = { package = "crossbeam-channel", version = "0.5", features = ["crossbeam-utils", "std"] } crossbeam-deque = { version = "0.8", features = ["crossbeam-epoch", "crossbeam-utils", "std"] } crossbeam-epoch = { version = "0.9", default-features = false, features = ["alloc", "lazy_static", "std"] } -crossbeam-queue = { version = "0.3", default-features = false, features = ["alloc", "std"] } +crossbeam-queue = { version = "0.3", features = ["alloc", "std"] } crossbeam-utils-ca01ad9e24f5d932 = { package = "crossbeam-utils", version = "0.7", features = ["lazy_static", "std"] } crossbeam-utils-c38e5c1d305a1b54 = { package = "crossbeam-utils", version = "0.8", features = ["lazy_static", "std"] } crossterm-647d43efb71741da = { package = "crossterm", version = "0.21" } @@ -640,6 +662,7 @@ dirs-next = { version = "2", default-features = false } dirs-sys = { version = "0.3", default-features = false } dirs-sys-next = { version = "0.1", default-features = false } dissimilar = { version = "1", default-features = false } +dotenv = { version = "0.15", default-features = false } dyn-clone = { version = "1", default-features = false } ed25519 = { version = "1", default-features = false, features = ["serde", "std"] } ed25519-dalek = { version = "1", features = ["batch", "merlin", "rand", "serde", "serde_bytes", "serde_crate", "std", "u64_backend"] } @@ -660,12 +683,14 @@ fixedbitset-6f8ce4dd05d13bba = { package = "fixedbitset", version = "0.2", defau fixedbitset-9fbad63c4bcf4a8f = { package = "fixedbitset", version = "0.4", default-features = false } flate2 = { version = "1", features = ["miniz_oxide", "rust_backend"] } flexstr = { version = "0.9", features = ["std"] } +flume = { version = "0.10", default-features = false, features = ["async", "futures-core", "futures-sink", "pin-project"] } fnv = { version = "1", features = ["std"] } form_urlencoded = { version = "1", default-features = false } futures = { version = "0.3", features = ["alloc", "async-await", "bilock", "executor", "futures-executor", "std", "unstable"] } futures-channel = { version = "0.3", features = ["alloc", "futures-sink", "sink", "std", "unstable"] } futures-core = { version = "0.3", features = ["alloc", "std", "unstable"] } futures-executor = { version = "0.3", features = ["std"] } +futures-intrusive = { version = "0.4", features = ["alloc", "parking_lot", "std"] } futures-io = { version = "0.3", features = ["std", "unstable"] } futures-macro = { version = "0.3", default-features = false } futures-sink = { version = "0.3", features = ["alloc", "std"] } @@ -690,9 +715,10 @@ h2 = { version = "0.3", default-features = false } hakari = { version = "0.10", default-features = false, features = ["cli-support", "include_dir", "owo-colors", "serde", "tabular", "toml"] } hashbrown-a6292c17cd707f01 = { package = "hashbrown", version = "0.11", features = ["ahash", "inline-more", "raw"] } hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["ahash", "inline-more"] } +hashlink = { version = "0.7", default-features = false } hdrhistogram = { version = "7", features = ["base64", "crossbeam-channel", "flate2", "nom", "serialization", "sync"] } heck-468e82937335b1c9 = { package = "heck", version = "0.3", default-features = false } -heck-9fbad63c4bcf4a8f = { package = "heck", version = "0.4" } +heck-9fbad63c4bcf4a8f = { package = "heck", version = "0.4", features = ["unicode", "unicode-segmentation"] } hex = { version = "0.4", features = ["alloc", "std"] } hkdf = { version = "0.12", default-features = false } hmac = { version = "0.12", default-features = false } @@ -744,6 +770,7 @@ lexical-core = { version = "0.7", features = ["arrayvec", "correct", "ryu", "sta libc = { version = "0.2", features = ["std"] } libloading = { version = "0.7", default-features = false } librocksdb-sys = { version = "0.6", features = ["bzip2", "bzip2-sys", "libz-sys", "lz4", "snappy", "static", "zlib", "zstd", "zstd-sys"] } +libsqlite3-sys = { version = "0.24", default-features = false, features = ["bundled", "bundled_bindings", "cc", "pkg-config", "unlock_notify", "vcpkg"] } libtest-mimic = { version = "0.4", default-features = false } libz-sys = { version = "1", default-features = false, features = ["static"] } linked-hash-map = { version = "0.5", default-features = false } @@ -907,7 +934,8 @@ rust-ini = { version = "0.13", default-features = false } rustc-demangle = { version = "0.1", default-features = false } rustc-hash = { version = "1", features = ["std"] } rustc_version = { version = "0.3", default-features = false } -rustls = { version = "0.20", default-features = false, features = ["log", "logging", "tls12"] } +rustls-cdcf2f9584511fe6 = { package = "rustls", version = "0.19", features = ["dangerous_configuration", "log", "logging"] } +rustls-56bd22fc3884b12 = { package = "rustls", version = "0.20", default-features = false, features = ["log", "logging", "tls12"] } rustls-native-certs = { version = "0.6", default-features = false } rustls-pemfile = { version = "1", default-features = false } rustversion = { version = "1", default-features = false } @@ -918,7 +946,8 @@ same-file = { version = "1", default-features = false } schemars = { version = "0.8", features = ["derive", "either", "schemars_derive"] } schemars_derive = { version = "0.8", default-features = false } scopeguard = { version = "1", features = ["use_std"] } -sct = { version = "0.7", default-features = false } +sct-3b31131e45eafb45 = { package = "sct", version = "0.6", default-features = false } +sct-ca01ad9e24f5d932 = { package = "sct", version = "0.7", default-features = false } semver-a6292c17cd707f01 = { package = "semver", version = "0.11" } semver-dff4ba8e3ae991db = { package = "semver", version = "1", features = ["serde", "std"] } semver-parser = { version = "0.10", default-features = false } @@ -955,8 +984,15 @@ slug = { version = "0.1", default-features = false } smallvec = { version = "1", default-features = false } socket2 = { version = "0.4", default-features = false, features = ["all"] } soketto = { version = "0.7" } +spin = { version = "0.9", features = ["barrier", "lazy", "lock_api", "lock_api_crate", "mutex", "once", "rwlock", "spin_mutex"] } +sqlformat = { version = "0.1", default-features = false } +sqlx = { version = "0.5", features = ["_rt-tokio", "macros", "migrate", "runtime-tokio-rustls", "sqlite", "sqlx-macros"] } +sqlx-core = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-rustls", "crc", "flume", "futures-executor", "libsqlite3-sys", "migrate", "runtime-tokio-rustls", "rustls", "sha2", "sqlite", "tokio-stream", "webpki", "webpki-roots"] } +sqlx-macros = { version = "0.5", default-features = false, features = ["_rt-tokio", "migrate", "runtime-tokio-rustls", "sha2", "sqlite"] } +sqlx-rt = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-rustls", "once_cell", "runtime-tokio-rustls", "tokio", "tokio-rustls"] } stable_deref_trait = { version = "1", features = ["alloc", "std"] } static_assertions = { version = "1", default-features = false } +stringprep = { version = "0.1", default-features = false } strip-ansi-escapes = { version = "0.1", default-features = false } strsim-93f6ce9d446188ac = { package = "strsim", version = "0.10", default-features = false } strsim-c38e5c1d305a1b54 = { package = "strsim", version = "0.8", default-features = false } @@ -997,8 +1033,9 @@ tinyvec_macros = { version = "0.1", default-features = false } tokio = { version = "1", features = ["bytes", "fs", "full", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "test-util", "time", "tokio-macros", "tracing"] } tokio-io-timeout = { version = "1", default-features = false } tokio-macros = { version = "1", default-features = false } -tokio-rustls = { version = "0.23", features = ["logging", "tls12"] } -tokio-stream = { version = "0.1", features = ["net", "sync", "time", "tokio-util"] } +tokio-rustls-3c51e837cfc5589a = { package = "tokio-rustls", version = "0.22", default-features = false } +tokio-rustls-2b5c6dc72f624058 = { package = "tokio-rustls", version = "0.23", features = ["logging", "tls12"] } +tokio-stream = { version = "0.1", features = ["fs", "net", "sync", "time", "tokio-util"] } tokio-util = { version = "0.7", features = ["codec", "compat", "futures-io", "tracing"] } toml = { version = "0.5", features = ["indexmap", "preserve_order"] } toml_edit = { version = "0.13" } @@ -1046,12 +1083,14 @@ unicode-normalization = { version = "0.1", features = ["std"] } unicode-segmentation = { version = "1", default-features = false } unicode-width = { version = "0.1" } unicode-xid = { version = "0.2" } +unicode_categories = { version = "0.1", default-features = false } unsigned-varint = { version = "0.7", default-features = false, features = ["std"] } untrusted = { version = "0.7", default-features = false } unzip-n = { version = "0.1", default-features = false } url = { version = "2", default-features = false } utf8parse = { version = "0.2" } variant_count = { version = "1", default-features = false } +vcpkg = { version = "0.2", default-features = false } vec_map = { version = "0.8", default-features = false } version_check = { version = "0.9", default-features = false } vte = { version = "0.10", features = ["arrayvec", "no_std"] } @@ -1065,8 +1104,10 @@ wasm-bindgen-macro = { version = "0.2", default-features = false, features = ["s wasm-bindgen-macro-support = { version = "0.2", default-features = false, features = ["spans"] } wasm-bindgen-shared = { version = "0.2", default-features = false } web-sys = { version = "0.3", default-features = false, features = ["BinaryType", "Blob", "CloseEvent", "Document", "Element", "ErrorEvent", "Event", "EventTarget", "FileReader", "History", "HtmlElement", "HtmlHeadElement", "Location", "MessageEvent", "Node", "ProgressEvent", "WebSocket", "Window"] } -webpki = { version = "0.22", default-features = false, features = ["alloc", "std"] } -webpki-roots = { version = "0.22", default-features = false } +webpki-647d43efb71741da = { package = "webpki", version = "0.21", features = ["std", "trust_anchor_util"] } +webpki-3c51e837cfc5589a = { package = "webpki", version = "0.22", default-features = false, features = ["alloc", "std"] } +webpki-roots-647d43efb71741da = { package = "webpki-roots", version = "0.21", default-features = false } +webpki-roots-3c51e837cfc5589a = { package = "webpki-roots", version = "0.22", default-features = false } which = { version = "4", default-features = false } worker = { git = "https://github.com/MystenLabs/narwhal", rev = "2c5e8236c0702a3ff47dd769c2bbc94b029bf4a9", default-features = false, features = ["benchmark"] } yaml-rust = { version = "0.4", default-features = false }