[sui-node] (Memory Profiling) Add Jemalloc to sui node, bench (Mysten…

…Labs#2984)

Cargo.toml

@@ -32,9 +32,10 @@ members = [
     "crates/x",
 ]
 
+# Debug symbols are really important for profiling
 [profile.release]
-debug = false
-strip = "debuginfo"
+debug = true
+# strip = "debuginfo"
 
 [profile.bench]
 debug = true

crates/sui-benchmark/Cargo.toml

@@ -34,5 +34,9 @@ move-core-types = { git = "https://github.com/move-language/move", rev = "f07e99
 narwhal-node = { git = "https://github.com/MystenLabs/narwhal", rev = "5be9046d6b8f7563740f4d03bba10550d3628672", package = "node" }
 workspace-hack = { path = "../workspace-hack"}
 
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+jemallocator = { version = "^0.5", features = ["profiling"] }
+jemalloc-ctl = "^0.5"
+
 [features]
 benchmark = ["narwhal-node/benchmark"]

crates/sui-benchmark/src/bin/bench.rs

@@ -18,20 +18,58 @@
 // ./bench microbench local-single-validator-thread latency
 
 use clap::*;
+use std::time::Duration;
 use sui_benchmark::benchmark::{
     bench_types, run_benchmark, validator_preparer::VALIDATOR_BINARY_NAME,
 };
 use tracing::subscriber::set_global_default;
 use tracing_subscriber::EnvFilter;
 
+// For memory profiling info see https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling
+// Example: set JE_MALLOC_CONF or _RJEM_MALLOC_CONF to:
+//   prof:true,lg_prof_interval:24,lg_prof_sample:19
+// The above means: turn on profiling, sample every 2^19 or 512KB bytes allocated,
+//   and dump out profile every 2^24 or 16MB of memory allocated.
+//
+// See [doc/src/contribute/observability.md] for more info.
+#[cfg(not(target_env = "msvc"))]
+use jemalloc_ctl::{epoch, stats};
+#[cfg(not(target_env = "msvc"))]
+use jemallocator::Jemalloc;
+
+#[cfg(not(target_env = "msvc"))]
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
 fn main() {
+    use jemalloc_ctl::config;
+    let malloc_conf = config::malloc_conf::mib().unwrap();
+    println!("Default Jemalloc conf: {}", malloc_conf.read().unwrap());
+
     let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
     let subscriber_builder =
         tracing_subscriber::fmt::Subscriber::builder().with_env_filter(env_filter);
     let subscriber = subscriber_builder.with_writer(std::io::stderr).finish();
     set_global_default(subscriber).expect("Failed to set subscriber");
     let benchmark = bench_types::Benchmark::parse();
     running_mode_pre_check(&benchmark);
+
+    #[cfg(not(target_env = "msvc"))]
+    std::thread::spawn(|| {
+        loop {
+            // many statistics are cached and only updated when the epoch is advanced.
+            epoch::advance().unwrap();
+
+            let allocated = stats::allocated::read().unwrap() / (1024 * 1024);
+            let resident = stats::resident::read().unwrap() / (1024 * 1024);
+            println!(
+                "Jemalloc: {} MB allocated / {} MB resident",
+                allocated, resident
+            );
+            std::thread::sleep(Duration::from_secs(1));
+        }
+    });
+
     let r = run_benchmark(benchmark);
     println!("{}", r);
 }

crates/sui-node/Cargo.toml

@@ -28,3 +28,10 @@ sui-types = { path = "../sui-types" }
 telemetry-subscribers = { git = "https://github.com/MystenLabs/mysten-infra", rev = "94d7da89f6a52d7f60a9802b0a03147a9c89c3e4" }
 mysten-network = { git = "https://github.com/MystenLabs/mysten-infra", rev = "94d7da89f6a52d7f60a9802b0a03147a9c89c3e4" }
 workspace-hack = { path = "../workspace-hack"}
+
+[profile.release]
+debug = true
+
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+jemallocator = { version = "^0.5", features = ["profiling"] }
+jemalloc-ctl = "^0.5"

crates/sui-node/src/main.rs

@@ -4,8 +4,9 @@
 use anyhow::Result;
 use clap::Parser;
 use multiaddr::Multiaddr;
-use std::path::PathBuf;
+use std::{path::PathBuf, time::Duration};
 use sui_config::{Config, NodeConfig};
+use tracing::info;
 
 #[derive(Parser)]
 #[clap(rename_all = "kebab-case")]
@@ -17,6 +18,22 @@ struct Args {
     listen_address: Option<Multiaddr>,
 }
 
+// For memory profiling info see https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling
+// Example: set JE_MALLOC_CONF or _RJEM_MALLOC_CONF to:
+//   prof:true,lg_prof_interval:24,lg_prof_sample:19
+// The above means: turn on profiling, sample every 2^19 or 512KB bytes allocated,
+//   and dump out profile every 2^24 or 16MB of memory allocated.
+//
+// See [doc/src/contribute/observability.md] for more info.
+#[cfg(not(target_env = "msvc"))]
+use jemalloc_ctl::{epoch, stats};
+#[cfg(not(target_env = "msvc"))]
+use jemallocator::Jemalloc;
+
+#[cfg(not(target_env = "msvc"))]
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
 #[tokio::main]
 async fn main() -> Result<()> {
     // Initialize logging
@@ -32,6 +49,28 @@ async fn main() -> Result<()> {
         config.network_address = listen_address;
     }
 
+    #[cfg(not(target_env = "msvc"))]
+    {
+        use jemalloc_ctl::config;
+        let malloc_conf = config::malloc_conf::mib().unwrap();
+        info!("Default Jemalloc conf: {}", malloc_conf.read().unwrap());
+
+        std::thread::spawn(|| {
+            loop {
+                // many statistics are cached and only updated when the epoch is advanced.
+                epoch::advance().unwrap();
+
+                let allocated = stats::allocated::read().unwrap() / (1024 * 1024);
+                let resident = stats::resident::read().unwrap() / (1024 * 1024);
+                info!(
+                    "Jemalloc: {} MB allocated / {} MB resident",
+                    allocated, resident
+                );
+                std::thread::sleep(Duration::from_secs(60));
+            }
+        });
+    }
+
     let node = sui_node::SuiNode::start(&config).await?;
     node.wait().await?;
 

crates/sui/Cargo.toml

@@ -38,6 +38,10 @@ move-cli = { git = "https://github.com/move-language/move", rev = "f07e99473e6ed
 move-package = { git = "https://github.com/move-language/move", rev = "f07e99473e6edfff22f30596dd493ac770f0bb4a" }
 workspace-hack = { path = "../workspace-hack"}
 
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+jemallocator = { version = "^0.5", features = ["profiling"] }
+jemalloc-ctl = "^0.5"
+
 [dev-dependencies]
 tempfile = "3.3.0"
 futures = "0.3.21"

doc/src/contribute/observability.md

@@ -194,7 +194,7 @@ To see nested spans visualized with [Jaeger](https://www.jaegertracing.io), do t
    $ SUI_TRACING_ENABLE=1 RUST_LOG="info,sui_core=trace" ./sui start
    ```
 1. Run some transfers with Sui CLI client, or run the benchmarking tool.
-4. Browse to `http://localhost:16686/` and select Sui as the service.
+1. Browse to `http://localhost:16686/` and select Sui as the service.
 
 > **Note:** Separate spans (which are not nested) are not connected as a single trace for now.
 
@@ -203,7 +203,33 @@ To see nested spans visualized with [Jaeger](https://www.jaegertracing.io), do t
 [Tokio-console](https://github.com/tokio-rs/console) is an awesome CLI tool designed to analyze and help debug Rust apps using Tokio, in real time! It relies on a special subscriber.
 
 1. Build Sui using a special flag: `RUSTFLAGS="--cfg tokio_unstable" cargo build`.
-2. Start Sui with `SUI_TOKIO_CONSOLE` set to 1.
-3. Clone the console repo and `cargo run` to launch the console.
+1. Start Sui with `SUI_TOKIO_CONSOLE` set to 1.
+1. Clone the console repo and `cargo run` to launch the console.
 
 > **Note:** Adding Tokio-console support may significantly slow down Sui validators/gateways.
+
+### Memory profiling
+
+jemalloc has a memory profiling mode that can be enabled at runtime with environment variables. See:
+
+* https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling
+* https://gist.github.com/ordian/928dc2bd45022cddd547528f64db9174
+
+For example, set `JE_MALLOC_CONF` or `JEMALLOC_SYS_WITH_MALLOC_CONF` to:
+`prof:true,lg_prof_interval:24,lg_prof_sample:19`
+
+The above setting means: turn on profiling, sample every 2^19 or 512KB bytes allocated,
+and dump out the profile every 2^24 or 16MB of memory allocated.
+
+To view the profile files:
+1. `brew install jemalloc libunwind gprof2dot`
+1. Start `./sui node --config-path ...`
+1. In the same `target/release` directory, run `jeprof --text sui-node jeprof.66*.heap` where 66 is the starting PID of the current sui-node process.
+
+TODO: Define where the `target/release` directory is since we refer to it as the same but haven't yet referenced it.
+
+If the profiling does not create `.heap` files, check your env vars.  There is a log line dumped at the
+start of sui-node, that should look like this:
+```shell
+   INFO sui_node: Default Jemalloc conf: prof:true,lg_prof_interval:24,lg_prof_sample:19
+```