Add custom Ying memory profiler and prevent giant allocations (Mysten…

…Labs#5649) * Experiment: add TaiAllocator * Remove Ying profiler from stress and put it in sui-node; remove old jemalloc profiling code * Clippy, workspace-hack, misc * Update pointer to ying profiler
sm86 · Oct 30, 2022 · ef090e1 · ef090e1
1 parent c279332
commit ef090e1
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 84 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -47,16 +47,15 @@ members = [
 version = "0.14.0"
 
 [profile.release]
-# The following two lines add minimal symbol information, which helps certain profilers like Bytehound
-# without significantly increasing binary size
-debug = true
-strip = 'debuginfo'
+# debug = 1 means line charts only, which is minimum needed for good stack traces
+debug = 1
+strip = 'none'
 # Exit process with SIGABRT when any thread panics
 panic = 'abort'
 
 [profile.bench]
-# Do not strip any debug info.  This helps the widest set of profiling tools
-debug = true
+# debug = 1 means line charts only, which is minimum needed for good stack traces
+debug = 1
 strip = 'none'
 
 [profile.simulator]

diff --git a/crates/sui-benchmark/src/bin/stress.rs b/crates/sui-benchmark/src/bin/stress.rs
@@ -252,6 +252,7 @@ async fn main() -> Result<()> {
         config.log_file = Some(opts.log_path);
     }
     let _guard = config.with_env().init();
+
     let registry: Arc<Registry> = Arc::new(metrics::start_prometheus_server(
         format!("{}:{}", opts.client_metric_host, opts.client_metric_port)
             .parse()

diff --git a/crates/sui-node/Cargo.toml b/crates/sui-node/Cargo.toml
@@ -31,12 +31,11 @@ sui-telemetry = { path = "../sui-telemetry" }
 sui-types = { path = "../sui-types" }
 narwhal-network = { path = "../../narwhal/network" }
 
+ying-profiler = { git = "https://github.com/velvia/ying-profiler", features = ["profile-spans"] }
+
 telemetry-subscribers.workspace = true
 mysten-network.workspace = true
 workspace-hack.workspace = true
 
 [target.'cfg(msim)'.dependencies]
 sui-simulator = { path = "../sui-simulator" }
-
-[target.'cfg(not(target_env = "msvc"))'.dependencies]
-jemalloc-ctl = "^0.5"
diff --git a/crates/sui-node/src/main.rs b/crates/sui-node/src/main.rs
@@ -11,7 +11,7 @@ use sui_node::metrics;
 use sui_telemetry::send_telemetry_event;
 use tokio::task;
 use tokio::time::sleep;
-use tracing::{debug, info};
+use tracing::info;
 
 #[derive(Parser)]
 #[clap(rename_all = "kebab-case", version)]
@@ -23,19 +23,12 @@ struct Args {
     listen_address: Option<Multiaddr>,
 }
 
-// Memory profiling is now done automatically based on increases in total memory usage.
-// Set MALLOC_CONF (when loading jemalloc dynamically) or _RJEM_MALLOC_CONF (when jemalloc is
-// linked statically) to:  prof:true
-// See [doc/src/contribute/observability.md] for more info.
-// For more memory profiling info see https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Heap-Profiling
-#[cfg(not(target_env = "msvc"))]
-use jemalloc_ctl::{epoch, stats};
+// Memory profiling is now done automatically by the Ying profiler.
+use ying_profiler::utils::ProfilerRunner;
+use ying_profiler::YingProfiler;
 
-// Ratio of memory used compared to before that triggers a new profiling dump
-const MEMORY_INCREASE_PROFILING_RATIO: f64 = 1.2;
-// Interval between checks for memory profile dumps
-const MEMORY_PROFILING_INTERVAL_SECS: u64 = 300;
-const PROF_DUMP: &[u8] = b"prof.dump\0";
+#[global_allocator]
+static YING_ALLOC: YingProfiler = YingProfiler;
 
 #[tokio::main]
 async fn main() -> Result<()> {
@@ -59,63 +52,10 @@ async fn main() -> Result<()> {
         config.network_address = listen_address;
     }
 
-    #[cfg(not(target_env = "msvc"))]
-    {
-        use jemalloc_ctl::config;
-        use std::ffi::CString;
-        use std::time::Duration;
-        use tracing::info;
-
-        let malloc_conf = config::malloc_conf::mib().unwrap();
-        info!("Default Jemalloc conf: {}", malloc_conf.read().unwrap());
-
-        std::thread::spawn(|| {
-            // This is the initial size of memory beyond which profiles are dumped
-            let mut last_allocated_mb = 100;
-            loop {
-                // many statistics are cached and only updated when the epoch is advanced.
-                epoch::advance().unwrap();
-
-                // NOTE: The below code does not return values when a malloc-based profiler like Bytehound
-                // is used.  Bytehound does not implement the stat APIs needed.
-                let allocated = stats::allocated::read().unwrap() / (1024 * 1024);
-                let resident = stats::resident::read().unwrap() / (1024 * 1024);
-                info!(
-                    "Jemalloc: {} MB allocated / {} MB resident",
-                    allocated, resident
-                );
-
-                // TODO: split this out into mysten-infra so everyone can pick it up
-                // The reason why we use manual code to dump out profiles is because the automatic JEPROF
-                // options dump out too often.  We really just want profiles when the retained memory
-                // keeps growing, as we want to know why.
-                // Setting the timestamp and memory size in the filename helps us pick the profiles
-                // to use when doing analysis.  Default JEPROF profiles just have a counter which is not
-                // helpful.  This helps correlate to time and total memory consumed.
-                //
-                // NOTE: One needs to set MALLOC_CONF to `prof:true` for the below to work
-                if (allocated as f64 / last_allocated_mb as f64) > MEMORY_INCREASE_PROFILING_RATIO {
-                    info!("Significant memory increase registered, dumping profile: new = {}, old = {}",
-                          allocated, last_allocated_mb);
-                    last_allocated_mb = allocated;
-
-                    // Formulate profiling filename based on ISO8601 timestamp and number of MBs
-                    let dt = chrono::offset::Local::now();
-                    let dt_str = dt.to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
-                    let dump_name = format!("jeprof.{}.{}MB.heap", dt_str, allocated);
-
-                    // Trigger profiling dump
-                    let dump_name_cstr = CString::new(dump_name).expect("Cannot create dump name");
-                    unsafe {
-                        if jemalloc_ctl::raw::write(PROF_DUMP, dump_name_cstr.as_ptr()).is_err() {
-                            debug!("Cannot dump memory profile, is _RJEM_MALLOC_CONF set to prof:true?");
-                        }
-                    }
-                }
-                std::thread::sleep(Duration::from_secs(MEMORY_PROFILING_INTERVAL_SECS));
-            }
-        });
-    }
+    // Spins up a thread to check memory usage every minute, and dump out stack traces/profiles
+    // if it has moved up or down more than 15%.  Also allow configuration of dump directory.
+    let profile_dump_dir = std::env::var("SUI_MEM_PROFILE_DIR").unwrap_or_default();
+    ProfilerRunner::new(60, 15, &profile_dump_dir).spawn();
 
     let is_validator = config.consensus_config().is_some();
     task::spawn(async move {

diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml
@@ -101,6 +101,7 @@ clap-f595c2ba2a3f28df = { package = "clap", version = "2", features = ["ansi_ter
 clap-7b89eefb6aaa9bf3 = { package = "clap", version = "3", features = ["atty", "clap_derive", "color", "derive", "once_cell", "std", "strsim", "suggestions", "termcolor"] }
 clap_lex = { version = "0.2", default-features = false }
 clear_on_drop = { version = "0.2", default-features = false }
+coarsetime = { version = "0.1", default-features = false }
 codespan = { version = "0.11", default-features = false, features = ["serde", "serialization"] }
 codespan-reporting = { version = "0.11", default-features = false, features = ["serde", "serialization"] }
 collectable = { version = "0.0.2", default-features = false }
@@ -610,11 +611,13 @@ web-sys = { version = "0.3", default-features = false, features = ["BinaryType",
 webpki = { version = "0.22", default-features = false, features = ["alloc", "std"] }
 webpki-roots = { version = "0.22", default-features = false }
 whoami = { version = "1" }
+wyhash = { version = "0.5", default-features = false }
 wyz = { version = "0.2", default-features = false, features = ["alloc"] }
 x509-parser = { version = "0.14" }
 yaml-rust = { version = "0.4", default-features = false }
 yansi = { version = "0.5", default-features = false }
 yasna = { version = "0.5", features = ["std", "time"] }
+ying-profiler = { git = "https://github.com/velvia/ying-profiler", default-features = false, features = ["profile-spans", "tracing"] }
 zeroize = { version = "1", features = ["alloc", "zeroize_derive"] }
 zstd-sys = { version = "2", features = ["legacy", "zdict_builder"] }
 
@@ -724,6 +727,7 @@ clap-7b89eefb6aaa9bf3 = { package = "clap", version = "3", features = ["atty", "
 clap_derive = { version = "3" }
 clap_lex = { version = "0.2", default-features = false }
 clear_on_drop = { version = "0.2", default-features = false }
+coarsetime = { version = "0.1", default-features = false }
 codespan = { version = "0.11", default-features = false, features = ["serde", "serialization"] }
 codespan-reporting = { version = "0.11", default-features = false, features = ["serde", "serialization"] }
 collectable = { version = "0.0.2", default-features = false }
@@ -1324,11 +1328,13 @@ webpki = { version = "0.22", default-features = false, features = ["alloc", "std
 webpki-roots = { version = "0.22", default-features = false }
 which = { version = "4", default-features = false }
 whoami = { version = "1" }
+wyhash = { version = "0.5", default-features = false }
 wyz = { version = "0.2", default-features = false, features = ["alloc"] }
 x509-parser = { version = "0.14" }
 yaml-rust = { version = "0.4", default-features = false }
 yansi = { version = "0.5", default-features = false }
 yasna = { version = "0.5", features = ["std", "time"] }
+ying-profiler = { git = "https://github.com/velvia/ying-profiler", default-features = false, features = ["profile-spans", "tracing"] }
 zeroize = { version = "1", features = ["alloc", "zeroize_derive"] }
 zeroize_derive = { version = "1", default-features = false }
 zstd-sys = { version = "2", features = ["legacy", "zdict_builder"] }

diff --git a/docker/sui-node/Dockerfile b/docker/sui-node/Dockerfile
@@ -51,9 +51,5 @@ COPY --from=builder /sui/target/release/sui-node /usr/local/bin
 
 ARG BUILD_DATE
 ARG GIT_REVISION
-# Use jemalloc as memory allocator
-ENV LD_PRELOAD /usr/lib/x86_64-linux-gnu/libjemalloc.so
-# Enable automatic memory profiling.
-ENV MALLOC_CONF prof:true,prof_prefix:/data/jeprof.out,lg_prof_interval:36
 LABEL build-date=$BUILD_DATE
 LABEL git-revision=$GIT_REVISION