Node 572 - HW validation

hw_validation/benchmark.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+graceful_exit() {
+    echo "Failed. Check logs for more info"
+    exit 1
+}
+
+print_exec() {
+    echo "Running: $@"
+    "$@" || graceful_exit
+}
+
+if ! command -v sysbench &>/dev/null; then
+    echo "sysbench not found. Run 'sudo apt install sysbench'"
+    exit
+fi
+
+TMP_DIR_FS=$(stat -f -c %T /tmp)
+if [ "$TMP_DIR_FS" == "tmpfs" ]; then
+    echo "/tmp is mounted as a tmpfs. Need a suitable location for benchmarking file io from disk. Exiting."
+    exit
+fi
+
+NUM_PROCS=$(nproc)
+OUTPUT_FILE="$(pwd)/bench_results_$(hostname)_$(date +%Y-%m-%dT%H-%M-%S).txt"
+
+# Wrapped in code block to reroute all output to log
+{
+
+    # CPU tests - 1 vs all
+    print_exec sysbench cpu run \
+        --threads=1 \
+        --time=30 \
+        --validate=on
+
+    print_exec sysbench cpu run \
+        --threads="$NUM_PROCS" \
+        --time=30 \
+        --validate=on
+
+    # Mem test - small blocks vs large blocks * single thread vs all threads
+    # Why? Large blocks can demonstrate faster mem speeds
+    print_exec sysbench memory run \
+        --threads=1 \
+        --memory-total-size=200G \
+        --memory-block-size=4K \
+        --validate=on
+
+    print_exec sysbench memory run \
+        --threads=1 \
+        --memory-total-size=200G \
+        --memory-block-size=2G \
+        --validate=on
+
+    print_exec sysbench memory run \
+        --threads="$NUM_PROCS" \
+        --memory-total-size=200G \
+        --memory-block-size=4K \
+        --validate=on
+
+    print_exec sysbench memory run \
+        --threads="$NUM_PROCS" \
+        --memory-total-size=200G \
+        --memory-block-size=2G \
+        --validate=on
+
+    print_exec sysbench memory run \
+        --threads="$NUM_PROCS" \
+        --memory-total-size=200G \
+        --memory-block-size=4K \
+        --memory-access-mode=rnd \
+        --validate=on
+
+    # File IO test
+    ## Benchmarks: sync, mmap, fsync all, fsync every 10th
+
+    ## Use tmp to not junk up CWD
+    TEMP_DIR="/tmp/sysbench_$(date +%s)"
+    mkdir -p "$TEMP_DIR"
+    pushd "$TEMP_DIR" || {
+        echo "Unable to create test directory $TEMP_DIR"
+        exit 1
+    }
+    print_exec sysbench fileio prepare --file-test-mode=rndrw
+
+    ## Single threaded
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=sync
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=mmap
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=sync --file-fsync-all=on
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=sync --file-fsync-freq=10
+
+    ## Multi threaded
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=sync --threads="$NUM_PROCS"
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=mmap --threads="$NUM_PROCS"
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=sync --threads="$NUM_PROCS" --file-fsync-all=on
+    print_exec sysbench fileio run --file-test-mode=rndrw --file-io-mode=sync --threads="$NUM_PROCS" --file-fsync-freq=10
+
+    print_exec sysbench fileio cleanup
+    popd || { exit 1; }
+    rmdir "$TEMP_DIR"
+
+} >>"$OUTPUT_FILE"

hw_validation/readme.md

@@ -0,0 +1,32 @@
+# IC Hardware Benchmark and Validation Scripts
+
+## Dependencies (ubuntu)
+
+* sysbench
+* stress-ng
+
+Install: `sudo apt install sysbench stress-ng`
+
+
+## benchmark.sh
+
+Uses `sysbench` to gather cpu, memory, file-io performance stats. It will write a log to the working directory.
+
+### Analyze
+
+Side by side including command line invocation and speed differences:
+
+E.g.: `diff -y bench_results_zh2-spm01.zh2.dfinity.network_2022-08-12T21-08-32.txt bench_results_zh2-asu01_2022-08-12T23-08-49.txt | fgrep -e"+ sysbench" -e"|" > bench_comparison_gen1_gen2.txt`
+
+
+## stress.sh
+
+Uses `stress-ng` to exercise all HW components.
+
+Run it: `./stress.sh` - it writes a log file to the working directory.
+
+### Analyze
+
+Visually scan log for errors.
+
+`stress-ng` recommends not to use the reported speeds as benchmarks.

hw_validation/stress.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+if ! command -v stress-ng &>/dev/null; then
+    echo "stress-ng not found. Run 'sudo apt install stress-ng'"
+    exit 1
+fi
+
+set -u
+set -x
+
+# Exercise all stressors sequentially. Use all processors.
+# Time out after 10 seconds for each stressor.
+# Print metrics. Verify outputs where relevant.
+# Note that using the `--all` parameter instead of `--sequential` may crash the machine.
+stress-ng --sequential "$(nproc)" \
+    --log-file "./stress_test_$(hostname)_$(date +%Y-%m-%dT%H-%M-%S).txt" \
+    --timeout 30 \
+    --metrics \
+    --verify \
+    --times \
+    --exclude chattr || {
+    echo "Failed. Check logs"
+    exit 1
+}