Implement automated partition testing (solana-labs#7222)

kruelmanta · Jan 22, 2020 · 7587656 · 7587656
1 parent 0230746
commit 7587656
Show file tree

Hide file tree

Showing 22 changed files with 193 additions and 46 deletions.
diff --git a/net/net.sh b/net/net.sh
@@ -1009,14 +1009,15 @@ logs)
   ;;
 netem)
   if [[ -n $netemConfigFile ]]; then
+    remoteNetemConfigFile="$(basename "$netemConfigFile")"
     if [[ $netemCommand = "add" ]]; then
       for ipAddress in "${validatorIpList[@]}"; do
         "$here"/scp.sh "$netemConfigFile" solana@"$ipAddress":~/solana
       done
     fi
     for i in "${!validatorIpList[@]}"; do
       "$here"/ssh.sh solana@"${validatorIpList[$i]}" 'solana/scripts/net-shaper.sh' \
-      "$netemCommand" ~solana/solana/"$netemConfigFile" "${#validatorIpList[@]}" "$i"
+      "$netemCommand" ~solana/solana/"$remoteNetemConfigFile" "${#validatorIpList[@]}" "$i"
     done
   else
     num_nodes=$((${#validatorIpList[@]}*netemPartition/100))

diff --git a/system-test/netem-configs/partial-loss-three-partitions b/system-test/netem-configs/partial-loss-three-partitions
@@ -0,0 +1,39 @@
+{
+  "partitions":[
+    34,
+    33,
+    33
+  ],
+  "interconnects":[
+    {
+      "a":0,
+      "b":1,
+      "config":"loss 15% delay 25ms"
+    },
+    {
+      "a":1,
+      "b":0,
+      "config":"loss 15% delay 25ms"
+    },
+    {
+      "a":0,
+      "b":2,
+      "config":"loss 10% delay 15ms"
+    },
+    {
+      "a":2,
+      "b":0,
+      "config":"loss 10% delay 15ms"
+    },
+    {
+      "a":2,
+      "b":1,
+      "config":"loss 5% delay 5ms"
+    },
+    {
+      "a":1,
+      "b":2,
+      "config":"loss 5% delay 5ms"
+    }
+  ]
+}
diff --git a/system-test/partition-testcases/colo-3-partition.yml b/system-test/partition-testcases/colo-3-partition.yml
@@ -0,0 +1,19 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "Colo - CPU Only - Partial Loss 3 Partitions"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "true"
+      CLOUD_PROVIDER: "colo"
+      TESTNET_TAG: "colo-perf-cpu-only"
+      NUMBER_OF_VALIDATOR_NODES: 4
+      ENABLE_GPU: "false"
+      NUMBER_OF_CLIENT_NODES: 2
+      CLIENT_OPTIONS: "bench-tps=2=--tx_count 15000 --thread-batch-sleep-ms 250"
+      ADDITIONAL_FLAGS: ""
+      APPLY_PARTITIONS: "true"
+      NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
+      PARTITION_ACTIVE_DURATION: 30
+      PARTITION_INACTIVE_DURATION: 30
+      PARTITION_ITERATION_COUNT: 5
+    agents:
+      - "queue=colo-deploy"
diff --git a/system-test/partition-testcases/colo-partition-long-sanity-test.yml b/system-test/partition-testcases/colo-partition-long-sanity-test.yml
@@ -0,0 +1,19 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "Colo - CPU Only - Partial Loss 3 Partitions"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "true"
+      CLOUD_PROVIDER: "colo"
+      TESTNET_TAG: "colo-perf-cpu-only"
+      NUMBER_OF_VALIDATOR_NODES: 4
+      ENABLE_GPU: "false"
+      NUMBER_OF_CLIENT_NODES: 1
+      CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250"
+      ADDITIONAL_FLAGS: ""
+      APPLY_PARTITIONS: "true"
+      NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
+      PARTITION_ACTIVE_DURATION: 60
+      PARTITION_INACTIVE_DURATION: 60
+      PARTITION_ITERATION_COUNT: 10
+    agents:
+      - "queue=colo-deploy"
diff --git a/system-test/partition-testcases/colo-partition-sanity-test.yml b/system-test/partition-testcases/colo-partition-sanity-test.yml
@@ -0,0 +1,19 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "Colo - CPU Only - Partial Loss 3 Partitions"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "true"
+      CLOUD_PROVIDER: "colo"
+      TESTNET_TAG: "colo-perf-cpu-only"
+      NUMBER_OF_VALIDATOR_NODES: 4
+      ENABLE_GPU: "false"
+      NUMBER_OF_CLIENT_NODES: 1
+      CLIENT_OPTIONS: "bench-tps=1=--tx_count 15000 --thread-batch-sleep-ms 250"
+      ADDITIONAL_FLAGS: ""
+      APPLY_PARTITIONS: "true"
+      NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
+      PARTITION_ACTIVE_DURATION: 30
+      PARTITION_INACTIVE_DURATION: 30
+      PARTITION_ITERATION_COUNT: 2
+    agents:
+      - "queue=colo-deploy"
diff --git a/system-test/partition-testcases/gce-5-node-3-partition.yml b/system-test/partition-testcases/gce-5-node-3-partition.yml
@@ -0,0 +1,22 @@
+steps:
+  - command: "system-test/testnet-automation.sh"
+    label: "GCE - CPU Only 5 Node - Partial Loss 3 Partitions"
+    env:
+      UPLOAD_RESULTS_TO_SLACK: "true"
+      CLOUD_PROVIDER: "gce"
+      TESTNET_TAG: "gce-perf-cpu-only"
+      NUMBER_OF_VALIDATOR_NODES: 5
+      ENABLE_GPU: "false"
+      VALIDATOR_NODE_MACHINE_TYPE: "--machine-type n1-standard-16"
+      NUMBER_OF_CLIENT_NODES: 2
+      CLIENT_OPTIONS: "bench-tps=2=--tx_count 15000 --thread-batch-sleep-ms 250"
+      TESTNET_ZONES: "us-west1-a"
+      USE_PUBLIC_IP_ADDRESSES: "true"
+      ADDITIONAL_FLAGS: ""
+      APPLY_PARTITIONS: "true"
+      NETEM_CONFIG_FILE: "system-test/netem-configs/partial-loss-three-partitions"
+      PARTITION_ACTIVE_DURATION: 30
+      PARTITION_INACTIVE_DURATION: 30
+      PARTITION_ITERATION_COUNT: 5
+    agents:
+      - "queue=testnet-deploy"
diff --git a/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-10-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "ec2"
       TESTNET_TAG: "aws-perf-cpu-only"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 10
       ENABLE_GPU: "false"

diff --git a/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml b/system-test/performance-testcases/aws-cpu-only-perf-5-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "ec2"
       TESTNET_TAG: "aws-perf-cpu-only"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 5
       ENABLE_GPU: "false"

diff --git a/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml b/system-test/performance-testcases/azure-cpu-only-perf-5-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "azure"
       TESTNET_TAG: "azure-perf-cpu-only"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 5
       ENABLE_GPU: "false"

diff --git a/system-test/performance-testcases/colo-cpu-only-perf.yml b/system-test/performance-testcases/colo-cpu-only-perf.yml
@@ -6,7 +6,6 @@ steps:
       CLOUD_PROVIDER: "colo"
       TESTNET_TAG: "colo-perf-cpu-only"
       ENABLE_GPU: "false"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 4
       NUMBER_OF_CLIENT_NODES: 2

diff --git a/system-test/performance-testcases/colo-gpu-perf-high-txcount.yml b/system-test/performance-testcases/colo-gpu-perf-high-txcount.yml
@@ -6,7 +6,6 @@ steps:
       CLOUD_PROVIDER: "colo"
       TESTNET_TAG: "colo-perf-gpu-enabled"
       ENABLE_GPU: "true"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 4
       NUMBER_OF_CLIENT_NODES: 2

diff --git a/system-test/performance-testcases/colo-gpu-perf.yml b/system-test/performance-testcases/colo-gpu-perf.yml
@@ -6,7 +6,6 @@ steps:
       CLOUD_PROVIDER: "colo"
       TESTNET_TAG: "colo-perf-gpu-enabled"
       ENABLE_GPU: "true"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 4
       NUMBER_OF_CLIENT_NODES: 2

diff --git a/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-10-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-perf-cpu-only"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 10
       ENABLE_GPU: "false"

diff --git a/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml b/system-test/performance-testcases/gce-cpu-only-perf-5-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-perf-cpu-only"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 5
       ENABLE_GPU: "false"
@@ -14,6 +13,6 @@ steps:
       CLIENT_OPTIONS: "bench-tps=2=--tx_count 20000 --thread-batch-sleep-ms 250"
       TESTNET_ZONES: "us-west1-a,us-west1-b,us-central1-a,europe-west4-a"
       USE_PUBLIC_IP_ADDRESSES: "true"
-      ADDITIONAL_FLAGS: "--dedicated"
+      ADDITIONAL_FLAGS: ""
     agents:
       - "queue=gce-deploy"
diff --git a/system-test/performance-testcases/gce-gpu-perf-10-node.yml b/system-test/performance-testcases/gce-gpu-perf-10-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-perf-gpu-enabled"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 10
       ENABLE_GPU: "true"

diff --git a/system-test/performance-testcases/gce-gpu-perf-100-node.yml b/system-test/performance-testcases/gce-gpu-perf-100-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-perf-gpu-enabled"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 100
       ENABLE_GPU: "true"

diff --git a/system-test/performance-testcases/gce-gpu-perf-25-node.yml b/system-test/performance-testcases/gce-gpu-perf-25-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-perf-gpu-enabled"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 25
       ENABLE_GPU: "true"

diff --git a/system-test/performance-testcases/gce-gpu-perf-5-node.yml b/system-test/performance-testcases/gce-gpu-perf-5-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-perf-gpu-enabled"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 5
       ENABLE_GPU: "true"

diff --git a/system-test/performance-testcases/gce-gpu-perf-50-node.yml b/system-test/performance-testcases/gce-gpu-perf-50-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-perf-gpu-enabled"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 600
       NUMBER_OF_VALIDATOR_NODES: 50
       ENABLE_GPU: "true"

diff --git a/system-test/stability-testcases/gce-stability-5-node.yml b/system-test/stability-testcases/gce-stability-5-node.yml
@@ -5,7 +5,6 @@ steps:
       UPLOAD_RESULTS_TO_SLACK: "true"
       CLOUD_PROVIDER: "gce"
       TESTNET_TAG: "gce-stability"
-      RAMP_UP_TIME: 0
       TEST_DURATION_SECONDS: 28800
       NUMBER_OF_VALIDATOR_NODES: 5
       ENABLE_GPU: "false"