eval: stabilise memory size plot and add more data points (#45)

sc2-sys · Nov 7, 2023 · cb03ec6 · cb03ec6
1 parent 74e5990
commit cb03ec6
Show file tree

Hide file tree

Showing 45 changed files with 71 additions and 90 deletions.
diff --git a/eval/plots/mem-size/mem_size.pdf b/eval/plots/mem-size/mem_size.pdf
diff --git a/eval/plots/mem-size/mem_size.png b/eval/plots/mem-size/mem_size.png
diff --git a/eval/results/mem-size/coco-fw-sig-enc_1.csv b/eval/results/mem-size/coco-fw-sig-enc_1.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697734213.5928617,1697734232.0
-1,1697734239.3558283,1697734257.0
-2,1697734263.449783,1697734281.0
+0,1699386917.1904116,1699386934.0
diff --git a/eval/results/mem-size/coco-fw-sig-enc_16.csv b/eval/results/mem-size/coco-fw-sig-enc_16.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697734486.314491,1697734506.0
-1,1697734512.4012096,1697734553.0
-2,1697734559.5635872,1697734579.0
+0,1699387012.0408642,1699387034.0
diff --git a/eval/results/mem-size/coco-fw-sig-enc_2.csv b/eval/results/mem-size/coco-fw-sig-enc_2.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697734287.5582354,1697734305.0
-1,1697734311.632289,1697734330.0
-2,1697734334.714039,1697734334.0
+0,1699386940.7697976,1699386957.0
diff --git a/eval/results/mem-size/coco-fw-sig-enc_32.csv b/eval/results/mem-size/coco-fw-sig-enc_32.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387674.7548294,1699387698.0
diff --git a/eval/results/mem-size/coco-fw-sig-enc_4.csv b/eval/results/mem-size/coco-fw-sig-enc_4.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697734338.755289,1697734356.0
-1,1697734362.8395243,1697734380.0
-2,1697734386.9313612,1697734404.0
+0,1699386963.8591907,1699386981.0
diff --git a/eval/results/mem-size/coco-fw-sig-enc_64.csv b/eval/results/mem-size/coco-fw-sig-enc_64.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699388451.5709944,1699388488.0
diff --git a/eval/results/mem-size/coco-fw-sig-enc_8.csv b/eval/results/mem-size/coco-fw-sig-enc_8.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697734411.0398479,1697734430.0
-1,1697734436.1101897,1697734455.0
-2,1697734461.2020004,1697734480.0
+0,1699386987.9530697,1699387005.0
diff --git a/eval/results/mem-size/coco-fw-sig_1.csv b/eval/results/mem-size/coco-fw-sig_1.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733827.1515234,1697733844.0
-1,1697733851.907207,1697733868.0
-2,1697733874.9889178,1697733892.0
+0,1699386793.7513826,1699386811.0
diff --git a/eval/results/mem-size/coco-fw-sig_16.csv b/eval/results/mem-size/coco-fw-sig_16.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697734133.9578545,1697734153.0
-1,1697734160.037981,1697734180.0
-2,1697734186.130995,1697734207.0
+0,1699386888.578647,1699386909.0
diff --git a/eval/results/mem-size/coco-fw-sig_2.csv b/eval/results/mem-size/coco-fw-sig_2.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733898.0983624,1697733936.0
-1,1697733942.2282069,1697733960.0
-2,1697733966.3211334,1697733984.0
+0,1699386817.3224385,1699386834.0
diff --git a/eval/results/mem-size/coco-fw-sig_32.csv b/eval/results/mem-size/coco-fw-sig_32.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387644.6701417,1699387668.0
diff --git a/eval/results/mem-size/coco-fw-sig_4.csv b/eval/results/mem-size/coco-fw-sig_4.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733990.425501,1697734007.0
-1,1697734013.4979336,1697734031.0
-2,1697734037.5878677,1697734055.0
+0,1699386840.4059284,1699386857.0
diff --git a/eval/results/mem-size/coco-fw-sig_64.csv b/eval/results/mem-size/coco-fw-sig_64.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387950.691294,1699387979.0
diff --git a/eval/results/mem-size/coco-fw-sig_8.csv b/eval/results/mem-size/coco-fw-sig_8.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697734061.7025578,1697734079.0
-1,1697734085.770161,1697734103.0
-2,1697734109.851056,1697734127.0
+0,1699386863.4899917,1699386882.0
diff --git a/eval/results/mem-size/coco-fw_1.csv b/eval/results/mem-size/coco-fw_1.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697735272.948704,1697735288.0
-1,1697735294.882638,1697735309.0
-2,1697735315.9643517,1697735330.0
+0,1699386682.2841241,1699386697.0
diff --git a/eval/results/mem-size/coco-fw_16.csv b/eval/results/mem-size/coco-fw_16.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697735685.286312,1697735704.0
-1,1697735710.3712687,1697735728.0
-2,1697735734.4552002,1697735752.0
+0,1699386769.1460922,1699386787.0
diff --git a/eval/results/mem-size/coco-fw_2.csv b/eval/results/mem-size/coco-fw_2.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697735337.0589366,1697735373.0
-1,1697735379.1998348,1697735394.0
-2,1697735400.2774098,1697735436.0
+0,1699386704.8947027,1699386719.0
diff --git a/eval/results/mem-size/coco-fw_32.csv b/eval/results/mem-size/coco-fw_32.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387617.51138,1699387638.0
diff --git a/eval/results/mem-size/coco-fw_4.csv b/eval/results/mem-size/coco-fw_4.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697735442.4481442,1697735458.0
-1,1697735464.5304694,1697735479.0
-2,1697735485.607388,1697735501.0
+0,1699386725.9975443,1699386740.0
diff --git a/eval/results/mem-size/coco-fw_64.csv b/eval/results/mem-size/coco-fw_64.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387913.551047,1699387943.0
diff --git a/eval/results/mem-size/coco-fw_8.csv b/eval/results/mem-size/coco-fw_8.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697735507.7148113,1697735633.0
-1,1697735639.1125834,1697735657.0
-2,1697735663.1923606,1697735679.0
+0,1699386747.0543973,1699386763.0
diff --git a/eval/results/mem-size/coco-nosev_1.csv b/eval/results/mem-size/coco-nosev_1.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699386485.5353503,1699386496.0
diff --git a/eval/results/mem-size/coco-nosev_16.csv b/eval/results/mem-size/coco-nosev_16.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699386553.3196282,1699386564.0
diff --git a/eval/results/mem-size/coco-nosev_2.csv b/eval/results/mem-size/coco-nosev_2.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699386502.1290562,1699386513.0
diff --git a/eval/results/mem-size/coco-nosev_32.csv b/eval/results/mem-size/coco-nosev_32.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699388225.418424,1699388237.0
diff --git a/eval/results/mem-size/coco-nosev_4.csv b/eval/results/mem-size/coco-nosev_4.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699386519.19493,1699386530.0
diff --git a/eval/results/mem-size/coco-nosev_64.csv b/eval/results/mem-size/coco-nosev_64.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387787.109399,1699387800.0
diff --git a/eval/results/mem-size/coco-nosev_8.csv b/eval/results/mem-size/coco-nosev_8.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699386536.2573595,1699386547.0
diff --git a/eval/results/mem-size/coco_1.csv b/eval/results/mem-size/coco_1.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733025.1773233,1697733039.0
-1,1697733045.97489,1697733060.0
-2,1697733067.0547917,1697733082.0
+0,1699386570.877572,1699386586.0
diff --git a/eval/results/mem-size/coco_16.csv b/eval/results/mem-size/coco_16.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733283.8869636,1697733300.0
-1,1697733306.9536378,1697733325.0
-2,1697733332.04397,1697733349.0
+0,1699386658.711661,1699386675.0
diff --git a/eval/results/mem-size/coco_2.csv b/eval/results/mem-size/coco_2.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733088.1544392,1697733103.0
-1,1697733109.20841,1697733124.0
-2,1697733130.291018,1697733145.0
+0,1699386593.469758,1699386609.0
diff --git a/eval/results/mem-size/coco_32.csv b/eval/results/mem-size/coco_32.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699388301.0678768,1699388322.0
diff --git a/eval/results/mem-size/coco_4.csv b/eval/results/mem-size/coco_4.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733151.3872952,1697733166.0
-1,1697733172.4563491,1697733188.0
-2,1697733194.5358543,1697733210.0
+0,1699386615.553849,1699386630.0
diff --git a/eval/results/mem-size/coco_64.csv b/eval/results/mem-size/coco_64.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699388141.5521042,1699388170.0
diff --git a/eval/results/mem-size/coco_8.csv b/eval/results/mem-size/coco_8.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697733216.639434,1697733233.0
-1,1697733239.7055316,1697733255.0
-2,1697733261.7911825,1697733277.0
+0,1699386636.6276655,1699386652.0
diff --git a/eval/results/mem-size/kata_1.csv b/eval/results/mem-size/kata_1.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697732765.5844097,1697732776.0
-1,1697732783.8956566,1697732794.0
-2,1697732800.9628544,1697732811.0
+0,1699386302.392664,1699386310.0
diff --git a/eval/results/mem-size/kata_16.csv b/eval/results/mem-size/kata_16.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697732972.6933293,1697732983.0
-1,1697732989.7416396,1697733000.0
-2,1697733006.8141966,1697733018.0
+0,1699386443.1146493,1699386450.0
diff --git a/eval/results/mem-size/kata_2.csv b/eval/results/mem-size/kata_2.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697732818.050173,1697732829.0
-1,1697732835.1180081,1697732846.0
-2,1697732852.1826577,1697732864.0
+0,1699386345.8243334,1699386353.0
diff --git a/eval/results/mem-size/kata_32.csv b/eval/results/mem-size/kata_32.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387490.7115676,1699387500.0
diff --git a/eval/results/mem-size/kata_4.csv b/eval/results/mem-size/kata_4.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697732870.2754269,1697732881.0
-1,1697732887.3352122,1697732898.0
-2,1697732904.404381,1697732915.0
+0,1699386357.8205192,1699386365.0
diff --git a/eval/results/mem-size/kata_64.csv b/eval/results/mem-size/kata_64.csv
@@ -0,0 +1,2 @@
+Run,StartTimeStampSec,EndTimeStampSec
+0,1699387741.5295427,1699387751.0
diff --git a/eval/results/mem-size/kata_8.csv b/eval/results/mem-size/kata_8.csv
@@ -1,4 +1,2 @@
 Run,StartTimeStampSec,EndTimeStampSec
-0,1697732921.504405,1697732932.0
-1,1697732938.5357118,1697732949.0
-2,1697732955.6084037,1697732966.0
+0,1699386399.941915,1699386408.0
diff --git a/tasks/eval/mem_size.py b/tasks/eval/mem_size.py
@@ -37,15 +37,10 @@ def get_default_vm_mem_size():
     return mem
 
 
-def update_vm_mem_size(baseline, new_mem_size):
+def update_vm_mem_size(toml_path, new_mem_size):
     """
     Update the default VM memory size in the Kata config file
     """
-    if baseline == "kata":
-        toml_path = join(KATA_CONFIG_DIR, "configuration-qemu.toml")
-    else:
-        toml_path = join(KATA_CONFIG_DIR, "configuration-qemu-sev.toml")
-
     updated_toml_str = """
     [hypervisor.qemu]
     default_memory = {mem_size}
@@ -101,9 +96,9 @@ def run(ctx, baseline=None, mem_size_mult=None):
             raise RuntimeError("Unrecognised baseline")
         baselines_to_run = [baseline]
 
-    mem_size_multiplier = [1, 2, 4, 8, 16]
+    mem_size_multiplier = [1, 2, 4, 8, 16, 32, 64]
     if mem_size_mult is not None:
-        mem_size_multiplier = [mem_size_mult]
+        mem_size_multiplier = [int(mem_size_mult)]
 
     results_dir = join(RESULTS_DIR, "mem-size")
     if not exists(results_dir):
@@ -115,7 +110,7 @@ def run(ctx, baseline=None, mem_size_mult=None):
     service_template_file = join(APPS_DIR, "mem-size", "service.yaml.j2")
     image_name = "csegarragonz/coco-helloworld-py"
     used_images = ["csegarragonz/coco-knative-sidecar", image_name]
-    num_runs = 3
+    num_runs = 1
 
     # Get the default memory size
     default_vm_mem_size = get_default_vm_mem_size()
@@ -145,11 +140,13 @@ def run(ctx, baseline=None, mem_size_mult=None):
             init_csv_file(result_file, "Run,StartTimeStampSec,EndTimeStampSec")
 
             # Update the configuration file to start VMs with more memory
-            update_vm_mem_size(bline, mem_size_mult * default_vm_mem_size)
+            update_vm_mem_size(
+                baseline_traits["conf_file"], mem_size_mult * default_vm_mem_size
+            )
 
             for nr in range(num_runs):
                 print(
-                    "Executing baseline {} ({} * {} mem) run {}/{}...".format(
+                    "Executing baseline {} ({} * {} MB mem) run {}/{}...".format(
                         bline, mem_size_mult, default_vm_mem_size, nr + 1, num_runs
                     )
                 )
@@ -159,7 +156,7 @@ def run(ctx, baseline=None, mem_size_mult=None):
 
         # Reset the VM memory size to the default value (different baselines
         # may use different Kata config files)
-        update_vm_mem_size(bline, default_vm_mem_size)
+        update_vm_mem_size(baseline_traits["conf_file"], default_vm_mem_size)
 
 
 @task
@@ -209,9 +206,9 @@ def plot(ctx):
         )
 
     # Misc
-    xlabels = ["{} * {}".format(x, get_default_vm_mem_size()) for x in xs]
-    ax.set_xticks(xs, xlabels, rotation=30)
-    ax.set_xlabel("Initial VM memory size")
+    xlabels = ["{}".format(int(x * get_default_vm_mem_size() / 1024)) for x in xs]
+    ax.set_xticks(xs, xlabels)
+    ax.set_xlabel("Initial VM memory size [GB]")
     ax.set_ylabel("Time [s]")
     ax.set_ylim(bottom=0)
     ax.set_title("Impact of initial VM memory size on start-up time")
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699387674.7548294,1699387698.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699388451.5709944,1699388488.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699387644.6701417,1699387668.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699387950.691294,1699387979.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699387617.51138,1699387638.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699387913.551047,1699387943.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699386485.5353503,1699386496.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699386553.3196282,1699386564.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699386502.1290562,1699386513.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Run,StartTimeStampSec,EndTimeStampSec
		0,1699388225.418424,1699388237.0