Skip to content

Commit

Permalink
release logs for 2.4.0 (ray-project#33905)
Browse files Browse the repository at this point in the history 
Release logs perf benchmark for 2.4.0
Also updated tool to sort the regressions

Signed-off-by: Clarence Ng <[email protected]>
Co-authored-by: Clarence Ng <[email protected]>
  • Loading branch information
@clarng @clarence-wu
clarng and clarence-wu authored Apr 17, 2023
1 parent 0dafee9 commit 4367c80
Show file tree
Hide file tree
Showing 11 changed files with 520 additions and 481 deletions.
60 changes: 30 additions & 30 deletions release/release_logs/2.4.0/benchmarks/many_actors.json
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
{
"_dashboard_memory_usage_mb": 500.24448,
"_dashboard_test_success": true,
"_peak_memory": 3.58,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t1.71GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2646\t0.87GiB\tpython distributed/test_many_actors.py\n464\t0.32GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n43\t0.1GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n764\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n2870\t0.08GiB\tray::DashboardTester.run\n40\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=aph0_Ckc\n2781\t0.06GiB\tray::MemoryMonitorActor.run\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.103.191:9031 --host=0.0.0.\n663\t0.05GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/_private/log_m",
"actors_per_second": 808.4407587905971,
"num_actors": 10000,
"perf_metrics": [
{
"perf_metric_name": "actors_per_second",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 808.4407587905971
},
{
"perf_metric_name": "dashboard_p50_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 30.46
},
{
"perf_metric_name": "dashboard_p95_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 1722.469
},
{
"perf_metric_name": "dashboard_p99_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 2693.468
}
],
"success": "1",
"time": 12.369490146636963
"_dashboard_memory_usage_mb": 513.560576,
"_dashboard_test_success": true,
"_peak_memory": 3.91,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n165\t2.02GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2388\t0.85GiB\tpython distributed/test_many_actors.py\n353\t0.37GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n41\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n670\t0.09GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n38\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --ServerApp.token=agh0_CkgwRgIhAP\n2610\t0.07GiB\tray::DashboardTester.run\n2523\t0.07GiB\tray::MemoryMonitorActor.run\n280\t0.04GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.113.246:9031 --host=0.0.0.\n553\t0.04GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/_private/log_m",
"actors_per_second": 772.644103201044,
"num_actors": 10000,
"perf_metrics": [
{
"perf_metric_name": "actors_per_second",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 772.644103201044
},
{
"perf_metric_name": "dashboard_p50_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 34.714
},
{
"perf_metric_name": "dashboard_p95_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 2419.503
},
{
"perf_metric_name": "dashboard_p99_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 3842.061
}
],
"success": "1",
"time": 12.942569494247437
}
72 changes: 36 additions & 36 deletions release/release_logs/2.4.0/benchmarks/many_nodes.json
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,38 @@
{
"_dashboard_memory_usage_mb": 259.821568,
"_dashboard_test_success": true,
"_peak_memory": 1.91,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t0.73GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n464\t0.22GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n3871\t0.21GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n43\t0.1GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n764\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n4156\t0.07GiB\tray::StateAPIGeneratorActor.start\n4093\t0.07GiB\tray::DashboardTester.run\n40\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=agh0_Ckc\n4006\t0.06GiB\tray::MemoryMonitorActor.run\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.103.29:9031 --host=0.0.0.0",
"num_tasks": 1000,
"perf_metrics": [
{
"perf_metric_name": "tasks_per_second",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 221.7103038870812
},
{
"perf_metric_name": "used_cpus_by_deadline",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 250.0
},
{
"perf_metric_name": "dashboard_p50_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 3.287
},
{
"perf_metric_name": "dashboard_p95_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 40.367
},
{
"perf_metric_name": "dashboard_p99_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 133.659
}
],
"success": "1",
"tasks_per_second": 221.7103038870812,
"time": 304.51039028167725,
"used_cpus": 250.0
"_dashboard_memory_usage_mb": 186.179584,
"_dashboard_test_success": true,
"_peak_memory": 4.23,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n277\t0.59GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n1900\t0.22GiB\tpython distributed/test_many_tasks.py --num-tasks=1000\n477\t0.16GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n811\t0.09GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n56\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n2221\t0.08GiB\tray::StateAPIGeneratorActor.start\n1486\t0.08GiB\tray::JobSupervisor\n46\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --allow-root --ip=127.0.0.1 --no-\n2047\t0.07GiB\tray::MemoryMonitorActor.run\n2144\t0.07GiB\tray::DashboardTester.run",
"num_tasks": 1000,
"perf_metrics": [
{
"perf_metric_name": "tasks_per_second",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 216.16404352694366
},
{
"perf_metric_name": "used_cpus_by_deadline",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 250.0
},
{
"perf_metric_name": "dashboard_p50_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 3.915
},
{
"perf_metric_name": "dashboard_p95_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 56.099
},
{
"perf_metric_name": "dashboard_p99_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 130.237
}
],
"success": "1",
"tasks_per_second": 216.16404352694366,
"time": 304.62611627578735,
"used_cpus": 250.0
}
60 changes: 30 additions & 30 deletions release/release_logs/2.4.0/benchmarks/many_pgs.json
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
{
"_dashboard_memory_usage_mb": 219.471872,
"_dashboard_test_success": true,
"_peak_memory": 2.34,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t1.06GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2303\t0.39GiB\tpython distributed/test_many_pgs.py\n464\t0.19GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n43\t0.1GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n590\t0.08GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/raylet/raylet --raylet_socket_name=\n764\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n40\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=aph0_CkY\n2441\t0.06GiB\tray::MemoryMonitorActor.run\n2525\t0.06GiB\tray::DashboardTester.run\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.104.246:9031 --host=0.0.0.",
"num_pgs": 1000,
"perf_metrics": [
{
"perf_metric_name": "pgs_per_second",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 17.712835160069687
},
{
"perf_metric_name": "dashboard_p50_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 3.14
},
{
"perf_metric_name": "dashboard_p95_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 12.292
},
{
"perf_metric_name": "dashboard_p99_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 209.376
}
],
"pgs_per_second": 17.712835160069687,
"success": "1",
"time": 56.45623588562012
"_dashboard_memory_usage_mb": 166.48192,
"_dashboard_test_success": true,
"_peak_memory": 4.85,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n256\t1.02GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n1705\t0.4GiB\tpython distributed/test_many_pgs.py\n471\t0.15GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n803\t0.11GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n61\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n608\t0.08GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/raylet/raylet --raylet_socket_name=\n1300\t0.07GiB\tray::JobSupervisor\n48\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --allow-root --ip=127.0.0.1 --no-\n1963\t0.07GiB\tray::DashboardTester.run\n1852\t0.07GiB\tray::MemoryMonitorActor.run",
"num_pgs": 1000,
"perf_metrics": [
{
"perf_metric_name": "pgs_per_second",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 17.929446347235622
},
{
"perf_metric_name": "dashboard_p50_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 3.307
},
{
"perf_metric_name": "dashboard_p95_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 128.925
},
{
"perf_metric_name": "dashboard_p99_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 197.038
}
],
"pgs_per_second": 17.929446347235622,
"success": "1",
"time": 55.77417063713074
}
18 changes: 9 additions & 9 deletions release/release_logs/2.4.0/benchmarks/many_tasks.json
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
"_dashboard_memory_usage_mb": 478.588928,
"_dashboard_memory_usage_mb": 659.90656,
"_dashboard_test_success": true,
"_peak_memory": 4.46,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n291\t2.13GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2292\t0.86GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n464\t0.6GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n2532\t0.1GiB\tray::DashboardTester.run\n43\t0.1GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n764\t0.08GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n2595\t0.07GiB\tray::StateAPIGeneratorActor.start\n38\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-notebook --NotebookApp.token=aph0_Ckg\n2430\t0.06GiB\tray::MemoryMonitorActor.run\n391\t0.05GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.120.51:9031 --host=0.0.0.0",
"_peak_memory": 4.44,
"_peak_process_memory": "PID\tMEM\tCOMMAND\n165\t1.99GiB\t/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s\n2232\t0.87GiB\tpython distributed/test_many_tasks.py --num-tasks=10000\n353\t0.74GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dashboa\n2457\t0.1GiB\tray::DashboardTester.run\n41\t0.09GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy\n670\t0.09GiB\t/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen\n38\t0.07GiB\t/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/jupyter-lab --ServerApp.token=agh0_CkYwRAIgH_\n2520\t0.07GiB\tray::StateAPIGeneratorActor.start\n2370\t0.07GiB\tray::MemoryMonitorActor.run\n280\t0.04GiB\t/home/ray/anaconda3/bin/python -m ray.util.client.server --address=172.31.96.232:9031 --host=0.0.0.0",
"num_tasks": 10000,
"perf_metrics": [
{
"perf_metric_name": "tasks_per_second",
"perf_metric_type": "THROUGHPUT",
"perf_metric_value": 195.39358126525104
"perf_metric_value": 324.3236630929032
},
{
"perf_metric_name": "used_cpus_by_deadline",
Expand All @@ -18,21 +18,21 @@
{
"perf_metric_name": "dashboard_p50_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 4.171
"perf_metric_value": 5.019
},
{
"perf_metric_name": "dashboard_p95_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 592.153
"perf_metric_value": 2788.74
},
{
"perf_metric_name": "dashboard_p99_latency_ms",
"perf_metric_type": "LATENCY",
"perf_metric_value": 1262.576
"perf_metric_value": 3431.297
}
],
"success": "1",
"tasks_per_second": 195.39358126525104,
"time": 351.17875385284424,
"tasks_per_second": 324.3236630929032,
"time": 330.83339619636536,
"used_cpus": 2500.0
}
Loading

0 comments on commit 4367c80

Please sign in to comment.