[CI] Re-balance concurrency groups to allow more quota for large te…

…sts (ray-project#24344)

Currently nightly tests are unable to finish in a day because of concurrency group limit on `large` tests. This is an attempt to adjust the limits so buildkite can run / finish more tests. I will observe which tests fall into the `enormous` group and adjust the test resource / concurrency group limits again.

release/ray_release/buildkite/concurrency.py

@@ -11,10 +11,11 @@
 
 
 CONCURRENY_GROUPS = {
-    "tiny": 32,
-    "small": 16,
-    "medium": 4,
-    "large": 2,
+    "tiny": 32,  # <= 1k vCPU
+    "small": 16,  # <= 2k vCPU
+    "medium": 6,  # <= 3k vCPU
+    "large": 8,  # <= 8k vCPU
+    "enormous": 1,  # <= 4k vCPU (?)
     "small-gpu": 8,
     "large-gpu": 4,
 }
@@ -27,9 +28,11 @@
 gpu_cpu_to_concurrency_groups = [
     Condition(min_gpu=9, max_gpu=-1, min_cpu=0, max_cpu=-1, group="large-gpu"),
     Condition(min_gpu=1, max_gpu=9, min_cpu=0, max_cpu=-128, group="small-gpu"),
-    Condition(min_gpu=0, max_gpu=0, min_cpu=513, max_cpu=-1, group="large"),
+    Condition(min_gpu=0, max_gpu=0, min_cpu=1025, max_cpu=-1, group="enormous"),
+    Condition(min_gpu=0, max_gpu=0, min_cpu=513, max_cpu=1024, group="large"),
     Condition(min_gpu=0, max_gpu=0, min_cpu=129, max_cpu=512, group="medium"),
     Condition(min_gpu=0, max_gpu=0, min_cpu=0, max_cpu=32, group="tiny"),
+    # Make sure "small" is the last in the list, because it is the fallback.
     Condition(min_gpu=0, max_gpu=0, min_cpu=0, max_cpu=128, group="small"),
 ]
 

release/ray_release/tests/test_buildkite.py

@@ -573,7 +573,9 @@ def test_concurrency(cpu, gpu, group):
         test_concurrency(12800, 9, "large-gpu")
         test_concurrency(12800, 8, "small-gpu")
         test_concurrency(12800, 1, "small-gpu")
-        test_concurrency(12800, 0, "large")
+        test_concurrency(12800, 0, "enormous")
+        test_concurrency(1025, 0, "enormous")
+        test_concurrency(1024, 0, "large")
         test_concurrency(513, 0, "large")
         test_concurrency(512, 0, "medium")
         test_concurrency(129, 0, "medium")