[Release tests] Create compute config for new dataset shuffle tests (r…

…ay-project#24239)

Use a separate compute config that uses smaller instance types and no object store memory limit for the new shuffle implementation. I verified that the config works on master for dataset_shuffle_* tests.

Related issue number

ray-project#24176: the added tests would verify the instance types which support the new shuffle implementations.

release/nightly_tests/shuffle/datasets_large_scale_compute_small_instances.yaml

@@ -0,0 +1,19 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+aws:
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+            VolumeSize: 500
+
+head_node_type:
+    name: head_node
+    instance_type:  m5.4xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: m5.4xlarge
+      min_workers: 19
+      max_workers: 19
+      use_spot: false

release/release_tests.yaml

@@ -3397,11 +3397,9 @@
   run:
     timeout: 7200
     script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9 --shuffle
-
     wait_for_nodes:
       num_nodes: 20
       timeout: 900
-
     type: sdk_command
     file_manager: sdk
 
@@ -3423,11 +3421,57 @@
   run:
     timeout: 7200
     script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9
+    wait_for_nodes:
+      num_nodes: 20
+      timeout: 900
+    type: sdk_command
+    file_manager: sdk
 
+- name: dataset_shuffle_random_shuffle_1tb_small_instances
+  group: core-dataset-tests
+  working_dir: nightly_tests
+  legacy:
+    test_name: dataset_shuffle_random_shuffle_1tb_small_instances
+    test_suite: dataset_test
+
+  stable: false
+
+  frequency: nightly
+  team: core
+  cluster:
+    cluster_env: shuffle/shuffle_app_config.yaml
+    cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
+
+  run:
+    timeout: 7200
+    script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9 --shuffle
     wait_for_nodes:
       num_nodes: 20
       timeout: 900
+    type: sdk_command
+    file_manager: sdk
 
+- name: dataset_shuffle_sort_1tb_small_instances
+  group: core-dataset-tests
+  working_dir: nightly_tests
+  legacy:
+    test_name: dataset_shuffle_sort_1tb_small_instances
+    test_suite: dataset_test
+
+  stable: false
+
+  frequency: nightly
+  team: core
+  cluster:
+    cluster_env: shuffle/shuffle_app_config.yaml
+    cluster_compute: shuffle/datasets_large_scale_compute_small_instances.yaml
+
+  run:
+    timeout: 7200
+    script: python dataset/sort.py --num-partitions=1000 --partition-size=1e9
+    wait_for_nodes:
+      num_nodes: 20
+      timeout: 900
     type: sdk_command
     file_manager: sdk