[CI] Add GCE variances to Data tests (ray-project#34105)

This PR configures BuildKite to run Data release tests on GCE. I excluded the parquet_metadata_resolution and shuffle_data_loader release tests because more work is required to migrate those tests. --------- Signed-off-by: Balaji Veeramani <[email protected]>
Ransnomy · Apr 17, 2023 · 8af7814 · 8af7814
1 parent 90ae25c
commit 8af7814
Show file tree

Hide file tree

Showing 8 changed files with 243 additions and 10 deletions.
diff --git a/release/nightly_tests/dataset/data_ingest_benchmark_compute_gce.yaml b/release/nightly_tests/dataset/data_ingest_benchmark_compute_gce.yaml
@@ -1,6 +1,6 @@
 cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
 region: us-west1
-allowed_azs: 
+allowed_azs:
     - us-west1-c
 
 max_workers: 19

diff --git a/release/nightly_tests/dataset/inference_gce.yaml b/release/nightly_tests/dataset/inference_gce.yaml
@@ -0,0 +1,28 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+- us-west1-b
+
+max_workers: 999
+
+head_node_type:
+    name: head_node
+    instance_type: n1-standard-32-nvidia-tesla-t4-1
+
+
+worker_node_types:
+    - name: worker_node
+      instance_type: n2-standard-32 # aws m5.8xlarge
+      min_workers: 0
+      max_workers: 0
+      use_spot: false
+      resources:
+        cpu: 32
+    - name: gpu_node
+      instance_type: n1-standard-32-nvidia-tesla-t4-1 # aws g4dn.16xlarge
+      min_workers: 1
+      max_workers: 1
+      use_spot: false
+      resources:
+        cpu: 64
+        gpu: 1
diff --git a/release/nightly_tests/dataset/pipelined_training_compute_gce.yaml b/release/nightly_tests/dataset/pipelined_training_compute_gce.yaml
@@ -0,0 +1,23 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+- us-west1-b
+
+
+max_workers: 999
+
+head_node_type:
+    name: head_node
+    instance_type: n2-highmem-16 # i3.8xlarge
+
+worker_node_types:
+    - name: memory_node
+      instance_type: n2-highmem-16 # i3.8xlarge
+      min_workers: 10
+      max_workers: 10
+      use_spot: false
+    - name: gpu_node
+      instance_type: n1-highmem-32-nvidia-tesla-v100-4 # p3.8xlarge
+      min_workers: 4
+      max_workers: 4
+      use_spot: false
diff --git a/release/nightly_tests/dataset/shuffle_compute_gce.yaml b/release/nightly_tests/dataset/shuffle_compute_gce.yaml
@@ -0,0 +1,22 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+- us-west1-b
+
+max_workers: 999
+
+head_node_type:
+    name: head_node
+    instance_type: n1-standard-16-nvidia-tesla-t4-1 # g3.4xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: n1-standard-16-nvidia-tesla-t4-1 # g3.4xlarge
+      min_workers: 4
+      max_workers: 4
+      use_spot: false
+    - name: worker_node_2
+      instance_type: c2-standard-30 # c5.9xlarge
+      min_workers: 2
+      max_workers: 2
+      use_spot: false
diff --git a/release/nightly_tests/dataset/single_node_benchmark_compute_gce.yaml b/release/nightly_tests/dataset/single_node_benchmark_compute_gce.yaml
@@ -0,0 +1,17 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+- us-west1-c
+
+max_workers: 0
+
+head_node_type:
+    name: head_node
+    instance_type: n2-standard-16 # m5.4xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: n2-standard-16 # m5.4xlarge
+      max_workers: 0
+      min_workers: 0
+      use_spot: false
diff --git a/release/nightly_tests/shuffle/100tb_shuffle_compute_gce.yaml b/release/nightly_tests/shuffle/100tb_shuffle_compute_gce.yaml
@@ -1,6 +1,6 @@
 cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
 region: us-west1
-allowed_azs: 
+allowed_azs:
     - us-west1-c
 
 gcp_advanced_configurations_json:

diff --git a/release/nightly_tests/shuffle/datasets_large_scale_compute_small_instances_gce.yaml b/release/nightly_tests/shuffle/datasets_large_scale_compute_small_instances_gce.yaml
@@ -1,14 +1,15 @@
 cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
 region: us-west1
-allowed_azs: 
+allowed_azs:
     - us-west1-c
 
-#aws:
-#    BlockDeviceMappings:
-#        - DeviceName: /dev/sda1
-#          Ebs:
-#            DeleteOnTermination: true
-#            VolumeSize: 1000
+gcp_advanced_configurations_json:
+  instance_properties:
+    disks:
+      - boot: true
+        auto_delete: true
+        initialize_params:
+          disk_size_gb: 1000
 
 head_node_type:
     name: head_node