[ci] skip flaky test on premerge (ray-project#41274)

Skip flaky tests on premerge, as signed off in https://docs.google.com/document/d/1IVgI7LNo21bPJO8kdYTwdWwEdZr5D9VtNrabUeX8gCc/edit#heading=h.isxl96agy1it. This will help save good amount of cost on PR, especially due to those gpu test runs.

Signed-off-by: can <[email protected]>

.buildkite/data.rayci.yml

@@ -142,6 +142,7 @@ steps:
     tags: 
       - python
       - data
+      - skip-on-premerge
     instance_type: medium
     soft_fail: true
     commands:

.buildkite/ml.rayci.yml

@@ -210,6 +210,7 @@ steps:
   - label: ":train: ml: flaky tests"
     tags: 
       - train
+      - skip-on-premerge
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //... ml --run-flaky-tests
@@ -239,6 +240,7 @@ steps:
   - label: ":train: ml: train gpu flaky tests"
     tags: 
       - train
+      - skip-on-premerge
       - gpu
     instance_type: gpu-large
     commands:

.buildkite/rllib.rayci.yml

@@ -222,6 +222,7 @@ steps:
   - label: ":brain: rllib: flaky multi-gpu tests"
     tags: 
       - rllib
+      - skip-on-premerge
       - gpu
     instance_type: gpu-large
     commands:
@@ -235,7 +236,9 @@ steps:
     job_env: forge
 
   - label: ":brain: rllib: flaky tests (learning tests)"
-    tags: rllib
+    tags: 
+      - rllib
+      - skip-on-premerge
     instance_type: large
     commands:
       # torch
@@ -262,7 +265,9 @@ steps:
     job_env: forge
 
   - label: ":brain: rllib: flaky tests (examples/rlmodule/models/tests_dir)"
-    tags: rllib
+    tags: 
+      - rllib
+      - skip-on-premerge
     instance_type: large
     commands:
       # examples
@@ -295,7 +300,9 @@ steps:
     job_env: forge
 
   - label: ":brain: rllib: flaky tests (memory leak)"
-    tags: rllib
+    tags: 
+      - rllib
+      - skip-on-premerge
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib 

.buildkite/serve.rayci.yml

@@ -169,6 +169,7 @@ steps:
   - label: ":ray-serve: serve: flaky tests"
     tags: 
       - serve
+      - skip-on-premerge
       - python
     instance_type: medium
     soft_fail: true

.buildkite/serverless.rayci.yml

@@ -43,7 +43,9 @@ steps:
     job_env: forge
 
   - label: ":serverless: serverless: flaky tests"
-    tags: python
+    tags: 
+      - python
+      - skip-on-premerge
     instance_type: medium
     soft_fail: true
     commands: