[chaos-test] chaos test pipeline ingestion (ray-project#20929)

since it has been passing my test run; i'll land it and mark it as unstable.

release/.buildkite/build_pipeline.py

@@ -102,6 +102,7 @@ def __init__(self, name: str, retry: int = 0):
     "~/ray/release/nightly_tests/chaos_test.yaml": [
         "chaos_many_actors",
         "chaos_many_tasks_no_object_store",
+        "chaos_pipelined_ingestion_1500_gb_15_windows",
     ],
     "~/ray/release/microbenchmark/microbenchmark.yaml": [
         "microbenchmark",

release/nightly_tests/chaos_test.yaml

@@ -47,3 +47,19 @@
     script: python dask_on_ray/large_scale_test.py --num_workers 150 --worker_obj_store_size_in_gb 70 --error_rate 0  --data_save_path /tmp/ray
 
   stable: false
+
+- name: chaos_pipelined_ingestion_1500_gb_15_windows
+  owner:
+    mail: "[email protected]"
+    slack: "@Chen Shen"
+
+  cluster:
+    app_config: dataset/pipelined_ingestion_app.yaml
+    compute_template: dataset/pipelined_ingestion_compute.yaml
+
+  run:
+    timeout: 7200
+    prepare: python wait_cluster.py 21 2400;  python setup_chaos.py --node-kill-interval 300
+    script: python dataset/pipelined_training.py --epochs 1 --num-windows 15  --num-files 915 --debug
+
+  stable: false

release/nightly_tests/dataset/pipelined_ingestion_app.yaml

@@ -1,5 +1,5 @@
 base_image: "anyscale/ray-ml:nightly-py37-gpu"
-
+env_vars: {"RAY_lineage_pinning_enabled": "1"}
 python:
   pip_packages:
     - pyarrow