CI][Chaos-test] chaos test now can set max-nodes-to-kill ray-project#…

…20962

python/ray/_private/test_utils.py

@@ -957,7 +957,8 @@ def teardown_tls(key_filepath, cert_filepath, temp_dir):
 def get_and_run_node_killer(node_kill_interval_s,
                             namespace=None,
                             lifetime=None,
-                            no_start=False):
+                            no_start=False,
+                            max_nodes_to_kill=2):
     assert ray.is_initialized(), (
         "The API is only available when Ray is initialized.")
 
@@ -1058,7 +1059,9 @@ def _get_alive_nodes(self, nodes):
         namespace=namespace,
         name="node_killer",
         lifetime=lifetime).remote(
-            head_node_id, node_kill_interval_s=node_kill_interval_s)
+            head_node_id,
+            node_kill_interval_s=node_kill_interval_s,
+            max_nodes_to_kill=max_nodes_to_kill)
     print("Waiting for node killer actor to be ready...")
     ray.get(node_killer.ready.remote())
     print("Node killer actor is ready now.")

release/nightly_tests/setup_chaos.py

@@ -7,6 +7,7 @@
 def parse_script_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("--node-kill-interval", type=int, default=60)
+    parser.add_argument("--max-nodes-to-kill", type=int, default=2)
     parser.add_argument(
         "--no-start",
         action="store_true",
@@ -31,7 +32,8 @@ def main():
         args.node_kill_interval,
         namespace="release_test_namespace",
         lifetime="detached",
-        no_start=args.no_start)
+        no_start=args.no_start,
+        max_nodes_to_kill=args.max_nodes_to_kill)
     print("Successfully deployed a node killer.")