aviary-cluster.yaml

# An unique identifier for the head node and workers of this cluster.
cluster_name: deploy

# Cloud-provider specific configuration.
provider:
    type: local
    head_ip: 172.16.251.127
    worker_ips: [172.16.248.191]

docker:
    image: "docker.io/vincentpli/aviary:0.0.1-651cb061e4b912927ee33a4e690c54f0c389beaa"
    container_name: "llmserve"

auth:
    ssh_user: root
    # You can comment out `ssh_private_key` if the following machines don't need a private key for SSH access to the Ray
    # cluster:
    #   (1) The machine on which `ray up` is executed.
    #   (2) The head node of the Ray cluster.
    #
    # The machine that runs ray up executes SSH commands to set up the Ray head node. The Ray head node subsequently
    # executes SSH commands to set up the Ray worker nodes. When you run ray up, ssh credentials sitting on the ray up
    # machine are copied to the head node -- internally, the ssh key is added to the list of file mounts to rsync to head node.
    ssh_private_key: ~/.ssh/id_rsa

min_workers: 1
max_workers: 1
# All the 'conda activate' are necessary to ensure we are in the
# python 3.10 conda env.
setup_commands:
    - echo "conda activate" >> ~/.bashrc

head_setup_commands:
    - conda activate

worker_setup_commands: []

head_start_ray_commands:
    - conda activate && ray stop
    - conda activate && ulimit -n 65536; ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --dashboard-host=0.0.0.0

worker_start_ray_commands:
    - conda activate && ray stop
    # We need to make sure RAY_HEAD_IP env var is accessible
    # after conda activate.
    - export RAY_HEAD_IP=172.16.251.127 && echo "export RAY_HEAD_IP=$RAY_HEAD_IP" >> ~/.bashrc && conda activate && ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076