cluster.yaml

#
# Copyright IBM Corporation 2021
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

cluster_name: rayvens-cluster
max_workers: 2
upscaling_speed: 1.0
idle_timeout_minutes: 5
provider:
    type: kubernetes
    use_internal_ips: true
    namespace: ray
    autoscaler_service_account:
        apiVersion: v1
        kind: ServiceAccount
        metadata:
            name: rayvens
    autoscaler_role:
        kind: Role
        apiVersion: rbac.authorization.k8s.io/v1
        metadata:
            name: rayvens
        rules:
            - apiGroups: [""]
              resources: ["pods", "pods/status", "pods/exec", "pods/log"]
              verbs: ["get", "watch", "list", "create", "delete", "patch"]
            - apiGroups: ["camel.apache.org"]
              resources: ["integrations", "integrations/status"]
              verbs: ["*"]
    autoscaler_role_binding:
        apiVersion: rbac.authorization.k8s.io/v1
        kind: RoleBinding
        metadata:
            name: rayvens
        subjects:
            - kind: ServiceAccount
              name: rayvens
        roleRef:
            kind: Role
            name: rayvens
            apiGroup: rbac.authorization.k8s.io
    services:
        - apiVersion: v1
          kind: Service
          metadata:
              name: rayvens-cluster-head
          spec:
              selector:
                  component: rayvens-cluster-head
              ports:
                  - name: client
                    protocol: TCP
                    port: 10001
                    targetPort: 10001
                  - name: ray-serve
                    protocol: TCP
                    port: 8000
                    targetPort: 8000
                  - name: dashboard
                    protocol: TCP
                    port: 8265
                    targetPort: 8265
head_node_type: head_node
available_node_types:
    worker_node:
        min_workers: 0
        max_workers: 2
        resources: { "worker": 1 }
        node_config:
            apiVersion: v1
            kind: Pod
            metadata:
                generateName: rayvens-cluster-worker-
            spec:
                serviceAccountName: rayvens
                restartPolicy: Never
                volumes:
                    - name: dshm
                      emptyDir:
                          medium: Memory
                    - name: podinfo
                      downwardAPI:
                          items:
                              - path: "labels"
                                fieldRef:
                                    fieldPath: metadata.labels
                              - path: "name"
                                fieldRef:
                                    fieldPath: metadata.name
                              - path: "namespace"
                                fieldRef:
                                    fieldPath: metadata.namespace
                containers:
                    - name: ray-node
                      imagePullPolicy: Always
                      image: localhost:5000/rayvens
                      command: ["/bin/bash", "-c", "--"]
                      args: ["trap : TERM INT; sleep infinity & wait;"]
                      volumeMounts:
                          - mountPath: /dev/shm
                            name: dshm
                          - name: podinfo
                            mountPath: /etc/podinfo
                      resources:
                          requests:
                              cpu: 1000m
                              memory: 2G
                          limits:
                              memory: 2G
    head_node:
        min_workers: 0
        max_workers: 0
        resources: { "head": 1 }
        node_config:
            apiVersion: v1
            kind: Pod
            metadata:
                generateName: rayvens-cluster-head-
                labels:
                    component: rayvens-cluster-head
            spec:
                serviceAccountName: rayvens
                restartPolicy: Never
                volumes:
                    - name: dshm
                      emptyDir:
                          medium: Memory
                    - name: podinfo
                      downwardAPI:
                          items:
                              - path: "labels"
                                fieldRef:
                                    fieldPath: metadata.labels
                              - path: "name"
                                fieldRef:
                                    fieldPath: metadata.name
                              - path: "namespace"
                                fieldRef:
                                    fieldPath: metadata.namespace
                containers:
                    - name: ray-node
                      image: localhost:5000/rayvens
                      command: ["/bin/bash", "-c", "--"]
                      args: ["trap : TERM INT; sleep infinity & wait;"]
                      ports:
                          - containerPort: 6379
                          - containerPort: 10001
                          - containerPort: 8265
                      volumeMounts:
                          - mountPath: /dev/shm
                            name: dshm
                          - name: podinfo
                            mountPath: /etc/podinfo
                      resources:
                          requests:
                              cpu: 1000m
                              memory: 2G
                          limits:
                              memory: 2G
                      env:
                          - name: RAY_ADDRESS
                            value: auto
head_start_ray_commands:
    - ray stop
    - ulimit -n 65536; ray start --head --autoscaling-config=~/ray_bootstrap_config.yaml --dashboard-host 0.0.0.0
worker_start_ray_commands:
    - ray stop
    - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379