devspace cluster (smartcontractkit#9773)

* core cluster * fix comments * proper nixpkgs * ECR example * split to runner and node profiles * speed up the sync, add remote-runner
faretdinovfi · Jul 24, 2023 · d0127a5 · d0127a5
1 parent 3e19151
commit d0127a5
Show file tree

Hide file tree

Showing 21 changed files with 1,130 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -76,3 +76,7 @@ MacOSX*
 *.out
 
 contracts/yarn.lock
+
+
+# Ignore DevSpace cache and log folder
+.devspace/
diff --git a/GNUmakefile b/GNUmakefile
@@ -53,6 +53,10 @@ chainlink-dev: operator-ui ## Build a dev build of chainlink binary.
 chainlink-test: operator-ui ## Build a test build of chainlink binary.
 	go build -tags test $(GOFLAGS) .
 
+.PHONY: chainlink-local-start
+chainlink-local-start:
+	./chainlink -c /etc/node-secrets-volume/default.toml -c /etc/node-secrets-volume/overrides.toml -secrets /etc/node-secrets-volume/secrets.toml node start -d -p /etc/node-secrets-volume/node-password -a /etc/node-secrets-volume/apicredentials --vrfpassword=/etc/node-secrets-volume/apicredentials
+
 .PHONY: install-solana
 install-solana: ## Build & install the chainlink-solana binary.
 	go install $(GOFLAGS) ./plugins/cmd/chainlink-solana

diff --git a/charts/chainlink-cluster/.helmignore b/charts/chainlink-cluster/.helmignore
@@ -0,0 +1,24 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+.devspace/
diff --git a/charts/chainlink-cluster/Chart.yaml b/charts/chainlink-cluster/Chart.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+name: chainlink-cluster
+description: Chainlink nodes cluster
+version: 0.1.0
+appVersion: '0.1.0'
diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md
@@ -0,0 +1,93 @@
+# Chainlink cluster
+Example CL nodes cluster for system level tests
+
+Enter the shell
+```
+nix develop
+```
+
+# Develop
+
+## New cluster
+We are using [devspace](https://www.devspace.sh/docs/getting-started/installation?x0=3)
+
+Configure the cluster, see `deployments.app.helm.values` and [values.yaml](./values.yaml) comments
+
+Set your registry for the image, example for `ECR`:
+```
+aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${aws_account}.dkr.ecr.us-west-2.amazonaws.com
+export DEVSPACE_IMAGE="${aws_account}.dkr.ecr.us-west-2.amazonaws.com/chainlink-devspace"
+```
+Enter the shell and deploy
+```
+nix develop
+cd charts/chainlink-cluster
+
+# set your unique namespace if it's a new cluster
+devspace use namespace cl-cluster
+devspace deploy
+```
+If you don't need a build use
+```
+devspace deploy --skip-build
+```
+
+Connect to your environment
+```
+devspace dev -p node
+make chainlink
+make chainlink-local-start
+```
+Fix something in the code locally, it'd automatically sync, rebuild it inside container and run again
+```
+make chainlink
+make chainlink-local-start
+```
+If you need to update the whole cluster run `deploy` again with a new set of images
+```
+devspace reset pods
+devspace deploy
+```
+Destroy the cluster
+```
+devspace purge
+```
+
+If you need to run some system level tests inside k8s use `runner` profile:
+```
+devspace dev -p runner
+```
+
+If you used `devspace dev ...` always use `devspace reset pods` to switch the pods back
+
+## Debug existing cluster
+If you need to debug CL node that is already deployed change `dev.app.container` and `dev.app.labelSelector` in [devspace.yaml](devspace.yaml) if they are not default and run:
+```
+devspace dev -p node
+or
+devspace dev -p runner
+```
+
+## Automatic file sync
+When you run `devspace dev` your files described in `dev.app.sync` of [devspace.yaml](devspace.yaml) will be uploaded to the switched container
+
+After that all the changes will be synced automatically
+
+Check `.profiles` to understand what is uploaded in profiles `runner` and `node`
+
+# Helm
+If you would like to use `helm` directly, please uncomment data in `values.yaml`
+## Install
+```
+helm install -f values-raw-helm.yaml cl-cluster .
+```
+
+## Helm Test
+```
+helm test cl-cluster
+```
+
+## Uninstall
+```
+helm uninstall cl-cluster
+```
diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml
@@ -0,0 +1,142 @@
+version: v2beta1
+name: chainlink
+
+vars:
+  DEVSPACE_IMAGE:
+    source: env
+
+# This is a list of `pipelines` that DevSpace can execute (you can define your own)
+pipelines:
+  #       create_deployments --all  can be used to apply manifests again
+  # This is the pipeline for the main command: `devspace dev` (or `devspace run-pipeline dev`)
+  dev:
+    run: |-
+      run_dependencies --all       # 1. Deploy any projects this project needs (see "dependencies")
+      ensure_pull_secrets --all    # 2. Ensure pull secrets
+      start_dev app                # 3. Start dev mode "app" (see "dev" section)
+  # You can run this pipeline via `devspace deploy` (or `devspace run-pipeline deploy`)
+  deploy:
+    run: |-
+      run_dependencies --all                            # 1. Deploy any projects this project needs (see "dependencies")
+      ensure_pull_secrets --all                         # 2. Ensure pull secrets
+      build_images --all -t $(git describe --always)    # 3. Build, tag (git commit hash) and push all images (see "images")
+      create_deployments --all                          # 4. Deploy Helm charts and manifests specfied as "deployments"
+
+images:
+  app:
+    image: ${DEVSPACE_IMAGE}
+    dockerfile: ../../core/chainlink.devspace.Dockerfile
+    context: ../..
+
+# This is a list of `deployments` that DevSpace can create for this project
+deployments:
+  app:
+    helm:
+      chart:
+        name: cl-cluster
+        path: .
+      # for simplicity, we define all the values here
+      # they can be defined the same way in values.yml
+      # devspace merging this "values" and "values.yml" before deploy
+      values:
+        runner:
+          image: ${DEVSPACE_IMAGE}
+          stateful: false
+        geth:
+          wsrpc-port: 8546
+          httprpc-port: 8544
+          networkid: 1337
+          blocktime: 1
+        mockserver:
+          port: 1080
+        db:
+          stateful: false
+        chainlink:
+          web_port: 6688
+          p2p_port: 8090
+          nodes:
+            - name: node-1
+              image: ${DEVSPACE_IMAGE}
+              version: latest
+            - name: node-2
+              image: ${DEVSPACE_IMAGE}
+              version: latest
+            - name: node-3
+              image: ${DEVSPACE_IMAGE}
+              version: latest
+            - name: node-4
+              image: ${DEVSPACE_IMAGE}
+              version: latest
+        podAnnotations: { }
+        nodeSelector: { }
+        tolerations: [ ]
+        affinity: { }
+
+profiles:
+  # this replaces only "runner" pod, usable when you'd like to run some system level tests inside k8s
+  - name: runner
+    patches:
+      - op: replace
+        path: dev.app.workingDir
+        value: /home/root/chainlink/integration-tests
+      - op: replace
+        path: dev.app.container
+        value: runner
+      - op: replace
+        path: dev.app.labelSelector.instance
+        value: runner-1
+      - op: remove
+        path: dev.app.sync[1].uploadExcludePaths[0]
+      - op: remove
+        path: dev.app.open
+      - op: remove
+        path: dev.app.ports[1]
+  - name: node
+    patches:
+      - op: replace
+        path: dev.app.container
+        value: node
+      - op: replace
+        path: dev.app.labelSelector.instance
+        value: node-1
+
+# This is a list of `dev` containers that are based on the containers created by your deployments
+dev:
+  app:
+    workingDir: /home/root/chainlink
+    container: node
+    labelSelector:
+      instance: node-1
+    # Sync files between the local filesystem and the development container
+    sync:
+      - path: ../../core/services/chainlink:/home/root/chainlink/core/services/chainlink
+        printLogs: true
+        disableDownload: true
+      - path: ../..:/home/root/chainlink
+        printLogs: true
+        disableDownload: true
+        uploadExcludePaths:
+          - integration-tests/
+          - .git/
+          - .github/
+          - belt/
+          - charts/
+          - contracts/
+          - node_modules/
+          - integration/
+          - integration-scripts/
+          - testdata/
+          - evm-test-helpers/
+          - tools/
+    # Open a terminal and use the following command
+    terminal:
+      command: bash
+    ssh:
+      enabled: true
+    proxyCommands:
+      - command: devspace
+      - command: kubectl
+      - command: helm
+      - gitCredentials: true
+    ports:
+      - port: "2345"
diff --git a/charts/chainlink-cluster/devspace_start.sh b/charts/chainlink-cluster/devspace_start.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+set +e  # Continue on errors
+
+COLOR_BLUE="\033[0;94m"
+COLOR_GREEN="\033[0;92m"
+COLOR_RESET="\033[0m"
+
+# Print useful output for user
+echo -e "${COLOR_BLUE}
+     %########%      
+     %###########%       ____                 _____                      
+         %#########%    |  _ \   ___ __   __ / ___/  ____    ____   ____ ___ 
+         %#########%    | | | | / _ \\\\\ \ / / \___ \ |  _ \  / _  | / __// _ \\
+     %#############%    | |_| |(  __/ \ V /  ____) )| |_) )( (_| |( (__(  __/
+     %#############%    |____/  \___|  \_/   \____/ |  __/  \__,_| \___\\\\\___|
+ %###############%                                  |_|
+ %###########%${COLOR_RESET}
+
+
+Welcome to your development container!
+
+This is how you can work with it:
+- Files will be synchronized between your local machine and this container
+- Some ports will be forwarded, so you can access this container via localhost
+- Run \`${COLOR_GREEN}go run main.go${COLOR_RESET}\` to start the application
+"
+
+# Set terminal prompt
+export PS1="\[${COLOR_BLUE}\]devspace\[${COLOR_RESET}\] ./\W \[${COLOR_BLUE}\]\\$\[${COLOR_RESET}\] "
+if [ -z "$BASH" ]; then export PS1="$ "; fi
+
+# Include project's bin/ folder in PATH
+export PATH="./bin:$PATH"
+
+# Open shell
+bash --norc
diff --git a/charts/chainlink-cluster/templates/chainlink-cm.yaml b/charts/chainlink-cluster/templates/chainlink-cm.yaml
@@ -0,0 +1,57 @@
+{{- range $cfg := .Values.chainlink.nodes }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app: {{ $.Release.Name }}-{{ $cfg.name }}-cm
+    release: {{ $.Release.Name }}
+  name: {{ $.Release.Name }}-{{ $cfg.name }}-cm
+data:
+  apicredentials: |
+    [email protected]
+    fj293fbBnlQ!f9vNs
+  node-password: T.tLHkcmwePT/p,]sYuntjwHKAsrhm#4eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ
+  init.sql: |
+    CREATE
+    EXTENSION pg_stat_statements;
+  default.toml: |
+  {{ if (hasKey $cfg "toml") }}
+    {{ $cfg.toml | nindent 4 }}
+  {{ else }}
+    RootDir = './clroot'
+    [Log]
+    JSONConsole = true
+    Level = 'debug'
+    [WebServer]
+    AllowOrigins = '*'
+    SecureCookies = false
+    SessionTimeout = '999h0m0s'
+    [OCR]
+    Enabled = true
+    [P2P]
+    [P2P.V1]
+    Enabled = true
+    ListenIP = '0.0.0.0'
+    ListenPort = 6690
+    [[EVM]]
+    ChainID = '1337'
+    MinContractPayment = '0'
+    [[EVM.Nodes]]
+    Name = 'node-0'
+    WSURL = 'ws://geth:8546'
+    HTTPURL = 'http://geth:8544'
+    [WebServer.TLS]
+    HTTPSPort = 0
+    {{ end }}
+  overrides.toml: |
+  {{ if (hasKey $cfg "overridesToml") }}
+    {{ $cfg.overridesToml | nindent 4 }}
+  {{ else }}
+  {{ end }}
+  secrets.toml: |
+  {{ if (hasKey $cfg "secretsToml") }}
+    {{ $cfg.secretsToml | nindent 4 }}
+  {{ else }}
+  {{ end }}
+---
+{{- end }}