Merge pull request orchest#916 from orchest/feat/orchest-k8s-controller

Introduce Orchest Kubernetes Operator

.flake8

@@ -1,6 +1,7 @@
 [flake8]
 exclude =
-    jupyter_server_config.py,
+    jupyter_server_config.py
+    services/orchest-api/app/migrations/versions/*.py
 
 ignore =
     # Space before :
@@ -13,6 +14,7 @@ ignore =
 
 per-file-ignores =
     orchest-sdk/python/orchest/__init__.py: F401
+    orchest-cli/orchestcli/cli.py: W505, E501
     services/orchest-api/app/app/core/sessions/__init__.py: F401
     services/orchest-ctl/app/orchest/__init__.py: F401
 

.github/pull_request_template.md

@@ -12,6 +12,9 @@ Fixes: #issue
       migrations (refer to `scripts/migration_manager.sh`).
 - [ ] In case I changed code in the `orchest-sdk` I followed its [release
       checklist](https://github.com/orchest/orchest/blob/master/orchest-sdk/python/RELEASE-CHECKLIST.md)
-- [ ] I haven't introduced breaking changes that would disrupt existing jobs.
-- [ ] In case I changed the dependencies in any `requirements.in` I have run `pip-compile`
-      to update the corresponding `requirements.txt`.
+- [ ] In case I changed code in the `orchest-cli` I followed its [release
+      checklist](https://github.com/orchest/orchest/blob/master/orchest-cli/RELEASE-CHECKLIST.md)
+- [ ] I haven't introduced breaking changes that would disrupt existing jobs, i.e. backwards
+      compatibility is maintained.
+- [ ] In case I changed the dependencies in any `requirements.in` I have run `pip-compile` to update
+      the corresponding `requirements.txt`.

.github/workflows/docker-ci.yml

@@ -28,7 +28,8 @@ jobs:
             -i base-kernel-julia,
             -i update-sidecar,
             -i node-agent,
-            -i orchest-ctl -i celery-worker -i memory-server -i session-sidecar,
+            -i orchest-controller,
+            -i celery-worker -i memory-server -i session-sidecar,
           ]
 
     steps:

.github/workflows/sdk-pypi.yml → .github/workflows/publish-to-pypi.yml

@@ -1,4 +1,4 @@
-name: Publish SDK to PyPi
+name: Publish SDK & CLI to PyPi
 
 # Whenever Orchest is released, also publish the Orchest SDK.
 on:
@@ -30,3 +30,9 @@ jobs:
           cd orchest-sdk/python
           python setup.py sdist
           twine upload dist/* --non-interactive --skip-existing
+
+      - name: Upload CLI
+        run: |
+          cd orchest-cli
+          python setup.py sdist
+          twine upload dist/* --non-interactive --skip-existing

.github/workflows/update-controller-image-version.yml

@@ -0,0 +1,47 @@
+name: Update controller image version
+
+on:
+  pull_request:
+    branches:
+      - master
+    types: [opened, reopened]
+
+jobs:
+  update_controller_image_version:
+    if: ${{ startsWith(github.head_ref, 'release/') }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Install yq dependency
+        run: |
+          sudo wget https://github.com/mikefarah/yq/releases/download/v4.25.1/yq_linux_amd64 \
+            -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
+
+      # Can't do string manipulation in GHA expressions yet.
+      - name: Get release version
+        id: release_version
+        run: |
+          export release_version=$(echo ${{ github.head_ref }} | sed 's/release\///g')
+          echo ::set-output name=release_version::$release_version
+
+      - name: Update controller image version
+        run: |
+          export IMAGE_NAME="orchest/orchest-controller:${RELEASE_VERSION}"
+          echo "Using controller: ${IMAGE_NAME}"
+          yq -i '.spec.template.spec.containers[0].image = strenv(IMAGE_NAME)' \
+            services/orchest-controller/deploy-controller/deployment.yaml
+        env:
+          RELEASE_VERSION: ${{steps.release_version.outputs.release_version}}
+
+      - name: Commit and push changes
+        # Note: --allow-empty so that a pull reopen or, for whatever
+        # reason, doing the same release again will not lead to issues.
+        run: |
+          git config --global user.name "orchest"
+          git config --global user.email "[email protected]"
+          git commit -a -m "Update controller image version (automated)" --allow-empty
+          git push

.gitignore

@@ -158,6 +158,9 @@ debug-dump.tar.gz
 cypress/cypress/videos
 cypress/cypress/screenshots
 
+# Go debugger
+__debug_bin
+
 # ----- Don't ignore -------
 # Compiled custom GitHub Action
 !.github/custom-actions/github-linear-sync/dist/

.prettierignore

@@ -4,5 +4,6 @@ venv/
 .venvs/
 npm_modules/
 dist/
-deploy/
+services/orchest-controller/deploy/
+services/orchest-controller/deploy-controller/
 pnpm-lock.yaml

README.md

@@ -62,8 +62,9 @@ docs](https://docs.orchest.io/en/stable/getting_started/installation.html).
 
 The software in this repository is licensed as follows:
 
-- All content residing under the "orchest-sdk/" directory of this repository is licensed under the
-  "Apache-2.0" license as defined in "orchest-sdk/LICENSE".
+- All content residing under the "orchest-sdk/" and "orchest-cli/" directories of this repository
+  are licensed under the "Apache-2.0" license as defined in "orchest-sdk/LICENSE" and
+  "orchest-cli/LICENSE" respectively.
 - Content outside of the above mentioned directory is available under the "AGPL-3.0" license.
 
 ## Slack Community