KM CI Pipeline #2861
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright 2021 Kontain Inc | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| name: KM CI Pipeline | |
| on: | |
| pull_request: | |
| branches: [master] | |
| paths-ignore: | |
| # See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet | |
| - "**.md" # all .md files in repo | |
| - "**/docs/**" # all content of all docs/ dirs in repo | |
| - compile_commands.json | |
| - .vscode/** | |
| - km-releases | |
| - payloads/longhaul-test/** | |
| - "**/L0-image**" | |
| push: | |
| branches: [master] | |
| paths-ignore: | |
| - "**.md" # all .md files in repo | |
| - "**/docs/**" # all content of all docs/ dirs in repo | |
| - compile_commands.json | |
| - .vscode/** | |
| - km-releases | |
| - payloads/longhaul-test/** | |
| - "**/L0-image**" | |
| schedule: | |
| # Posix cron format: | |
| # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 | |
| # Minute Hour DayOfMonth MonthOfYear DayOfWeek | |
| - cron: "0 7 * * *" # Daily build midnight pacific time (UTC + 7) | |
| # Github doc says: | |
| # Scheduled workflows run on the latest commit on the default or base branch. | |
| # The shortest interval you can run scheduled workflows is once every 5 minutes. | |
| # Manual trigger. | |
| # See https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/ | |
| workflow_dispatch: | |
| inputs: | |
| run_type: | |
| description: "Run type: regular or nightly" | |
| default: "regular" | |
| required: true | |
| env: | |
| BUILDENV_IMAGE_VERSION: latest # use this for all buildenv containers | |
| IMAGE_VERSION: ci-${{ github.run_number }} # use this for all other containers | |
| SP_SUBSCRIPTION_ID: ${{ secrets.SP_SUBSCRIPTION_ID }} | |
| SP_APPID: ${{ secrets.SP_APPID }} | |
| SP_PASSWORD: ${{ secrets.SP_PASSWORD }} | |
| SP_TENANT: ${{ secrets.SP_TENANT }} | |
| # TRACE: true # uncomment to enable '-x' in all bash scripts | |
| jobs: | |
| # starts self-hosted runner in AWS and Azure. They are NOT ephemeral and will run until cleanup in the stop-runner | |
| start-runners: | |
| uses: ./.github/workflows/start-cloud-runners.yaml | |
| secrets: inherit | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| km-build: | |
| name: Build KM | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: ./.github/actions/km-build | |
| - name: Build faktory | |
| run: make -C tools/faktory all | |
| - name: Build shim | |
| run: | | |
| make -C cloud/k8s/shim | |
| mkdir build/opt/kontain/bin/shim | |
| cp -r build/cloud/k8s/shim/* build/opt/kontain/bin/shim | |
| # passing hello_test.kmd that is required by testenv of payloads - it already has correct RPATH. | |
| # passing hello_test.c and libhelper.a | |
| # because we will need to rebuild hello_test.kmd with correct RPATH for runtime environments | |
| - name: Upload km-build artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: km-build | |
| path: | | |
| build/cloud/k8s/shim | |
| build/opt/kontain/bin | |
| build/opt/kontain/lib | |
| build/opt/kontain/include | |
| tests/hello_test.kmd | |
| tests/hello_test.c | |
| tests/libhelper.a | |
| container-runtime/docker_config.sh | |
| build/opt/kontain/runtime | |
| build/cloud/k8s/shim/containerd-shim-krun-v2 | |
| retention-days: 7 | |
| km-build-payloads: | |
| name: Build payloads with test and demo images | |
| runs-on: ubuntu-20.04 | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| needs: km-build | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Download km-build artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: km-build | |
| path: . | |
| - name: Change permissions for build/opt/kontain/bin/* files | |
| run: | | |
| chmod +x build/opt/kontain/bin/* | |
| - name: Login into Azure Repository | |
| run: make -C cloud/azure login-cli | |
| - name: Prepare KM build env | |
| run: make -C tests pull-buildenv-image .buildenv-local-lib | |
| - name: Build payloads and create test images | |
| run: | | |
| make -C payloads -j pull-buildenv-image | |
| make -C payloads -j clean | |
| make -C payloads -j all | |
| make -C payloads -j testenv-image | |
| make -C payloads -j push-testenv-image | |
| - name: Create payloads runenv and demo-runenv images | |
| # Note: this builds both runenv and demo-runenv images | |
| run: | | |
| make -C payloads -j runenv-image | |
| make -C payloads -j push-demo-runenv-image | |
| # Note: custom Python build needs to happen after python payload build | |
| - name: Python.km custom build | |
| run: | | |
| make -C payloads/python build-modules pack-modules CUSTOM_MODULES="markupsafe wrapt" | |
| make -C payloads/python custom CUSTOM_MODULES="markupsafe wrapt" | |
| # Builds a static linked, stripped binary for krun/crun | |
| # This uses the NIX virtual machine, just like the crun CI. | |
| # NIX takes a long time, so we build the static krun binary in | |
| # parallel with the rest of KM build and test. | |
| krun-static-build: | |
| name: Build static KRUN binary | |
| runs-on: ubuntu-20.04 | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| # make sure krun has been build and artifact exists. | |
| - uses: ./.github/actions/configure-krun | |
| with: | |
| token: ${{secrets.GH_TOKEN}} | |
| build-only: "yes" | |
| # Build the k8s release artifact. Static KRUN is built separately and | |
| # in parallel from the rest of KM. The k8s artifact contains: | |
| # km - km executable | |
| # containerd-shim-krun-v2 - containerd shim for krun | |
| # krun - krun executable | |
| k8s-release-bin: | |
| name: make k8s release bundle | |
| runs-on: ubuntu-20.04 | |
| needs: [km-build, krun-static-build] | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| # get build directory created by km-build job | |
| - name: Download km-build artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: km-build | |
| path: . | |
| # download artifact and confugure krun | |
| - uses: ./.github/actions/configure-krun | |
| with: | |
| target-dir: build/opt/kontain | |
| token: ${{secrets.GH_TOKEN}} | |
| build-only: "yes" | |
| - name: Login into Azure Repository | |
| run: make -C cloud/azure login-cli | |
| - name: Prepare KM build env | |
| run: make -C tests pull-buildenv-image .buildenv-local-lib | |
| - name: Create kkm release artifact | |
| run: make withdocker TARGET=kkm-pkg | |
| # Note: need to have kkm.run ready for this step | |
| # make withdocker TARGET=kkm-pkg | |
| # because dependencies force it to re-build everything. We want to use pre-built km, krun (static), kkm and so on | |
| # | |
| - name: Create kontain bundles | |
| run: | | |
| BLDTOP=$(realpath build) | |
| chmod 755 ${BLDTOP}/opt/kontain/bin/km | |
| chmod 755 ${BLDTOP}/opt/kontain/bin/krun | |
| chmod 755 ${BLDTOP}/opt/kontain/bin/krun-label-trigger | |
| chmod 755 ${BLDTOP}/cloud/k8s/shim/containerd-shim-krun-v2 | |
| make k8s-bundle | |
| - name: Upload km-k8s-rel-artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: km-k8s-rel-artifact | |
| path: build/kontain_bin.tar.gz | |
| km-test-azure: | |
| name: KM tests, KVM on Azure | |
| runs-on: ${{ fromJSON(needs.start-runners.outputs.run-ons)['azure'] }} | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| needs: [start-runners, km-build] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Download km-build artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: km-build | |
| path: . | |
| - uses: ./.github/actions/run-tests | |
| with: | |
| kernel: kvm | |
| - name: Kernel logs | |
| if: ${{ always() }} | |
| run: sudo dmesg --ctime | |
| km-payloads-azure: | |
| name: Payloads on Azure | |
| runs-on: ${{ fromJSON(needs.start-runners.outputs.run-ons)['azure'] }} | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| needs: [start-runners, krun-static-build, km-build-payloads, km-test-azure] | |
| steps: | |
| # no need to checkout - just continue where tests left | |
| - name: Pull testenv images | |
| run: | | |
| make -C payloads -j pull-testenv-image | |
| - name: KM with KVM Payloads Test - Azure | |
| if: ${{ ! (github.event_name == 'schedule' || | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.run_type == 'nightly')) }} | |
| run: make -C payloads test-withdocker DOCKER_INTERACTIVE= | |
| timeout-minutes: 20 | |
| - name: KM with KVM Payloads Test All (on schedule) - Azure | |
| if: ${{ github.event_name == 'schedule' || | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.run_type == 'nightly') }} | |
| run: make -C payloads test-all-withdocker DOCKER_INTERACTIVE= | |
| timeout-minutes: 20 | |
| # download artifact and confugure krun | |
| - uses: ./.github/actions/configure-krun | |
| with: | |
| target-dir: /opt/kontain | |
| token: ${{secrets.GH_TOKEN}} | |
| - name: KM with KVM Validate runenv images - Azure | |
| run: | | |
| make -C payloads/ -j pull-demo-runenv-image | |
| make -C payloads validate-runenv-image DOCKER_INTERACTIVE= | |
| timeout-minutes: 20 | |
| - name: Kernel logs | |
| if: ${{ failure() }} | |
| run: sudo dmesg --ctime | |
| # TODO: Maybe not needed - review | |
| kkm-test-ubuntu: | |
| name: KM tests, KKM on CI VM | |
| runs-on: ubuntu-20.04 | |
| needs: [km-build] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: ./.github/actions/run-tests | |
| with: | |
| kernel: kkm | |
| - name: Kernel logs | |
| if: ${{ always() }} | |
| run: sudo dmesg --ctime | |
| kkm-test-aws: | |
| name: KM tests, KKM on AWS | |
| runs-on: ${{ fromJSON(needs.start-runners.outputs.run-ons)['ec2'] }} | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| needs: [start-runners, km-build] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Download km-build artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: km-build | |
| path: . | |
| - uses: ./.github/actions/run-tests | |
| with: | |
| kernel: kkm | |
| - name: Kernel logs | |
| if: ${{ always() }} | |
| run: sudo dmesg --ctime | |
| kkm-payloads-aws: | |
| name: Payloads on AWS | |
| runs-on: ${{ fromJSON(needs.start-runners.outputs.run-ons)['ec2'] }} | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| needs: [start-runners, krun-static-build, km-build-payloads, kkm-test-aws] | |
| steps: | |
| # no need to checkout - just continue where stopped after tests | |
| - name: Pull testenv image | |
| run: | | |
| make -C payloads -j pull-testenv-image | |
| - name: KM with KKM Payloads Test - AWS | |
| if: ${{ ! (github.event_name == 'schedule' || | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.run_type == 'nightly')) }} | |
| run: make -C payloads test-withdocker HYPERVISOR_DEVICE=/dev/kkm DOCKER_INTERACTIVE= | |
| timeout-minutes: 20 | |
| - name: KM with KKM Payloads Test - AWS (nightly) | |
| if: ${{ github.event_name == 'schedule' || | |
| (github.event_name == 'workflow_dispatch' && github.event.inputs.run_type == 'nightly') }} | |
| run: make -C payloads test-all-withdocker HYPERVISOR_DEVICE=/dev/kkm DOCKER_INTERACTIVE= | |
| timeout-minutes: 20 | |
| # download artifact and configure krun | |
| - uses: ./.github/actions/configure-krun | |
| with: | |
| target-dir: /opt/kontain | |
| token: ${{secrets.GH_TOKEN}} | |
| - name: KM with KKM Validate runenv images - AWS | |
| run: | | |
| make -C payloads/ -j pull-demo-runenv-image | |
| make -C payloads validate-runenv-image HYPERVISOR_DEVICE=/dev/kkm DOCKER_INTERACTIVE= | |
| timeout-minutes: 20 | |
| - name: Kernel logs | |
| if: ${{ failure() }} | |
| run: sudo dmesg --ctime | |
| minikube-testing: | |
| name: kubernetes tests against minikube | |
| runs-on: ubuntu-20.04 | |
| needs: [k8s-release-bin] | |
| strategy: | |
| matrix: | |
| # We want to test against containerd and crio. crio support is a bit brittle in | |
| # minikube. In our experience (late 2021) it works better with the podman minikube | |
| # driver than the docker minikube driver. | |
| runtime: ["containerd"] # "cri-o" later | |
| driver: ["docker", "podman"] | |
| # include: | |
| # - runtime: cri-o | |
| # driver: podman | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Download km-k8s-rel-artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: km-k8s-rel-artifact | |
| path: /tmp/bin_release | |
| - name: Start local file server | |
| run: | | |
| (cd /tmp/bin_release; python3 -m http.server 8000) & | |
| - name: Start minikube | |
| run: | | |
| minikube version | |
| minikube start --container-runtime=${{ matrix.runtime }} --driver=${{ matrix.driver }} --wait=all || minikube logs | |
| - name: Install Kontain on minikube k8s | |
| run: | | |
| # merges and applies all the files for CI overlay | |
| curl -o kontain-kustomize.sh -sL https://raw.githubusercontent.com/kontainapp/k8s-deploy/current/kontain-kustomize.sh | |
| chmod +x kontain-kustomize.sh | |
| ./kontain-kustomize.sh --km-url=http://host.minikube.internal:8000/kontain_bin.tar.gz | |
| - name: Check k8s install | |
| run: | | |
| echo "Pods " | |
| kubectl get pod -A | |
| kubectl logs -n kube-system -l app=kontain-init | |
| - name: Run user test | |
| run: | | |
| # start the test pod, which will just sleep | |
| kubectl apply -f https://raw.githubusercontent.com/kontainapp/k8s-deploy/current/tests/test.yaml | |
| sleep 10 | |
| # exec `uname -r` on the test pod. Should get back '.kontain.KKM' appended to the linux release name | |
| pname=$(kubectl get pod -l kontain=test-app -o jsonpath="{.items[0].metadata.name}") | |
| rname=$(kubectl exec "${pname}" -- uname -r) | |
| echo "uname -r returned ${rname}" | |
| echo "${rname}" | egrep -q -e '.*\.kontain\.(KKM|KVM)$' || exit 1 | |
| echo "Kontain Runtime OK". | |
| ecs-hack-test: | |
| name: test alternate KM triggers | |
| runs-on: ubuntu-20.04 | |
| needs: [k8s-release-bin] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: ./.github/actions/configure-kkm | |
| - name: Download km-k8s-rel-artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: km-k8s-rel-artifact | |
| path: /tmp/bin_release | |
| - name: Stop Docker | |
| run: sudo systemctl stop docker | |
| - name: Untar release | |
| run: | | |
| sudo mkdir -p /opt/kontain | |
| sudo tar -C /opt/kontain -xf /tmp/bin_release/kontain_bin.tar.gz | |
| - name: Install KM and Reconfigure Docker | |
| run: | | |
| sudo mv /opt/kontain/shim/containerd-shim-krun-v2 /usr/bin/containerd-shim-krun-v2 | |
| cat << EOF | sudo tee /etc/docker/daemon.json | |
| { | |
| "default-runtime": "krun", | |
| "runtimes": { | |
| "krun": { | |
| "path": "/opt/kontain/bin/krun-label-trigger" | |
| } | |
| } | |
| } | |
| EOF | |
| - name: Restart Docker | |
| run: | | |
| sudo systemctl daemon-reload | |
| sudo systemctl restart docker | |
| - name: Check krun personality | |
| run: | | |
| OUT=$(docker run --rm -v /.kontain busybox uname -r) | |
| echo ${OUT} | grep -q '.*\.kontain.KKM$' | |
| - name: Check crun personality | |
| run: | | |
| OUT=$(docker run --rm busybox uname -r) | |
| echo ${OUT} | grep -v -q '.*\.kontain.KKM$' | |
| codeql-scan: | |
| # CodeQL Code Scanning/Static Analysis | |
| name: Analyze | |
| runs-on: ubuntu-latest | |
| if: (github.triggering_actor != 'dependabot[bot]') | |
| permissions: | |
| security-events: write | |
| container: | |
| image: kontainkubecr.azurecr.io/buildenv-km-fedora:latest | |
| credentials: | |
| username: ${{ secrets.SP_APPID }} | |
| password: ${{ secrets.SP_PASSWORD }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| language: ["cpp"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # Initializes the CodeQL tools for scanning. | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v2 | |
| with: | |
| languages: ${{ matrix.language }} | |
| # enhanced security checks. | |
| queries: security-extended | |
| - name: Build KM | |
| run: | | |
| make -C km all | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v2 | |
| slack-workflow-status: | |
| name: Notify slack, if needed | |
| runs-on: ubuntu-latest | |
| # 'contains' shows how to add conditions, e.g. on workflow 'name:', or many other contexts. | |
| # see https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions | |
| if: | |
| (failure() && github.ref == 'refs/heads/master' && github.triggering_actor != 'dependabot[bot]') || | |
| contains(github.workflow, 'noisy') | |
| # Dependencies. (A skipped dependency is considered satisfied) | |
| needs: | |
| [ | |
| km-build, | |
| km-test-azure, | |
| km-payloads-azure, | |
| kkm-test-ubuntu, | |
| kkm-test-aws, | |
| kkm-payloads-aws, | |
| minikube-testing, | |
| codeql-scan, | |
| ecs-hack-test, | |
| ] | |
| steps: | |
| - name: Send notification to slack | |
| uses: Gamesight/slack-workflow-status@master | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| include_jobs: true | |
| channel: "#build_and_test" | |
| name: "CI workflow status" | |
| icon_emoji: ":thumbsdown:" | |
| # This step is to clean up on-demand runner. They work in conjunction with start-runner. | |
| # Make sure to add dependencies in both "needs" clauses | |
| stop-runner: | |
| if: (always() && github.actor != 'dependabot[bot]') | |
| # Dependencies. (A skipped dependency is considered satisfied) | |
| needs: | |
| [ | |
| start-runners, | |
| km-build, | |
| km-test-azure, | |
| kkm-test-aws, | |
| km-payloads-azure, | |
| kkm-payloads-aws, | |
| ] | |
| uses: ./.github/workflows/stop-cloud-runners.yaml | |
| with: | |
| dependencies: ${{ toJSON(needs) }} | |
| secrets: inherit |