[DPE-4485] Split Kyuubi OCI image from Charmed Spark OCI image (#87)

canonical · Jun 14, 2024 · bbdeabb · bbdeabb
1 parent a24230f
commit bbdeabb
Show file tree

Hide file tree

Showing 33 changed files with 852 additions and 179 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -15,7 +15,7 @@ jobs:
       - name: YAML Lint
         run:  |
           yamllint -d "{extends: relaxed, rules: {line-length: {max: 250}}}" \
-            --no-warnings rockcraft.yaml
+            --no-warnings images/charmed-spark/rockcraft.yaml
 
   build:
     runs-on: ubuntu-latest
@@ -33,25 +33,35 @@ jobs:
         run: |
           sudo snap install yq
           sudo snap install rockcraft --classic --edge    
-          sudo snap install --devmode --channel edge skopeo
 
       - name: Build image
         run: sudo make build
 
       - name: Build image (Jupyter)
         run: sudo make build FLAVOUR=jupyter
 
+      - name: Build image (Kyuubi)
+        run: sudo make build FLAVOUR=kyuubi
+
       - name: Get Artifact Name
         id: artifact
         run: |
-          ARTIFACT=$(make help | grep 'Artifact: ')
-          echo "name=${ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          BASE_ARTIFACT=$(make help | grep 'Artifact: ')
+          echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          JUPYTER_ARTIFACT=$(make help FLAVOUR=jupyter | grep 'Artifact: ')
+          echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
+          echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Change artifact permissions
-        run: sudo chmod a+r ${{ steps.artifact.outputs.name }}
+        run: sudo chmod a+r  ${{ steps.artifact.outputs.base_artifact_name }} ${{ steps.artifact.outputs.jupyter_artifact_name }} ${{ steps.artifact.outputs.kyuubi_artifact_name }}
 
       - name: Upload locally built artifact
         uses: actions/upload-artifact@v4
         with:
           name: charmed-spark
-          path:  ${{ steps.artifact.outputs.name }}
+          path: |
+            ${{ steps.artifact.outputs.base_artifact_name }}
+            ${{ steps.artifact.outputs.jupyter_artifact_name }}
+            ${{ steps.artifact.outputs.kyuubi_artifact_name }}
+
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -27,12 +27,16 @@ jobs:
       - name: Get Artifact Name
         id: artifact
         run: |
-          ARTIFACT=$(make help | grep 'Artifact: ')
-          echo "name=${ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          BASE_ARTIFACT=$(make help | grep 'Artifact: ')
+          echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          JUPYTER_ARTIFACT=$(make help FLAVOUR=jupyter | grep 'Artifact: ')
+          echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
+          echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Install and configure microk8s
         run: |
-          make microk8s MICROK8S=${{ matrix.k8s_version }}
+          make microk8s-setup MICROK8S_CHANNEL=${{ matrix.k8s_version }}
 
       - name: Download artifact
         uses: actions/download-artifact@v4
@@ -43,26 +47,44 @@ jobs:
       - name: Run tests
         run: |
           # Unpack Artifact
-          mv charmed-spark/${{ steps.artifact.outputs.name }} .
-          rmdir charmed-spark
+          mv charmed-spark/${{ steps.artifact.outputs.base_artifact_name }} .
 
           # Import artifact into docker with new tag
-          sudo make import TARGET=docker REPOSITORY=ghcr.io/canonical/ PREFIX=test- \
-            -o ${{ steps.artifact.outputs.name }} 
+          sudo make docker-import REPOSITORY=ghcr.io/canonical/ PREFIX=test- \
+            -o ${{ steps.artifact.outputs.base_artifact_name }} 
 
           # Import artifact into microk8s to be used in integration tests
-          sudo make import TARGET=microk8s PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
-            -o $(find .make_cache -name "*.tag")
+          sudo make microk8s-import PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
+            -o ${{ steps.artifact.outputs.base_artifact_name }}
           
           sg microk8s -c "make tests"
 
       - name: Run tests (Jupyter)
         run: |
+          # Unpack Artifact
+          mv charmed-spark/${{ steps.artifact.outputs.jupyter_artifact_name }} .
+
           # Import artifact into docker with new tag
-          sudo make import \
-            FLAVOUR=jupyter TARGET=microk8s \
-            TAG=$(yq .version rockcraft.yaml) \
-            REPOSITORY=ghcr.io/canonical/ PREFIX=test- \
-            -o $(find .make_cache -name "*.tag")
+          sudo make microk8s-import \
+            FLAVOUR=jupyter \
+            TAG=$(yq .version images/charmed-spark/rockcraft.yaml) \
+            REPOSITORY=ghcr.io/canonical/ \
+            PREFIX=test- \
+            -o ${{ steps.artifact.outputs.jupyter_artifact_name }}
 
           sg microk8s -c "make tests FLAVOUR=jupyter"
+
+      - name: Run tests (Kyuubi)
+        run: |
+          # Unpack Artifact
+          mv charmed-spark/${{ steps.artifact.outputs.kyuubi_artifact_name }} .
+          rmdir charmed-spark
+
+          # Import artifact into docker with new tag
+          sudo make microk8s-import \
+            FLAVOUR=kyuubi \
+            TAG=$(yq .version images/charmed-spark/rockcraft.yaml) \
+            REPOSITORY=ghcr.io/canonical/ PREFIX=test- \
+            -o ${{ steps.artifact.outputs.kyuubi_artifact_name }}
+
+          sg microk8s -c "make tests FLAVOUR=kyuubi"
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -23,8 +23,8 @@ jobs:
     - name: Extract ROCK metadata
       shell: bash
       run: |
-        VERSION=$(yq '(.version|split("-"))[0]' rockcraft.yaml)
-        BASE=$(yq '(.base|split("@"))[1]' rockcraft.yaml)
+        VERSION=$(yq '(.version|split("-"))[0]' images/charmed-spark/rockcraft.yaml)
+        BASE=$(yq '(.base|split("@"))[1]' images/charmed-spark/rockcraft.yaml)
         echo "version=${VERSION}" >> $GITHUB_OUTPUT
         echo "base=${BASE}" >> $GITHUB_OUTPUT
       id: rock_metadata
@@ -67,8 +67,12 @@ jobs:
       - name: Get Artifact Name
         id: artifact
         run: |
-          ARTIFACT=$(make help | grep 'Artifact: ')
-          echo "name=${ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          BASE_ARTIFACT=$(make help | grep 'Artifact: ')
+          echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          JUPYTER_ARTIFACT=$(make help FLAVOUR=jupyter | grep 'Artifact: ')
+          echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
+          echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Download artifact
         uses: actions/download-artifact@v4
@@ -79,8 +83,7 @@ jobs:
       - name: Publish Image to Channel
         run: |
           # Unpack artifact
-          mv charmed-spark/${{ steps.artifact.outputs.name }} .
-          rmdir charmed-spark
+          mv charmed-spark/${{ steps.artifact.outputs.base_artifact_name }} .
           
           REPOSITORY="ghcr.io/canonical/"
           RISK=${{ needs.release_checks.outputs.risk }}
@@ -90,12 +93,12 @@ jobs:
           IMAGE_NAME=$(make help REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2  | xargs)
                     
           # Import artifact into docker with new tag
-          sudo make import TARGET=docker REPOSITORY=${REPOSITORY} TAG=${TAG}\
+          sudo make docker-import REPOSITORY=${REPOSITORY} TAG=${TAG}\
             -o ${{ steps.artifact.outputs.name }} 
 
           # Add relevant labels
           COMMIT_ID=$(git log -1 --format=%H)
-          DESCRIPTION=$(yq .description rockcraft.yaml | xargs)
+          DESCRIPTION=$(yq .description images/charmed-spark/rockcraft.yaml | xargs)
           
           echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" -
 
@@ -113,20 +116,24 @@ jobs:
 
       - name: Publish JupyterLab Image to Channel
         run: |
-      
+          # Unpack artifact
+          mv charmed-spark/${{ steps.artifact.outputs.jupyter_artifact_name }} .
+
           REPOSITORY="ghcr.io/canonical/"
           RISK=${{ needs.release_checks.outputs.risk }}
           TRACK=${{ needs.release_checks.outputs.track }}
           if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi
       
           # Import artifact into docker with new tag
-          sudo make import TARGET=docker FLAVOUR=jupyter \
-            REPOSITORY=${REPOSITORY} TAG=${TAG}\
-            -o $(find .make_cache -name "*.tag")
+          sudo make docker-import \
+            FLAVOUR=jupyter \
+            REPOSITORY=${REPOSITORY} \
+            TAG=${TAG} \
+            -o ${{ steps.artifact.outputs.jupyter_artifact_name }}
 
           IMAGE_NAME=$(make help FLAVOUR=jupyter REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2  | xargs)
 
-          DESCRIPTION=$(sed -n '/^#\ \-/,/^#\ \-/  p' build/Dockerfile.jupyter | sed 's/^\#\ //g' | sed '$ d' | tail -n+2 | xargs)
+          DESCRIPTION=$(yq .flavours.jupyter.image_description images/metadata.yaml | xargs)
 
           echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" -
 
@@ -142,3 +149,41 @@ jobs:
             echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"
             docker push ${IMAGE_NAME}:${VERSION_TAG}
           fi
+
+
+      - name: Publish Kyuubi Image to Channel
+        run: |
+          # Unpack artifact
+          mv charmed-spark/${{ steps.artifact.outputs.kyuubi_artifact_name }} .
+          rmdir charmed-spark
+
+          REPOSITORY="ghcr.io/canonical/"
+          RISK=${{ needs.release_checks.outputs.risk }}
+          TRACK=${{ needs.release_checks.outputs.track }}
+          if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi
+      
+          # Import artifact into docker with new tag
+          sudo make docker-import \
+            FLAVOUR=kyuubi \
+            REPOSITORY=${REPOSITORY} \
+            TAG=${TAG} \
+            -o ${{ steps.artifact.outputs.kyuubi_artifact_name }}
+
+          IMAGE_NAME=$(make help FLAVOUR=kyuubi REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2  | xargs)
+
+          DESCRIPTION=$(yq .flavours.kyuubi.image_description images/metadata.yaml | xargs)
+
+          echo "FROM ${IMAGE_NAME}:${TAG}" | docker build --label org.opencontainers.image.description="${DESCRIPTION}" --label org.opencontainers.image.revision="${COMMIT_ID}" --label org.opencontainers.image.source="${{ github.repositoryUrl }}" -t "${IMAGE_NAME}:${TAG}" -
+
+          echo "Publishing ${IMAGE_NAME}:${TAG}"
+          docker push ${IMAGE_NAME}:${TAG}
+
+          if [[ "$RISK" == "edge" ]]; then
+            VERSION_LONG=$(make help FLAVOUR=kyuubi | grep "Tag\:" | cut -d ":" -f2  | xargs)
+            VERSION_TAG="${VERSION_LONG}-${{ needs.release_checks.outputs.base }}_edge"
+
+            docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG}
+
+            echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}"
+            docker push ${IMAGE_NAME}:${VERSION_TAG}
+          fi
diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
@@ -14,14 +14,14 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - name: Install skopeo
+      - name: Install rockcraft (for skopeo)
         run: |
-          sudo snap install --devmode --channel edge skopeo
+          sudo snap install rockcraft --classic --edge
       - name: Get Artifact Name
         id: artifact
         run: |
-          ARTIFACT=$(make help | grep 'Artifact: ')
-          echo "name=${ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
+          BASE_ARTIFACT=$(make help | grep 'Artifact: ')
+          echo "base_artifact_name=${BASE_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
       - name: Download artifact
         uses: actions/download-artifact@v4
         with:
@@ -30,10 +30,9 @@ jobs:
       - name: Import locally
         run: |
           # Unpack artifact
-          mv charmed-spark/${{ steps.artifact.outputs.name }} .
-          rmdir charmed-spark
-          sudo skopeo --insecure-policy copy \
-            docker-archive:${{ steps.artifact.outputs.name }} \
+          mv charmed-spark/${{ steps.artifact.outputs.base_artifact_name }} .
+          sudo rockcraft.skopeo --insecure-policy copy \
+            docker-archive:${{ steps.artifact.outputs.base_artifact_name }} \
             docker-daemon:trivy/charmed-spark:test
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master

diff --git a/.gitignore b/.gitignore
@@ -5,4 +5,5 @@ env/
 .make_cache/
 derby.log
 metastore_db/
-spark-sql.out
+spark-sql.out
+.vscode
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -10,11 +10,10 @@ cd charmed-spark-rock
 
 ### Installing Prerequisites
 ```bash
-sudo snap install rockcraft --edge
+sudo snap install rockcraft --edge --classic
 sudo snap install docker
 sudo snap install lxd
 sudo snap install yq
-sudo snap install skopeo --edge --devmode
 ```
 
 ### Configuring Prerequisites