Adding Github Actions to build Pinot/Presto/Superset docker images (a…

…pache#8458)

.github/workflows/build-pinot-docker-image.yml

@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Docker Image Build and Publish
+
+on:
+  schedule:
+  - cron: '0 1 * * *' # run at 1 AM UTC
+
+  workflow_dispatch:
+    inputs:
+      gitUrl:
+        description: "The Pinot git repo to check out to build, use https."
+        default: "https://github.com/apache/pinot.git"
+        required: true
+      commit:
+        description: "The branch/commit to check out to build Pinot image."
+        default: "master"
+        required: true
+      dockerImageName:
+        description: "The docker image name, default to 'apachepinot/pinot'."
+        default: "apachepinot/pinot"
+        required: true
+      tags:
+        description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+        default: ""
+
+jobs:
+  build-pinot-docker-image:
+    name: Build Pinot Docker Image
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [ "linux/amd64", "linux/arm64" ]
+    steps:
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - uses: docker/setup-qemu-action@v1
+      name: Set up QEMU
+    - uses: docker/setup-buildx-action@v1
+      name: Set up Docker Buildx
+    - uses: actions/checkout@v3
+    - name: Build and push the Docker image
+      env:
+        DOCKER_FILE_BASE_DIR: "docker/images/pinot"
+        DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+        BUILD_PLATFORM: ${{ matrix.platform }}
+        PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+        PINOT_BRANCH: ${{ github.event.inputs.commit }}
+        TAGS: ${{ github.event.inputs.tags }}
+      run: .github/workflows/scripts/docker/.pinot_docker_image_build_and_push.sh

.github/workflows/build-presto-docker-image.yml

@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Presto Pinot Docker Image Build and Publish
+
+on:
+  schedule:
+  - cron: '0 0 * * *' # run at 0 AM UTC
+
+  workflow_dispatch:
+    inputs:
+      gitUrl:
+        description: "The git url to check out to build Presto, use https not ssh."
+        default: "https://github.com/prestodb/presto.git"
+        required: true
+      commit:
+        description: "The branch/commit to check out to build Presto image.."
+        default: "master"
+        required: true
+      platform:
+        description: "The platform of the image to build, e.g. linux/amd64"
+        default: "linux/amd64"
+        required: true
+      dockerImageName:
+        description: "The docker image name, default is 'apachepinot/pinot-presto'"
+        default: "apachepinot/pinot-presto"
+        required: true
+      tags:
+        description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+        default: ""
+
+jobs:
+  build-presto-pinot-docker-image:
+    name: Build Presto Pinot Docker Image
+    runs-on: ubuntu-latest
+    steps:
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - uses: docker/setup-qemu-action@v1
+      name: Set up QEMU
+    - uses: docker/setup-buildx-action@v1
+      name: Set up Docker Buildx
+    - uses: actions/checkout@v3
+    - name: Build and push the Docker image
+      env:
+        DOCKER_FILE_BASE_DIR: "docker/images/pinot-presto"
+        DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+        BUILD_PLATFORM: ${{ github.event.inputs.platform }}
+        PRESTO_GIT_URL: ${{ github.event.inputs.gitUrl }}
+        PRESTO_BRANCH: ${{ github.event.inputs.commit }}
+        TAGS: ${{ github.event.inputs.tags }}
+      run: .github/workflows/scripts/docker/.presto_docker_image_build_and_push.sh

.github/workflows/build-superset-docker-image.yml

@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Superset Pinot Docker Image Build and Publish
+
+on:
+  schedule:
+  - cron: '0 0 * * *' # run at 0 AM UTC
+
+  workflow_dispatch:
+    inputs:
+      supersetImageTag:
+        description: "Base superset docker image tag from https://hub.docker.com/r/apache/superset"
+        default: "latest"
+        required: true
+      platform:
+        description: "The platform of the image to build, e.g. linux/amd64"
+        default: "linux/amd64"
+        required: true
+      dockerImageName:
+        description: "The docker image name, default is 'apachepinot/pinot-superset'"
+        default: "apachepinot/pinot-superset"
+        required: true
+      tags:
+        description: "Tags to push of the image, comma separated, e.g. tag1,tag2,tag3"
+        default: ""
+
+jobs:
+  build-superset-pinot-docker-image:
+    name: Build Superset Pinot Docker Image
+    runs-on: ubuntu-latest
+    steps:
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - uses: docker/setup-qemu-action@v1
+      name: Set up QEMU
+    - uses: docker/setup-buildx-action@v1
+      name: Set up Docker Buildx
+    - uses: actions/checkout@v3
+    - name: Build and push the Docker image
+      env:
+        DOCKER_FILE_BASE_DIR: "docker/images/pinot-superset"
+        DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+        SUPERSET_IMAGE_TAG: ${{ github.event.inputs.supersetImageTag }}
+        BUILD_PLATFORM: ${{ github.event.inputs.platform }}
+        TAGS: ${{ github.event.inputs.tags }}
+      run: .github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh

.github/workflows/scripts/docker/.pinot_docker_image_build_and_push.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+  DOCKER_IMAGE_NAME="apachepinot/pinot"
+fi
+if [ -z "${PINOT_GIT_URL}" ]; then
+  PINOT_GIT_URL="https://github.com/apache/pinot.git"
+fi
+if [ -z "${PINOT_BRANCH}" ]; then
+  PINOT_BRANCH="master"
+fi
+if [ -z "${BUILD_PLATFORM}" ]; then
+  BUILD_PLATFORM="linux/arm64,linux/amd64"
+fi
+
+COMMIT_ID=`git rev-parse --short HEAD`
+DATE=`date +%Y%m%d`
+VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
+tags=()
+if [ -z "${TAGS}" ]; then
+  tags=("${VERSION}-${COMMIT_ID}-${DATE}")
+  tags+=("latest")
+else
+  declare -a tags=($(echo ${TAGS} | tr "," " "))
+fi
+
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"
+do
+  echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+  DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
+done
+
+cd ${DOCKER_FILE_BASE_DIR}
+
+docker buildx build \
+    --no-cache \
+    --platform=${BUILD_PLATFORM} \
+    --file Dockerfile \
+    --build-arg PINOT_GIT_URL=${PINOT_GIT_URL} --build-arg PINOT_BRANCH=${PINOT_BRANCH} \
+    ${DOCKER_BUILD_TAGS} \
+    --push \
+    .

.github/workflows/scripts/docker/.presto_docker_image_build_and_push.sh

@@ -0,0 +1,70 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+  DOCKER_IMAGE_NAME="apachepinot/pinot-presto"
+fi
+if [ -z "${PRESTO_GIT_URL}" ]; then
+  PRESTO_GIT_URL="https://github.com/prestodb/presto.git"
+fi
+if [ -z "${PRESTO_BRANCH}" ]; then
+  PINOT_BRANCH="master"
+fi
+if [ -z "${BUILD_PLATFORM}" ]; then
+  BUILD_PLATFORM="linux/amd64"
+fi
+
+# Get presto commit id
+ROOT_DIR=`pwd`
+rm -rf /tmp/presto
+git clone -b ${PRESTO_BRANCH} --single-branch ${PRESTO_GIT_URL} /tmp/presto
+cd /tmp/presto
+COMMIT_ID=`git rev-parse --short HEAD`
+VERSION=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout`
+rm -rf /tmp/presto
+DATE=`date +%Y%m%d`
+cd ${ROOT_DIR}
+
+tags=()
+if [ -z "${TAGS}" ]; then
+  tags=("${VERSION}-${COMMIT_ID}-${DATE}")
+  tags+=("latest")
+else
+  declare -a tags=($(echo ${TAGS} | tr "," " "))
+fi
+
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"
+do
+  echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+  DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
+done
+
+
+cd ${DOCKER_FILE_BASE_DIR}
+
+docker buildx build \
+    --no-cache \
+    --platform=${BUILD_PLATFORM} \
+    --file Dockerfile \
+    --build-arg PRESTO_GIT_URL=${PRESTO_GIT_URL} --build-arg PRESTO_BRANCH=${PRESTO_BRANCH} \
+    ${DOCKER_BUILD_TAGS} \
+    --push \
+    .

.github/workflows/scripts/docker/.superset_docker_image_build_and_push.sh

@@ -0,0 +1,59 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+  DOCKER_IMAGE_NAME="apachepinot/pinot-superset"
+fi
+if [ -z "${SUPERSET_IMAGE_TAG}" ]; then
+  SUPERSET_IMAGE_TAG="latest"
+fi
+if [ -z "${BUILD_PLATFORM}" ]; then
+  BUILD_PLATFORM="linux/amd64"
+fi
+
+DATE=`date +%Y%m%d`
+docker pull apache/superset:${SUPERSET_IMAGE_TAG}
+COMMIT_ID=`docker images apache/superset:${SUPERSET_IMAGE_TAG} --format "{{.ID}}"`
+
+tags=()
+if [ -z "${TAGS}" ]; then
+  tags=("${COMMIT_ID}-${DATE}")
+  tags+=("latest")
+else
+  declare -a tags=($(echo ${TAGS} | tr "," " "))
+fi
+
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"
+do
+  echo "Plan to build and push docker images for: ${DOCKER_IMAGE_NAME}:${tag}"
+  DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:${tag} "
+done
+
+cd ${DOCKER_FILE_BASE_DIR}
+
+docker buildx build \
+    --no-cache \
+    --platform=${BUILD_PLATFORM} \
+    --file Dockerfile \
+    --build-arg SUPERSET_IMAGE_TAG=${SUPERSET_IMAGE_TAG} \
+    ${DOCKER_BUILD_TAGS} \
+    --push \
+    .