Skip to content

Commit

Permalink
Merge branch 'master' into javascript
Browse files Browse the repository at this point in the history
  • Loading branch information
robertwb committed Apr 10, 2022
2 parents d2ef2f6 + 5603bd3 commit 978fe63
Show file tree
Hide file tree
Showing 951 changed files with 46,745 additions and 14,387 deletions.
18 changes: 13 additions & 5 deletions .github/REVIEWERS.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# If you would like to be excluded from consideration for reviewing a certain label,
# add yourself to that label's exclusionList
# FallbackReviewers is for reviewers who can review any area of the code base that might
# not receive a label. These should generally be more experienced committers.
labels:
- name: "Go"
reviewers: ["damccorm", "lostluck", "jrmccluskey", "youngoli", "riteshghorse"]
exclusionList: [] # These users will never be suggested as reviewers
# I don't know the other areas well enough to assess who the normal committers/contributors who might want to be reviewers are
fallbackReviewers: [] # List of committers to use when no label matches
- name: Go
reviewers:
- damccorm
- lostluck
- jrmccluskey
- youngoli
- riteshghorse
exclusionList: []
fallbackReviewers: []
52 changes: 5 additions & 47 deletions .github/workflows/build_playground_backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ name: Build And Deploy Playground Backend Application
on:
push:
tags: 'v*'
branches: ['master', 'release-*']
pull_request:
paths: ['playground/backend/**']
branches: ['playground-staging']
workflow_dispatch:
jobs:
build_playground_backend_docker_image:
Expand Down Expand Up @@ -65,57 +67,13 @@ jobs:
if: startsWith(github.ref, 'refs/tags/')
- name: Setup GCP account
run: |
echo "${{ secrets.PLAYGROUND_GCP_ACCESS_KEY }}" | base64 -d > /tmp/gcp_access.json
echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json
if: startsWith(github.ref, 'refs/tags/')
- name: Login to Docker Registry
run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }}
if: startsWith(github.ref, 'refs/tags/')
- name: Prepare Build Java
run: |
export
./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag=${{ env.DOCKERTAG }}
if: startsWith(github.ref, 'refs/tags/')
- name: Prepare Build Python
run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
if: startsWith(github.ref, 'refs/tags/')
- name: Prepare Build Go
run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
if: startsWith(github.ref, 'refs/tags/')
- name: Prepare Build SCIO
run: ./gradlew playground:backend:containers:scio:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
if: startsWith(github.ref, 'refs/tags/')
- name: Prepare Build Router
run: ./gradlew playground:backend:containers:router:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
if: startsWith(github.ref, 'refs/tags/')
- name: Clean All Build directories
run: ./gradlew clean
- name: Deploy Backend Application Java
env:
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=backend-java${{env.STAND_SUFFIX}}"
working-directory: playground/terraform/applications/backend-java
if: startsWith(github.ref, 'refs/tags/')
- name: Deploy Backend Application Go
env:
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=backend-go${{env.STAND_SUFFIX}}"
working-directory: playground/terraform/applications/backend-go
if: startsWith(github.ref, 'refs/tags/')
- name: Deploy Backend Application Python
env:
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=backend-python${{env.STAND_SUFFIX}}"
working-directory: playground/terraform/applications/backend-python
if: startsWith(github.ref, 'refs/tags/')
- name: Deploy Backend Application SCIO
env:
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=backend-scio${{env.STAND_SUFFIX}}"
working-directory: playground/terraform/applications/backend-scio
if: startsWith(github.ref, 'refs/tags/')
- name: Deploy Backend Application Router
- name: Deploy Backend Applications
env:
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=backend-router${{env.STAND_SUFFIX}}"
working-directory: playground/terraform/applications/backend-router
run: ./gradlew playground:terraform:deployBackend -Pdocker-tag=${{env.DOCKERTAG}} -Pproject_id=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} -Pproject_environment='beta' -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository'
if: startsWith(github.ref, 'refs/tags/')
17 changes: 5 additions & 12 deletions .github/workflows/build_playground_frontend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ name: Build And Deploy Playground Frontend Application
on:
push:
tags: 'v*'
branches: ['master', 'release-*']
pull_request:
paths: ['playground/frontend/**']
paths: ['playground/backend/**']
branches: ['playground-staging']
workflow_dispatch:

jobs:
Expand Down Expand Up @@ -52,8 +54,6 @@ jobs:
- name: Remove default github maven configuration
# This step is a workaround to avoid a decryption issue
run: rm ~/.m2/settings.xml
- name: Run PreCommit
run: ./gradlew playground:frontend:precommit
- name: install npm
uses: actions/setup-node@v2
with:
Expand All @@ -74,20 +74,13 @@ jobs:
if: startsWith(github.ref, 'refs/tags/')
- name: Setup GCP account
run: |
echo "${{ secrets.PLAYGROUND_GCP_ACCESS_KEY }}" | base64 -d > /tmp/gcp_access.json
echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json
if: startsWith(github.ref, 'ref/tags/')
- name: Login to Docker Registry
run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }}
if: startsWith(github.ref, 'ref/tags/')
- name: Create config
run: ./gradlew playground:frontend:createConfig -PplaygroundBackendJavaRouteUrl="https://backend-java${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendGoRouteUrl="https://backend-go${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendPythonRouteUrl="https://backend-python${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendUrl="https://backend-router${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendScioRouteUrl="https://backend-scio${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendUrl="https://backend-router${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/"
if: startsWith(github.ref, 'refs/tags/')
- name: Prepare Build Frontend
run: ./gradlew playground:frontend:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{env.DOCKERTAG}}
if: startsWith(github.ref, 'refs/tags/')
- name: Deploy Frontend Application
env:
GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=frontend${{env.STAND_SUFFIX}}"
working-directory: playground/terraform/applications/frontend
run: ./gradlew playground:terraform:deployFrontend -Pdocker-tag=${{env.DOCKERTAG}} -Pproject_id=${{ secrets.GCP_PLAYGROUND_PROJECT_ID }} -Pproject_environment='beta'
if: startsWith(github.ref, 'refs/tags/')
8 changes: 4 additions & 4 deletions .github/workflows/build_wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ jobs:
if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
steps:
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@master
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
Expand All @@ -191,7 +191,7 @@ jobs:
name: source_zip
path: source/
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@master
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
Expand Down Expand Up @@ -298,7 +298,7 @@ jobs:
name: wheelhouse-${{ matrix.os }}
path: wheelhouse/
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@master
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
Expand Down Expand Up @@ -334,7 +334,7 @@ jobs:
if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
steps:
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@master
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/java_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ jobs:
with:
go-version: '1.16.12'
- name: Authenticate on GCP
uses: google-github-actions/setup-gcloud@master
uses: google-github-actions/setup-gcloud@v0
with:
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
service_account_key: ${{ secrets.GCP_SA_KEY }}
Expand Down
60 changes: 46 additions & 14 deletions .github/workflows/playground_deploy_examples.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,51 @@ name: Collect And Deploy Playground Examples
on:
push:
tags: 'v*'
branch: 'beta-version'
branches: ['master', 'release-*']
pull_request:
workflow_dispatch:

env:
BEAM_ROOT_DIR: ../../
BEAM_EXAMPLE_CATEGORIES: ../categories.yaml
BEAM_VERSION: 2.33.0
K8S_NAMESPACE: playground-backend
HELM_APP_NAME: playground-backend
jobs:
check_examples:
name: Check examples
runs-on: ubuntu-latest
outputs:
example_has_changed: ${{ steps.check_has_example.outputs.example_has_changed }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: install deps
run: pip install -r requirements.txt
working-directory: playground/infrastructure
- shell: pwsh
name: get Difference
id: check_file_changed
run: |
$diff = git diff --name-only master
Write-Host "::set-output name=example_diff::$diff"
- name: has Examples
run: |
output=$(python3 checker.py ${{ steps.check_file_changed.outputs.example_diff }})
echo "::set-output name=example_has_changed::$output"
id: check_has_example
working-directory: playground/infrastructure
- name: Print has_example
run: echo "${{ steps.check_has_example.outputs.example_has_changed }}"
deploy_examples:
name: Deploy examples
runs-on: ubuntu-latest
env:
BEAM_ROOT_DIR: ../../
BEAM_EXAMPLE_CATEGORIES: ../categories.yaml
BEAM_VERSION: 2.33.0
K8S_NAMESPACE: playground-backend
HELM_APP_NAME: playground-backend
needs: [ check_examples ]
if: needs.check_examples.outputs.example_has_changed == 'True'
steps:
- name: Check out the repo
uses: actions/checkout@v2
Expand All @@ -52,7 +84,7 @@ jobs:
chmod 700 get_helm.sh &&\
./get_helm.sh
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@v0.3.0
uses: google-github-actions/setup-gcloud@v0
- name: install deps
run: pip install -r requirements.txt
working-directory: playground/infrastructure
Expand All @@ -61,7 +93,7 @@ jobs:
run: rm ~/.m2/settings.xml
- name: Setup GCP account
run: |
echo "${{ secrets.PLAYGROUND_GCP_ACCESS_KEY }}" | base64 -d > /tmp/gcp_access.json
echo "${{ secrets.GCP_PLAYGROUND_SA_KEY }}" | base64 -d > /tmp/gcp_access.json
which gcloud
gcloud auth activate-service-account --project=apache-beam-testing --key-file=/tmp/gcp_access.json
- name: Set Docker Tag
Expand All @@ -75,17 +107,17 @@ jobs:
run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }}
- name: Build And Push Java Backend
run: |
./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag=${{ env.DOCKERTAG }}
./gradlew playground:backend:containers:java:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' -Pbase-image='apache/beam_java8_sdk:${{ env.BEAM_VERSION }}' -Pdocker-tag=${{ env.DOCKERTAG }}
- name: Build And Push Go Backend
run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
- name: Build And Push Python Backend
run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
run: ./gradlew playground:backend:containers:python:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
- name: Clean All Build directories
run: ./gradlew clean
- name: Install helm chart
run: |
kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\
helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . --set global.registry="${{ secrets.PLAYGROUND_REGISTRY_NAME }}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" --set global.tag=${{ env.DOCKERTAG }} &&\
helm install --namespace $K8S_NAMESPACE $HELM_APP_NAME . --set global.registry="${{ secrets.PLAYGROUND_REGISTRY_NAME }}/${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}/playground-repository" --set global.tag=${{ env.DOCKERTAG }} &&\
sleep 120
working-directory: playground/infrastructure/helm
- name: Run Python Examples CI
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pr-bot-new-prs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ name: pr-bot-new-prs
# Run every 30 minutes
on:
schedule:
- cron: '30 * * * *'
- cron: '0,30 * * * *'
workflow_dispatch:

jobs:
Expand Down
13 changes: 11 additions & 2 deletions .github/workflows/pr-bot-pr-updates.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,29 @@
name: pr-bot-pr-updates

on:
pull_request_review:
pull_request_review_comment:
pull_request_target:
types: ["synchronize"] # Synchronize is the action that runs after pushes to the user branch
issue_comment:

jobs:
process-pr-update:
# Give GITHUB_TOKEN permissions to write pull request comments and to the state branch, and read PR related info
permissions:
contents: write
pull-requests: write
checks: read
issues: read
statuses: read

# Don't run on forks
if: github.repository == 'apache/beam'
runs-on: ubuntu-latest

steps:
# Pin to master so users can't do anything malicious on their own branch and run it here.
- uses: actions/checkout@v2
with:
ref: 'master'
- run: npm install
working-directory: 'scripts/ci/pr-bot'

Expand Down
Original file line number Diff line number Diff line change
@@ -1,25 +1,39 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

nohup /opt/mitmproxy/mitmdump -s /opt/mitmproxy/allow_list_proxy.py -p 8081 &
while [ ! -f /root/.mitmproxy/mitmproxy-ca.pem ] ;
do
sleep 2
done
openssl x509 -in /root/.mitmproxy/mitmproxy-ca.pem -inform PEM -out /root/.mitmproxy/mitmproxy-ca.crt
cp /root/.mitmproxy/mitmproxy-ca.crt /usr/local/share/ca-certificates/extra/
update-ca-certificates
cat /root/.mitmproxy/mitmproxy-ca.pem >> /usr/local/lib/python3.7/site-packages/certifi/cacert.pem
name: pr-bot-prs-needing-attention

# Run every day at 12:07 UTC
on:
schedule:
- cron: '7 12 * * *'
workflow_dispatch:

jobs:
process-prs:
# Don't run on forks
if: github.repository == 'apache/beam'
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
- run: npm install
working-directory: 'scripts/ci/pr-bot'

# Runs a set of commands using the runners shell
- run: npm run findPrsNeedingAttention
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
working-directory: 'scripts/ci/pr-bot'
Loading

0 comments on commit 978fe63

Please sign in to comment.