Skip to content

Commit

Permalink
ci: conditionally skip python and frontend tests (apache#12583)
Browse files Browse the repository at this point in the history
* ci: conditionally skip python tests

* parametrize check script and implement for frontend

* add missing script

* fix incorrect check

* move CHECKS to ARGS and fix typo

* fix SC2086 nit
  • Loading branch information
villebro authored Feb 5, 2021
1 parent 3235d91 commit 5d55c43
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 0 deletions.
11 changes: 11 additions & 0 deletions .github/workflows/superset-frontend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if frontend changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh frontend
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: lint
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
npm run lint
npm run prettier-check
- name: unit tests
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
npm run test -- --coverage
- name: Upload code coverage
if: steps.check.outcome == 'failure'
working-directory: ./superset-frontend
run: |
bash <(curl -s https://codecov.io/bash) -cF javascript
9 changes: 9 additions & 0 deletions .github/workflows/superset-python-misc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,27 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
apt-get-install
pip-upgrade
pip install -r requirements/testing.txt
- name: pylint
if: steps.check.outcome == 'failure'
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset

Expand Down
27 changes: 27 additions & 0 deletions .github/workflows/superset-python-presto-hive.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
Expand All @@ -64,11 +73,14 @@ jobs:
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
Expand Down Expand Up @@ -106,17 +118,29 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Create csv upload directory
if: steps.check.outcome == 'failure'
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
if: steps.check.outcome == 'failure'
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
if: steps.check.outcome == 'failure'
run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
Expand All @@ -125,10 +149,13 @@ jobs:
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
36 changes: 36 additions & 0 deletions .github/workflows/superset-python-unittest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
Expand All @@ -52,11 +61,14 @@ jobs:
pip install -r requirements/testing.txt
setup-mysql
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (MySQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
Expand Down Expand Up @@ -91,11 +103,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
Expand All @@ -104,11 +125,14 @@ jobs:
pip install -r requirements/testing.txt
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
Expand All @@ -135,11 +159,20 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
- name: Setup Python
if: steps.check.outcome == 'failure'
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
uses: ./.github/actions/cached-dependencies
with:
run: |
Expand All @@ -148,10 +181,13 @@ jobs:
pip install -r requirements/testing.txt
mkdir ${{ github.workspace }}/.temp
- name: Run celery
if: steps.check.outcome == 'failure'
run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
- name: Python unit tests (SQLite)
if: steps.check.outcome == 'failure'
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
run: |
bash <(curl -s https://codecov.io/bash) -cF python
54 changes: 54 additions & 0 deletions scripts/ci_check_no_file_changes.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Usage instructions:
#
# to check for python changes, run with CHECKS=python
# To check for frontend changes, run with CHECKS=frontend
# To check for python and frontend changes, run with CHECKS="python frontend"

URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files"
FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename')

cat<<EOF
CHANGED FILES:
$FILES
EOF

for CHECK in "$@"
do
if [[ ${CHECK} == "python" ]]; then
REGEX="(^tests\/|^superset\/|^setup\.py|^requirements\/.+\.txt)"
echo "Searching for changes in python files"
elif [[ ${CHECK} == "frontend" ]]; then
REGEX="(^superset-frontend\/)"
echo "Searching for changes in frontend files"
else
echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE code"
exit 1
fi

if [[ "${FILES}" =~ ${REGEX} ]]; then
echo "Detected changes... Exiting with FAILURE code"
exit 1
fi
done
echo "No changes detected... Exiting with SUCCESS code"
exit 0

0 comments on commit 5d55c43

Please sign in to comment.