Skip to content

Commit

Permalink
[ci] Run shell scripts in CI (taichi-dev#3034)
Browse files Browse the repository at this point in the history 
* [ci] Run shell scripts in CI

* post submit file

* set env when running gpu machine

* unify m1 build
  • Loading branch information
@Leonz5288
Leonz5288 authored Oct 8, 2021
1 parent 43f8746 commit 8f8643d
Show file tree
Hide file tree
Showing 4 changed files with 276 additions and 88 deletions.
235 changes: 235 additions & 0 deletions .github/workflows/postsubmit.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
name: Postsubmit Checks
on:
push:
branches:
- master

jobs:
build_and_test_cpu_required:
# This job will be required to pass before merging to master branch.
name: Required Build and Test (CPU)
timeout-minutes: 60
strategy:
matrix:
include:
- os: ubuntu-latest
python: 3.6
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'

- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python }}

- name: Download Pre-Built LLVM 10.0.0
run: |
python misc/ci_download.py
mkdir taichi-llvm
cd taichi-llvm
unzip ../taichi-llvm.zip
env:
CI_PLATFORM: ${{ matrix.os }}

- name: Build & Install
run: .github/workflows/scripts/unix_build.sh
env:
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
CXX: clang++

- name: Test
run: .github/workflows/scripts/unix_test.sh

build_and_test_cpu:
name: Build and Test (CPU)
needs: build_and_test_cpu_required
timeout-minutes: 60
strategy:
matrix:
include:
- os: macos-latest
python: 3.7
with_cc: OFF
with_cpp_tests: ON
- os: ubuntu-latest
python: 3.9
with_cc: OFF
with_cpp_tests: OFF
- os: ubuntu-latest
python: 3.8
with_cc: ON
with_cpp_tests: OFF
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'

- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python }}

- name: Download Pre-Built LLVM 10.0.0
run: |
python misc/ci_download.py
mkdir taichi-llvm
cd taichi-llvm
unzip ../taichi-llvm.zip
env:
CI_PLATFORM: ${{ matrix.os }}

- name: Build & Install
run: .github/workflows/scripts/unix_build.sh
env:
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
CXX: clang++
# [DEBUG] Copy this step around to enable debugging inside Github Action instances.
#- name: Setup tmate session
# uses: mxschmitt/action-tmate@v3
# with:
# limit-access-to-actor: true

- name: Test
run: .github/workflows/scripts/unix_test.sh
env:
RUN_CPP_TESTS: ${{ matrix.with_cpp_tests }}

build_and_test_gpu_linux:
name: Build and Test (GPU)
runs-on: [self-hosted, cuda, vulkan, cn]
timeout-minutes: 60
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'

- name: Build
run: |
export PATH=$PATH:/usr/local/cuda/bin
.github/workflows/scripts/unix_build.sh
env:
LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON
BUILD_NUM_THREADS: 8
LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm
CXX: clang++-8

- name: Test
run: .github/workflows/scripts/unix_test.sh
env:
DISPLAY: :1
GPU_TEST: ON

build_and_test_windows:
name: Build and Test (Windows)
runs-on: windows-latest
timeout-minutes: 60
steps:

- name: Install 7Zip PowerShell
shell: powershell
run: Install-Module 7Zip4PowerShell -Force -Verbose

- uses: actions/checkout@v2
with:
submodules: 'recursive'

- uses: actions/setup-python@v2
with:
python-version: 3.7

- name: Add msbuild to PATH
uses: microsoft/[email protected]

- name: Download And Install Vulkan
shell: powershell
run: |
Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe" -OutFile VulkanSDK.exe
$installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
$installer.WaitForExit();
- name: Build
shell: powershell
run: |
$env:Path += ";C:/VulkanSDK/1.2.189.0/Bin"
cd C:\
Remove-item alias:curl
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
7z x clang-10.0.0-win.zip -otaichi_clang
$env:PATH += ";C:\taichi_llvm\bin"
$env:PATH += ";C:\taichi_clang\bin"
clang --version
cd D:\a\taichi\taichi
python -m pip install -r requirements_dev.txt
cd python
git fetch origin master
$env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
python build.py build
cd ..\dist
$env:WHL = $(dir *.whl)
python -m pip install $env:WHL
env:
PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
CI_SETUP_CMAKE_ARGS: -G "Visual Studio 16 2019" -A x64 -DLLVM_DIR=C:\taichi_llvm\lib\cmake\llvm -DTI_WITH_VULKAN:BOOL=ON
VULKAN_SDK: C:/VulkanSDK/1.2.189.0

- name: Test
shell: powershell
run: |
$env:PATH += ";C:\taichi_llvm\bin"
$env:PATH += ";C:\taichi_clang\bin"
python -c "import taichi"
python examples/algorithm/laplace.py
python bin/taichi diagnose
python bin/taichi changelog
python bin/taichi test -vr2 -t2
env:
PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe

build_and_test_m1:
name: Build and Test (Apple M1)
timeout-minutes: 60
strategy:
matrix:
include:
- os: macos-latest
python: 3.8
defaults:
run:
# https://github.com/actions/runner/issues/805#issuecomment-844426478
shell: "/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}"
runs-on: [self-hosted, m1]
steps:
- uses: actions/checkout@v2
with:
submodules: 'recursive'

- name: Build
run: |
python3 -m pip uninstall taichi -y
rm -rf $HOME/Library/Python/3.8/lib/python/site-packages/taichi
git --version
export CXX=clang++
python3 -m pip install -r requirements_dev.txt
cd python
git fetch origin master
TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 build.py build
cd ..
export NUM_WHL=`ls dist/*.whl | wc -l`
if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
python3 -m pip install dist/*.whl
env:
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON

- name: Test
run: |
export PATH=$PATH:$HOME/Library/Python/3.8/bin
python3 examples/algorithm/laplace.py
TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1`
TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
ti test -vr2 -t4 -x
105 changes: 17 additions & 88 deletions .github/workflows/presubmit.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@ name: Presubmit Checks
on:
pull_request:
types: [opened, synchronize, reopened]
push:
branches:
- master

jobs:
title_format:
Expand Down Expand Up @@ -79,35 +76,13 @@ jobs:
CI_PLATFORM: ${{ matrix.os }}

- name: Build & Install
run: |
TAICHI_REPO_DIR=`pwd`
export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH
export CXX=clang++
python -m pip install -r requirements_dev.txt
cd python
git fetch origin master
TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python build.py build
cd ..
export NUM_WHL=`ls dist/*.whl | wc -l`
if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
pip install dist/*.whl
run: .github/workflows/scripts/unix_build.sh
env:
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
CXX: clang++

- name: Test
run: |
TAICHI_REPO_DIR=`pwd`
export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH
# Note we only need this since we cannot write into system python package.
export PATH=$PATH:$HOME/.local/bin
hash -r
python examples/algorithm/laplace.py
ti diagnose
ti changelog
# taichi_cpp_tests requires passing in TI_LIB_DIR manually.
TI_LIB_DIR=`python -c "import taichi;print(taichi.__path__[0])" | tail -1`
TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
ti test -vr2 -t2
run: .github/workflows/scripts/unix_test.sh

build_and_test_cpu:
name: Build and Test (CPU)
Expand Down Expand Up @@ -148,39 +123,18 @@ jobs:
CI_PLATFORM: ${{ matrix.os }}

- name: Build & Install
run: |
TAICHI_REPO_DIR=`pwd`
export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH
export CXX=clang++
python -m pip install -r requirements_dev.txt
cd python
git fetch origin master
TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python build.py build
cd ..
NUM_WHL=`ls dist/*.whl | wc -l`
if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
pip install dist/*.whl
run: .github/workflows/scripts/unix_build.sh
env:
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
CXX: clang++
# [DEBUG] Copy this step around to enable debugging inside Github Action instances.
#- name: Setup tmate session
# uses: mxschmitt/action-tmate@v3
# with:
# limit-access-to-actor: true

- name: Test
run: |
TAICHI_REPO_DIR=`pwd`
export PATH=$TAICHI_REPO_DIR/taichi-llvm/bin/:$PATH
export PATH=$PATH:$HOME/.local/bin
hash -r
python examples/algorithm/laplace.py
ti diagnose
ti changelog
# taichi_cpp_tests requires passing in TI_LIB_DIR manually.
TI_LIB_DIR=`python -c "import taichi;print(taichi.__path__[0])" | tail -1`
[ "$RUN_CPP_TESTS" = "ON" ] && TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
ti test -vr2 -t2
run: .github/workflows/scripts/unix_test.sh
env:
RUN_CPP_TESTS: ${{ matrix.with_cpp_tests }}

Expand All @@ -196,37 +150,21 @@ jobs:

- name: Build
run: |
git --version
export PATH=$LLVM_LIB_ROOT_DIR/bin:/usr/local/cuda/bin:$PATH
export LLVM_DIR=$LLVM_LIB_ROOT_DIR/lib/cmake/llvm
export CXX=clang++-8
export PYTHON=/usr/bin/python3
$PYTHON -m pip uninstall taichi -y
$PYTHON -m pip install -r requirements_dev.txt
cd python
git fetch origin master
TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS $PYTHON build.py build
cd ..
export NUM_WHL=`ls dist/*.whl | wc -l`
if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
$PYTHON -m pip install dist/*.whl
export PATH=$PATH:/usr/local/cuda/bin
.github/workflows/scripts/unix_build.sh
env:
LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON
BUILD_NUM_THREADS: 8
LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm
CXX: clang++-8

- name: Test
run: |
export PYTHON=/usr/bin/python3
export PATH=$PATH:$HOME/.local/bin
export DISPLAY=:1
hash -r
glewinfo
$PYTHON examples/algorithm/laplace.py
ti diagnose
ti changelog
ti test -vr2 -t2 -k "not ndarray and not torch"
ti test -vr2 -t1 -k "ndarray or torch"
run: .github/workflows/scripts/unix_test.sh
env:
DISPLAY: :1
GPU_TEST: ON

build_and_test_windows:
name: Build and Test (Windows)
Expand Down Expand Up @@ -316,20 +254,11 @@ jobs:

- name: Build
run: |
python3 -m pip uninstall taichi -y
rm -rf $HOME/Library/Python/3.8/lib/python/site-packages/taichi
git --version
export CXX=clang++
python3 -m pip install -r requirements_dev.txt
cd python
git fetch origin master
TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 build.py build
cd ..
export NUM_WHL=`ls dist/*.whl | wc -l`
if [ $NUM_WHL -ne 1 ]; then echo 'ERROR: created more than 1 whl.' && exit 1; fi
python3 -m pip install dist/*.whl
.github/workflows/scripts/unix_build.sh
env:
CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
CXX: clang++

- name: Test
run: |
Expand Down
Loading

0 comments on commit 8f8643d

Please sign in to comment.