Skip to content
Python package

Python package #750

Sign in to view logs

Python package

Python package #750

Workflow file for this run

.github/workflows/python-package.yml at 6f85050
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Python package
on:
push:
branches: [master]
pull_request:
branches: [master]
workflow_dispatch:
jobs:
flake8:
runs-on: ubuntu-latest
timeout-minutes: 30
defaults:
run:
shell: bash -l {0}
strategy:
matrix:
python-version: [3.9, "3.10", "3.11"]
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
# Set fetch-depth to 0 so all history is retrieved; this is needed so we get the git tags
# which we use for setting the package version (via setuptools-scm).
fetch-depth: 0
- name: Setup Miniconda
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: "true"
uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: "flake8_env"
python-version: ${{ matrix.python-version }}
miniforge-version: latest
use-mamba: true
auto-update-conda: false
channels: conda-forge
channel-priority: flexible
show-channel-urls: true
- name: Configure Conda
run: |
conda config --set unsatisfiable_hints_check_depth 0 # setting unsatisfiable_hints=False is broken
- name: Install dependencies
run: |
python dev_tools/gen_requirements.py --out flake8_reqs.txt flake8
mamba install -q -y --file flake8_reqs.txt
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
conda_build:
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}
env:
python_version: 3.11
ANACONDA_USER: rtosholdings
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Miniconda
uses: conda-incubator/setup-miniconda@v3
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: "true"
with:
activate-environment: "conda_build"
python-version: ${{ env.python_version }}
miniforge-version: latest
use-mamba: true
auto-update-conda: false
channels: conda-forge
channel-priority: flexible
show-channel-urls: true
- name: Install dependencies
run: |
set -ex
python dev_tools/gen_requirements.py --out conda_reqs.txt conda
mamba install -q -y --override-channels -c conda-forge -c defaults --file conda_reqs.txt
conda list
- name: Build package
id: build_package
run: |
set -ex
conda config --set conda_build.pkg_format 2 # create .conda package format
conda config --set conda_build.zstd_compression_level 19 # set .conda package format compression level
export BUILD_VERSION=$(python -c "from setuptools_scm import get_version; print(get_version(version_scheme='post-release'))")
echo "BUILD_VERSION=${BUILD_VERSION}" >> "$GITHUB_OUTPUT"
mkdir conda_pkgs_output
conda mambabuild conda_recipe --override-channels -c ${ANACONDA_USER} -c conda-forge -c defaults --output-folder ./conda_pkgs_output --no-test
- name: Publish artifacts
uses: actions/upload-artifact@v4
with:
name: conda-build-artifacts
path: conda_pkgs_output/*/riptable-*.conda
if-no-files-found: "error"
outputs:
build_version: ${{steps.build_package.outputs.BUILD_VERSION}}
pypi_build:
runs-on: ubuntu-latest
timeout-minutes: 30
defaults:
run:
shell: bash -l {0}
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
# Set fetch-depth to 0 so all history is retrieved; this is needed so we get the git tags
# which we use for setting the package version (via setuptools-scm).
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
- name: Install dependencies
run: |
set -ex
pip install build
- name: Package sources
run: |
set -ex
python -m build --sdist
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: build-artifacts
path: dist/
if-no-files-found: error
conda_test:
needs: [conda_build]
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash -l {0}
strategy:
matrix:
os: ["ubuntu-latest", "windows-2022"]
python-version: [3.9, "3.10", "3.11"]
numpy-version: [1.23, 1.24]
env:
ANACONDA_USER: rtosholdings
BUILD_VERSION: ${{needs.conda_build.outputs.build_version}}
steps:
- name: Checkout repo (sparse)
uses: actions/checkout@v4
with:
sparse-checkout: dev_tools
sparse-checkout-cone-mode: false
- name: Setup Miniconda
uses: conda-incubator/setup-miniconda@v3
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: "true"
with:
activate-environment: "conda_test"
python-version: ${{ matrix.python-version }}
miniforge-version: latest
use-mamba: true
auto-update-conda: false
channels: conda-forge
channel-priority: flexible
show-channel-urls: true
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: conda-build-artifacts
path: conda_pkgs_output/
- name: Install dependencies
run: |
set -ex
ls -lF
mamba install -q -y --override-channels -c conda-forge -c defaults conda-build
mamba list
- name: Init testing package
run: |
set -ex
conda index --no-progress ./conda_pkgs_output
mamba create -q -y -n conda_test --override-channels -c ./conda_pkgs_output -c ${ANACONDA_USER} -c conda-forge -c defaults python=${{ matrix.python-version }} numpy=${{ matrix.numpy-version }} "riptable==${BUILD_VERSION}"
mamba list
python -c 'import riptable; print(riptable, riptable.__version__); print(riptable.rc, riptable.rc.__version__)'
python dev_tools/gen_requirements.py --out tests_reqs.txt tests
mamba install -q -y --override-channels -c conda-forge -c defaults --file tests_reqs.txt
conda list
- name: Test riptable
run: |
set -ex
python -m pytest --pyargs riptable.tests
# disable tooling integration tests until they work
# ipython -m pytest riptable/test_tooling_integration
# disable hypothesis tests until they run faster, are more consistent, and are easier to investigate
# pytest --hypothesis-show-statistics -k test_ -m 'not xfail' riptable/hypothesis_tests
pypi_test:
needs: [pypi_build]
runs-on: ${{ matrix.os }}
timeout-minutes: 30
defaults:
run:
shell: bash -l {0}
strategy:
matrix:
os: [ubuntu-latest, windows-2022]
python-version: [3.9, "3.10", "3.11"]
numpy-version: [1.23, 1.24]
steps:
- name: Checkout repo (sparse)
uses: actions/checkout@v4
with:
sparse-checkout: dev_tools
sparse-checkout-cone-mode: false
- name: Setup Miniconda
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: "true"
uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: "pypi_test"
python-version: ${{ matrix.python-version }}
miniforge-version: latest
use-mamba: true
auto-update-conda: false
channels: conda-forge
channel-priority: flexible
show-channel-urls: true
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: build-artifacts
path: dist/
- name: Install platform dependencies
run: |
set -ex
ls -lF
python dev_tools/gen_requirements.py --out pypi_reqs.txt pypi
mamba create -q -y -n pypi_test python=${{ matrix.python-version }} --file pypi_reqs.txt
mamba list
- name: Install riptable and dependencies
# Needed for pip install of riptide_cpp from sdist
run: |
python -m pip install --upgrade pip
pip install numpy==${{ matrix.numpy-version }}.*
# Pin build-constraints for numpy (see https://github.com/pypa/pip/issues/9542#issuecomment-1242347397)
echo "numpy==${{ matrix.numpy-version }}.*" > constraints.txt
# Pip install riptable, along with riptide_cpp and all dependencies.
PIP_CONSTRAINT=constraints.txt pip install -v ./dist/riptable*.gz
python -c 'import riptable; print(riptable, riptable.__version__); print(riptable.rc, riptable.rc.__version__)'
- name: Install test dependencies
run: |
python dev_tools/gen_requirements.py --out runtime_reqs.txt runtime
python dev_tools/gen_requirements.py --out tests_reqs.txt tests
pip install -r runtime_reqs.txt -r tests_reqs.txt
conda list
- name: Test with pytest
run: |
python -m pytest --pyargs riptable.tests
- name: Tooling integration tests
run: |
echo "DISABLED until tooling tests can be updated"
#ipython -m riptable.test_tooling_integration.run
# disable hypothesis tests until they run faster, are more consistent, and are easier to investigate
#- name: Property based hypothesis tests
# run: |
# pytest --hypothesis-show-statistics -k test_ -m 'not xfail' riptable/hypothesis_tests
conda_deploy:
if: ${{ github.event_name == 'workflow_dispatch' && github.ref_type == 'tag' }}
needs: [conda_build, conda_test, pypi_test]
runs-on: ubuntu-latest
env:
# pin to py3.11, as anaconda-client broken on py3.12 (see https://github.com/Anaconda-Platform/anaconda-client/issues/689)
python_version: 3.11
ANACONDA_USER: rtosholdings
ANACONDA_TOKEN: ${{ secrets.anaconda_token }}
steps:
- name: Setup Miniconda
uses: conda-incubator/setup-miniconda@v3
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: "true"
with:
activate-environment: "conda_deploy"
python-version: ${{ env.python_version }}
miniforge-version: latest
use-mamba: true
auto-update-conda: false
channels: conda-forge
channel-priority: flexible
show-channel-urls: true
- name: Install dependencies
shell: bash -l {0}
run: |
mamba install anaconda-client -q -y
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: conda-build-artifacts
path: conda_pkgs_output/
- name: Upload to Anaconda
shell: bash -l {0}
run: |
set -ex
anaconda --token "${ANACONDA_TOKEN}" upload --label main --user ${ANACONDA_USER} ./conda_pkgs_output/*/riptable-*.conda
pypi_deploy:
if: ${{ github.event_name == 'workflow_dispatch' && github.ref_type == 'tag' }}
# since riptable is all python source code, only a source build is required from one os
needs: [pypi_build, pypi_test, conda_test]
runs-on: ubuntu-latest
steps:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install setuptools wheel twine
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: build-artifacts
path: dist/
- name: Publish artifacts to PyPI
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
set -ex
twine upload dist/* --verbose