Skip to content

Commit

Permalink
BLD: Moving powershell to python
Browse files Browse the repository at this point in the history
  • Loading branch information
richafrank committed Mar 23, 2016
1 parent c843a07 commit 7852d69
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 24 deletions.
12 changes: 7 additions & 5 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ install:
- conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9
- source activate testenv
- IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION"
- NPY_VERSION_MAJ_MIN=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}
- CACHE_DIR="$HOME/.cache/.pip/pip_np""$NPY_VERSION_MAJ_MIN"
- CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}
- CONDA_PY=$TRAVIS_PYTHON_VERSION
- CACHE_DIR="$HOME/.cache/.pip/pip_np""CONDA_NPY"
- pip install --upgrade pip coverage coveralls --cache-dir=$CACHE_DIR
- pip install -r etc/requirements.txt --cache-dir=$CACHE_DIR
- pip install -r etc/requirements_dev.txt --cache-dir=$CACHE_DIR
Expand All @@ -47,13 +48,14 @@ script:
for recipe in $(ls -d conda/*/ | xargs -I {} basename {}); do
if [[ "$recipe" = "zipline" ]]; then continue; fi
conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
RECIPE_OUTPUT=$(conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output)
conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
RECIPE_OUTPUT=$(conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --output)
if [[ -f "$RECIPE_OUTPUT" && "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; fi
done
# unshallow the clone so the conda build can clone it.
- git fetch --unshallow
- exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3))
- exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$CONDA_PY --numpy=$CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3))
- ZP_OUTPUT=$(echo "$ZP_OUT" | grep "anaconda upload" | awk '{print $NF}')
- if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci; fi
# reactivate env (necessary for coveralls)
Expand Down
20 changes: 1 addition & 19 deletions appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -105,25 +105,7 @@ install:
- conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes -q
# https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation
- ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64"

- ps: >-
foreach ($recipe in dir .\conda -dir -name) { `
if ($recipe -eq "zipline") { continue } `
iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1; `
if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } `
$RECIPE_OUTPUT=conda build conda/$recipe --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --output; `
if ((Test-Path $RECIPE_OUTPUT) -and ($env:ANACONDA_TOKEN)) { `
iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $RECIPE_OUTPUT -u quantopian --label ci --no-progress" 2>&1; `
if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } `
} `
}
- ps: iex "$env:CMD_IN_ENV conda build conda/zipline -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1 | tee -variable ZP_OUT
- ps: $ZP_OUTPUT=(($ZP_OUT | sls "anaconda upload") -split ' ')[-1]
- ps: >-
if ((Test-Path $ZP_OUTPUT) -and ($env:ANACONDA_TOKEN)) { `
iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci --no-progress" 2>&1; `
if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } `
}
- "%CMD_IN_ENV% python .\\ci\\make_conda_packages.py"

# test that we can conda install zipline in a new env
- conda create -n installenv --yes -q --use-local python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% zipline -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
Expand Down
58 changes: 58 additions & 0 deletions ci/make_conda_packages.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import os
import re
import subprocess


def get_immediate_subdirectories(a_dir):
return [name for name in os.listdir(a_dir)
if os.path.isdir(os.path.join(a_dir, name))]


def iter_stdout(cmd):
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)

try:
for line in iter(p.stdout.readline, b''):
yield line.decode().rstrip()
finally:
retcode = p.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, cmd[0])


PKG_PATH_PATTERN = re.compile(".* anaconda upload (?P<pkg_path>.+)$")


def main():
for recipe in get_immediate_subdirectories('conda'):
cmd = ["conda", "build", os.path.join('conda', recipe),
"--python", os.environ['CONDA_PY'],
"--numpy", os.environ['CONDA_NPY'],
"--skip-existing",
"-c", "quantopian",
"-c", "https://conda.anaconda.org/quantopian/label/ci"]

output = None

for line in iter_stdout(cmd):
print(line)

if not output:
match = PKG_PATH_PATTERN.match(line)
if match:
output = match.group('pkg_path')

if (output and os.path.exists(output) and
os.environ.get('ANACONDA_TOKEN')):

cmd = ["anaconda", "-t", os.environ['ANACONDA_TOKEN'],
"upload", output, "-u", "quantopian", "--label", "ci"]

for line in iter_stdout(cmd):
print(line)


if __name__ == '__main__':
main()

0 comments on commit 7852d69

Please sign in to comment.