devenv: env per Python version (conda#11233)

* Rework to allow multiple envs Makes environments more descriptive (no longer `base` prefix, uses `devenv-$PYTHON` prefix) and allow side-by-side installs of multiple different Python versions. * Modify PYTHONPATH * devenv in ~/.condarc & --dry-run * .bat line endings * POSIX compliance * exists/pending in --dry-run * Rename to development environment * Also set CONDA_PYTHON_EXE
prod-feng · Feb 19, 2022 · 2967d90 · 2967d90
1 parent b039ae2
commit 2967d90
Show file tree

Hide file tree

Showing 4 changed files with 455 additions and 95 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,4 +1,4 @@
 *.sh text eol=lf
 *.psm1 text eol=lf
 dev/start text eol=lf
-*activate.bat text eol=crlf
+*.bat text eol=crlf
diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat
@@ -1,15 +1,15 @@
-copy "%RECIPE_DIR%\build.sh" .
-FOR /F "delims=" %%i IN ('cygpath.exe -u "%LIBRARY_PREFIX%"') DO set "LIBRARY_PREFIX=%%i"
-SET PREFIXW=%PREFIX%
-FOR /F "delims=" %%i IN ('cygpath.exe -u "%PREFIX%"') DO set "PREFIX=%%i"
-FOR /F "delims=" %%i IN ('cygpath.exe -u "%PYTHON%"') DO set "PYTHON=%%i"
-FOR /F "delims=" %%i IN ('cygpath.exe -u "%RECIPE_DIR%"') DO set "RECIPE_DIR=%%i"
-FOR /F "delims=" %%i IN ('cygpath.exe -u "%SP_DIR%"') DO set "SP_DIR=%%i"
-FOR /F "delims=" %%i IN ('cygpath.exe -u "%SRC_DIR%"') DO set "SRC_DIR=%%i"
-FOR /F "delims=" %%i IN ('cygpath.exe -u "%STDLIB_DIR%"') DO set "STDLIB_DIR=%%i"
-set MSYSTEM=MINGW%ARCH%
-set MSYS2_PATH_TYPE=inherit
-set CHERE_INVOKING=1
-bash -lc ./build.sh
-if %errorlevel% neq 0 exit /b %errorlevel%
-exit /b 0
+copy "%RECIPE_DIR%\build.sh" .
+FOR /F "delims=" %%i IN ('cygpath.exe -u "%LIBRARY_PREFIX%"') DO set "LIBRARY_PREFIX=%%i"
+SET PREFIXW=%PREFIX%
+FOR /F "delims=" %%i IN ('cygpath.exe -u "%PREFIX%"') DO set "PREFIX=%%i"
+FOR /F "delims=" %%i IN ('cygpath.exe -u "%PYTHON%"') DO set "PYTHON=%%i"
+FOR /F "delims=" %%i IN ('cygpath.exe -u "%RECIPE_DIR%"') DO set "RECIPE_DIR=%%i"
+FOR /F "delims=" %%i IN ('cygpath.exe -u "%SP_DIR%"') DO set "SP_DIR=%%i"
+FOR /F "delims=" %%i IN ('cygpath.exe -u "%SRC_DIR%"') DO set "SRC_DIR=%%i"
+FOR /F "delims=" %%i IN ('cygpath.exe -u "%STDLIB_DIR%"') DO set "STDLIB_DIR=%%i"
+set MSYSTEM=MINGW%ARCH%
+set MSYS2_PATH_TYPE=inherit
+set CHERE_INVOKING=1
+bash -lc ./build.sh
+if %errorlevel% neq 0 exit /b %errorlevel%
+exit /b 0
diff --git a/dev/start b/dev/start
@@ -1,45 +1,214 @@
 #!/usr/bin/env sh
 # NOTE: This script should be sourced! The shebang is only here to help syntax highlighters.
 
-DEVENV="${1:-./devenv}"
-PYTHON="${PYTHON:-3.8}"
-_CONDA="${DEVENV}/bin/conda"
-
-if ! [ -f "$DEVENV/conda-meta/history" ]; then
-    if [ "$(uname)" = Darwin ]; then
-        if [ ! -f miniconda3.sh ]; then
-            curl "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" -o miniconda3.sh
-        fi
-        bash miniconda3.sh -bfp "$DEVENV"
-        rm -f miniconda3.sh
-        "${_CONDA}" install -yq -p "$DEVENV" python="$PYTHON" --file tests/requirements.txt -c defaults
-    elif [ "$(uname)" = Linux ]; then
-        if [ ! -f miniconda3.sh ]; then
-            curl "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -o miniconda3.sh
+if ! (return 0 2> /dev/null); then
+    echo "ERROR: Source this script: source '$0'." >&2
+    exit 1
+fi
+
+cleanup() {
+    unset _BASEEXE
+    unset _CMD
+    unset _DEVENV
+    unset _DRYRUN
+    unset _ENV
+    unset _ENVEXE
+    unset _NAME
+    unset _PYTHON
+    unset _PYTHONEXE
+    unset _SRC
+    unset _UPDATE
+    unset _UPDATED
+}
+cleanup
+
+# parse args
+while [ $# -gt 0 ]; do
+    case $1 in
+        -p|--python)
+            _PYTHON="$2"
+            shift
+            shift
+            ;;
+        --python=*)
+            _PYTHON="${1#*=}"
+            shift
+            ;;
+        -u|--update)
+            _UPDATE=0
+            shift
+            ;;
+        -d|--devenv)
+            _DEVENV="$2"
+            shift
+            shift
+            ;;
+        --devenv=*)
+            _DEVENV="${1#*=}"
+            shift
+            ;;
+        -n|--dry-run)
+            _DRYRUN=0
+            shift
+            ;;
+        -h|--help)
+            # since zsh is MacOS standard fallback to $0 if not in bash
+            echo "Usage: source ${BASH_SOURCE:-$0} [options]"
+            echo ""
+            echo "Options:"
+            echo "  -p, --python  VERSION  Python version for the env to activate. (default: 3.8)"
+            echo "  -u, --update           Force update packages. (default: update every 24 hours)"
+            echo "  -d, --devenv  PATH     Path to base env install, can also be defined in ~/.condarc."
+            echo "                         Path is appended with $(uname). (default: devenv)"
+            echo "  -n, --dry-run          Display env to activate. (default: false)"
+            echo "  -h, --help             Display this."
+            return 0
+            ;;
+        *)
+            echo "Error: unknown option $1" >&2
+            return 1
+            ;;
+    esac
+done
+
+# get source path
+# since zsh is MacOS standard fallback to $0 if not in bash
+_SRC="$(cd "$(dirname "$(dirname "${BASH_SOURCE:-$0}")")" 2>&1 > /dev/null; pwd -P)"
+
+# read devenv key from ~/.condarc
+if [ -f ~/.condarc ]; then
+    _DEVENV="${_DEVENV:-$(grep "^devenv:" ~/.condarc | tail -n 1 | sed -e "s/^.*:[[:space:]]*//" -e "s/[[:space:]]*$//")}"
+fi
+# fallback to devenv in source default
+_DEVENV="${_DEVENV:-${_SRC}/devenv}"
+# tilde expansion
+_DEVENV="${_DEVENV/#\~/${HOME}}"
+# include OS
+_DEVENV="${_DEVENV}/$(uname)"
+# ensure exists and absolute path
+mkdir -p "${_DEVENV}"
+_DEVENV="$( cd "${_DEVENV}" 2>&1 > /dev/null ; pwd -P)"
+
+# fallback to default values
+_PYTHON="${_PYTHON:-3.8}"
+_UPDATE="${_UPDATE:-1}"
+_DRYRUN="${_DRYRUN:-1}"
+
+# other environment values
+_NAME="devenv-${_PYTHON}-c"
+_ENV="${_DEVENV}/envs/${_NAME}"
+_UPDATED="${_ENV}/.devenv-updated"
+case "$(uname)" in
+    Darwin|Linux) _BASEEXE="${_DEVENV}/bin/conda" ; _ENVEXE="${_ENV}/bin/conda" ; _PYTHONEXE="${_ENV}/bin/python" ;;
+               *) _BASEEXE="${_DEVENV}/Scripts/conda.exe" ; _ENVEXE="${_ENV}/Scripts/conda.exe" ; _PYTHONEXE="${_ENV}/Scripts/python.exe" ;;
+esac
+
+# dryrun printout
+if [ ${_DRYRUN} = 0 ]; then
+    echo "Python: ${_PYTHON}"
+    echo "Updating: $([ ${_UPDATE} = 0 ] && echo "[yes]" || echo "[pending]")"
+    echo "Devenv: ${_DEVENV} $([ -e "${_DEVENV}" ] && echo "[exists]" || echo "[pending]")"
+    echo ""
+    echo "Name: ${_NAME}"
+    echo "Path: ${_ENV} $([ -e "${_ENV}" ] && echo "[exists]" || echo "[pending]")"
+    echo ""
+    echo "Source: ${_SRC}"
+    return 0
+fi
+
+# deactivate any prior envs
+if ! [ ${CONDA_SHLVL:-0} = 0 ]; then
+    echo "Deactivating ${CONDA_SHLVL} environment(s)..."
+    while ! [ ${CONDA_SHLVL:-0} = 0 ]; do
+        if ! conda deactivate; then
+            echo "Error: failed to deactivate environment(s)" 1>&2
+            return 1
         fi
-        bash miniconda3.sh -bfp "$DEVENV"
-        rm -f miniconda3.sh
-        "${_CONDA}" install -yq -p "$DEVENV" python="$PYTHON" --file tests/requirements.txt -c defaults
-        "${_CONDA}" install -yq -p "$DEVENV" patchelf  # for conda-build
+    done
+fi
+
+# does miniconda install exist?
+if ! [ -f "${_DEVENV}/conda-meta/history" ]; then
+    # downloading miniconda
+    if [ "$(uname)" = "Darwin" ]; then
+        [ -f "${_DEVENV}/miniconda.sh" ] || _CMD='curl -s "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" -o "${_DEVENV}/miniconda.sh"'
+    elif [ "$(uname)" = "Linux" ]; then
+        [ -f "${_DEVENV}/miniconda.sh" ] || _CMD='curl -s "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -o "${_DEVENV}/miniconda.sh"'
     else
-        if [ ! -f "miniconda3.exe" ]; then
-            powershell.exe -Command "(new-object System.Net.WebClient).DownloadFile('https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe','miniconda3.exe')"
+        [ -f "${_DEVENV}/miniconda.exe" ] || _CMD='powershell.exe -Command "Invoke-WebRequest -Uri 'https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe' -OutFile '${_DEVENV}/miniconda.exe' | Out-Null"'
+    fi
+    if [ ${_CMD:+0} = 0 ]; then
+        echo "Downloading miniconda..."
+        eval ${_CMD}
+        if ! [ $? = 0 ]; then
+            echo "Error: failed to download miniconda" 1>&2
+            return 1
         fi
-        cmd.exe /c "start /wait \"\" miniconda3.exe /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /S /D=%CD%\$(cygpath -w $DEVENV)"
-        _CONDA="$DEVENV/Scripts/conda"
-        "$DEVENV/Scripts/conda" install -yq -p "$DEVENV" python="$PYTHON" --file tests/requirements.txt -c defaults
-        rm -f miniconda3.exe
+    fi
+
+    # installing miniconda
+    echo "Installing development environment..."
+    case "$(uname)" in
+        Darwin|Linux) bash "${_DEVENV}/miniconda.sh" -bfp "${_DEVENV}" > /dev/null ;;
+                   *) cmd.exe /c "start /wait \"\" \"${_DEVENV}\miniconda.exe\" /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /S /D=${_DEVENV} > NUL" ;;
+    esac
+    if ! [ $? = 0 ]; then
+        echo "Error: failed to install development environment" 1>&2
+        return 1
+    fi
+fi
+
+# create empty env if it doesn't exist
+if ! [ -d "${_ENV}" ]; then
+    echo "Creating ${_NAME}..."
+    if ! "${_BASEEXE}" create -yq --prefix "${_ENV}" > /dev/null; then
+        echo "Error: failed to create ${_NAME}" 1>&2
+        return 1
     fi
 fi
 
+# check if explicitly updating or if 24 hrs since last update
+if [ ${_UPDATE} = 0 ] || ! [ -f "${_UPDATED}" ] || (( $(( $(date +%s) - $(date -r "${_UPDATED}" +%s) )) >= 86400 )); then
+    echo "Updating ${_NAME}..."
+
+    if ! "${_BASEEXE}" update -yq --all > /dev/null; then
+        echo "Error: failed to update development environment" 1>&2
+        return 1
+    fi
 
-if ! [ -f "$DEVENV/.dev-start-ed" ]; then
-  "${_CONDA}" install -yq -p "$DEVENV" python="$PYTHON" --file tests/requirements.txt -c defaults
-  touch "$DEVENV/.dev-start-ed"
+    if ! "${_BASEEXE}" install \
+        -yq \
+        --prefix "${_ENV}" \
+        --override-channels \
+        -c defaults \
+        --file "${_SRC}/tests/requirements.txt" \
+        $([ "$(uname)" = "Linux" ] && echo "patchelf") \
+        "python=${_PYTHON}" > /dev/null; then
+        echo "Error: failed to update ${_NAME}" 1>&2
+        return 1
+    fi
+
+    # update timestamp
+    touch "${_UPDATED}"
 fi
 
+# initialize conda command
+echo "Initializing shell integration..."
+eval "$(cd "${_SRC}" ; "${_BASEEXE}" init --dev bash)" > /dev/null
+if ! [ $? = 0 ]; then
+    echo "Error: failed to initialize shell integration" 1>&2
+    return 1
+fi
 
-case "$(uname)" in
-    Darwin|Linux)  eval "$($DEVENV/bin/python -m conda init --dev bash)" ;;
-               *)  eval "$($DEVENV/python -m conda init --dev bash)" ;;
-esac
+# activate env
+echo "Activating ${_NAME}..."
+if ! conda activate "${_ENV}" > /dev/null; then
+    echo "Error: failed to activate ${_NAME}" 1>&2
+    return 1
+fi
+export CONDA_EXE="${_ENVEXE}"
+export CONDA_PYTHON_EXE="${_PYTHONEXE}"
+export PYTHONPATH="${_SRC}:${PYTHONPATH}"
+
+cleanup
+unset -f cleanup