DEV - Update Meson-gitpod (scipy#1)

* 🏗️ [ENH] - Update Dockerfile - split into Meson and dev * ENH - Split Dockerbuilds * DEV: Add pip to avoid warning * ENH: Add docs to Dockerfile * DEV: Improve Docker build * DEV: Quick fix for env setup * DEV: Testing - will revert * DEV: Only use docutils * DEV: Revert test * DEV: Docutils install - will remove later
anshu-gt · Apr 29, 2022 · a430453 · a430453
1 parent e849b98
commit a430453
Show file tree

Hide file tree

Showing 10 changed files with 287 additions and 39 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -7,13 +7,15 @@ on:
     branches:
       - main
     paths:
+      # will keep both until there is a reason to get rid of scipy-dev entirely
       - 'environment_meson.yml'
+      - 'environment.yml'
 
 jobs:
-  build:
+  build-dev-base:
     name: Build base Docker image
     runs-on: ubuntu-latest
-    environment: scipy-meson
+    environment: scipy-dev
     if: "github.repository_owner == 'scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')"
     steps:
       - name: Clone repository
@@ -34,12 +36,52 @@ jobs:
         id: getrefs
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      - name: Cache Docker layers
-        uses: actions/cache@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: "."
+          file: "./tools/docker_dev/meson.Dockerfile"
+          push: ${{ github.event_name != 'pull_request' }}
+          # https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#github-cache
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: |
+            scipy/scipy-dev:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}
+            scipy/scipy-dev:latest
+      - name: Image digest
+        # Return details of the image build: sha and shell
+        run: echo ${{ steps.docker_build.outputs.digest }}
+
+  build-meson-base:
+    name: Build base Docker image
+    runs-on: ubuntu-latest
+    environment: scipy-dev
+    if: "github.repository_owner == 'scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')"
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v3
         with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: ${{ runner.os }}-buildx-
+          submodules: recursive
+      - name: Lint Docker
+        uses: brpaz/[email protected]
+        with:
+          dockerfile: ./tools/docker_dev/meson.Dockerfile
+      - name: Get refs
+        shell: bash
+        run: |
+          export raw_branch=${GITHUB_REF#refs/heads/}
+          echo "::set-output name=branch::${raw_branch//\//-}"
+          echo "::set-output name=date::$(date +'%Y%m%d')"
+          echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)"
+        id: getrefs
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
@@ -55,7 +97,8 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
           tags: |
-            scipy/scipy-dev:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, scipy/scipy-dev:latest
+            scipy/scipy-meson:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}
+            scipy/scipy-meson:latest
       - name: Image digest
         # Return details of the image build: sha and shell
         run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.github/workflows/gitpod.yml b/.github/workflows/gitpod.yml
@@ -8,7 +8,7 @@ jobs:
   build:
     name: Build Gitpod Docker image 
     runs-on: ubuntu-latest
-    environment: scipy-meson
+    environment: scipy-dev
     if: "github.repository_owner == 'scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')"
     steps:
       - name: Clone repository
@@ -27,12 +27,6 @@ jobs:
         id: getrefs
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: ${{ runner.os }}-buildx-
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:
@@ -47,8 +41,13 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
+          build-args: |
+            BASE_CONTAINER="scipy/scipy-meson:latest"
+            BUILD_ARG="python dev.py --build-only -j2
+            CONDA_ENV="scipy-meson"
           tags: |
-            scipy/scipy-gitpod:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, scipy/scipy-gitpod:latest
+            scipy/scipy-gitpod:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}
+            scipy/scipy-gitpod:latest
       - name: Image digest 
         # Return details of the image build: sha and shell
         run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -13,6 +13,7 @@ tasks:
       mkdir -p .vscode
       cp tools/docker_dev/settings.json .vscode/settings.json
       conda activate scipy-meson
+      python -m pip install docutils
       git submodule update --init
       python dev.py --build-only -j2
       echo "🛠 Completed rebuilding SciPy!! 🛠 "
@@ -31,13 +32,13 @@ ports:
 vscode:
   extensions:
     - eamodio.gitlens
-    - njpwerner.autodocstring
+    - bungcip.better-toml
     - lextudio.restructuredtext
-    - ritwickdey.liveserver
     - ms-python.python
+    - njpwerner.autodocstring
+    - ritwickdey.liveserver
+    - trond-snekvik.simple-rst
     - yzhang.markdown-all-in-one
-    - bungcip.better-toml
-    - mhutchie.git-graph
 
 # --------------------------------------------------------
 # using prebuilds for the container - note: atm this only

diff --git a/environment.yml b/environment.yml
@@ -42,5 +42,7 @@ dependencies:
   # For CLI
   - rich-click
   - click
+  # Else Docker raise a warning
+  - pip
   - pip:
       - doit==0.35.0
diff --git a/environment_meson.yml b/environment_meson.yml
@@ -45,5 +45,7 @@ dependencies:
   # For CLI
   - rich-click
   - click
+  # Else Docker raise a warning
+  - pip
   - pip:
       - doit==0.35.0
diff --git a/tools/docker_dev/Dockerfile b/tools/docker_dev/Dockerfile
@@ -4,8 +4,8 @@
 # Usage: 
 # -------
 # 
-# To make a local build of the container, from the 'Docker-dev' directory:
-# docker build  --rm -f "Dockerfile" -t <build-tag> "."    
+# To make a local build of the container, from the root directory:
+# docker build  --rm -f "./tools/docker_dev/Dockerfile" -t <build-tag> "."   
 # 
 # To use the container use the following command. It assumes that you are in
 # the root folder of the scipy git repository, making it available as

diff --git a/tools/docker_dev/gitpod.Dockerfile b/tools/docker_dev/gitpod.Dockerfile
@@ -1,5 +1,15 @@
+# Dockerfile for SciPy - gitpod-based development
+# Usage: 
+# -------
+# 
+# To make a local build of the container, from the root directory:
+# docker build --rm -f "./tools/docker_dev/gitpod.Dockerfile" -t <build-tag> "."    
+# docker build --rm -f "./tools/docker_dev/gitpod.Dockerfile" -t scipy/scipy-gitpod:latest "."  --build-arg BASE_CONTAINER="scipy/scipy-meson:latest" --build-arg CONDA_ENV="scipy-meson" --build-arg BUILD_ARG="python dev.py --build-only -j2"
+# 
 # Doing a local shallow clone - keeps the container secure
 # and much slimmer than using COPY directly or cloning a remote
+# To use the Meson container you'll need to use
+# --build-arg BASE_CONTAINER="scipy/scipy-meson:latest"
 ARG BASE_CONTAINER=scipy/scipy-dev:latest
 FROM ${BASE_CONTAINER} as clone
 
@@ -8,20 +18,27 @@ FROM ${BASE_CONTAINER} as clone
 COPY --chown=gitpod . /tmp/scipy_repo
 RUN git clone --depth 1 file:////tmp/scipy_repo /tmp/scipy
 
+# -----------------------------------------------------------------------------
 # Using the Scipy-dev Docker image as a base
 # This way, we ensure we have all the needed compilers and dependencies
-# while reducing the build time
+# while reducing the build time - making this a  build ARG so we can reuse for other images
+# to use the meson image instead --build-arg BASE_CONTAINER="scipy/scipy-meson:latest"
 ARG BASE_CONTAINER=scipy/scipy-dev:latest
 FROM ${BASE_CONTAINER} as build
 
+# Build argument - can pass Meson arguments during the build:
+# --build-arg BUILD_ARG="python dev.py --build-only -j2"
+# --build-arg CONDA_ENV="scipy-meson"
+ARG BUILD_ARG="python setup.py build_ext --inplace" \
+    CONDA_ENV=scipy-dev
+
 # -----------------------------------------------------------------------------
 USER root
 
 # -----------------------------------------------------------------------------
 # ---- ENV variables ----
 # ---- Directories needed ----
-ENV WORKSPACE=/workspace/scipy/ \
-    CONDA_ENV=scipy-dev
+ENV WORKSPACE=/workspace/scipy/ 
 
 # -----------------------------------------------------------------------------
 # Change default shell - this avoids issues with Conda later - note we do need
@@ -35,15 +52,18 @@ COPY --from=clone --chown=gitpod /tmp/scipy ${WORKSPACE}
 
 WORKDIR ${WORKSPACE}
 
-# Build scipy to populate the cache used by ccache
+# Build SciPy to populate the cache used by ccache
 # Must re-activate conda to ensure the ccache flags are picked up
-RUN git submodule update --init --depth=1 -- scipy/_lib/boost
-RUN git submodule update --init --depth=1 -- scipy/sparse/linalg/_propack/PROPACK
-RUN git submodule update --init --depth=1 -- scipy/_lib/unuran
-RUN git submodule update --init --depth=1 -- scipy/_lib/highs
+RUN git submodule update --init --depth=1 -- scipy/_lib/boost &&\ 
+    git submodule update --init --depth=1 -- scipy/sparse/linalg/_propack/PROPACK && \
+    git submodule update --init --depth=1 -- scipy/_lib/unuran && \
+    git submodule update --init --depth=1 -- scipy/_lib/highs
+
 RUN conda activate ${CONDA_ENV} && \
-    python setup.py build_ext --inplace && \
-    ccache -s
+    ${BUILD_ARG} && \
+    ccache -s && \ 
+    # needed for rst preview in gitpod
+    python3 -m pip install docutils
 
 # Gitpod will load the repository into /workspace/scipy. We remove the
 # directoy from the image to prevent conflicts