[wip][dependencies] pip-compile dependencies as a build step (ray-pro…

…ject#39610) It's currently complicated to update dependencies, as it requires a (manual) recompilation of the `requirements_compiled.txt` file. This PR adds a separate build job that runs pip compile on all input files to generate an updated `requirements_compiled.txt`, that is than exposed as an artifact in the buildkite job. To update a dependency in the future, the process would be: 1. Open a PR that updates the respective requirements in the input requirements file 2. Wait for the CI job to run through and output the updated compiled requirements file 3. If the compilation was successful, add the contents of the file to the PR Signed-off-by: Kai Fricke <[email protected]>
nemo9cby · Sep 22, 2023 · f311495 · f311495
1 parent 1bf3e9f
commit f311495
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 2 deletions.
diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml
@@ -47,8 +47,18 @@ steps:
     commands:
       - bazel run //ci/ray_ci:build_in_docker -- docker --python-version py38 
         --platform cu118 --image-type ray-ml
-    depends_on: 
+    depends_on:
       - manylinux
       - forge
       - ray-mlpy38cu118base
     job_env: forge
+
+  - label: ":tapioca: build: pip-compile dependencies"
+    instance_type: small
+    commands:
+      - rm ./python/requirements_compiled.txt
+      - ./ci/ci.sh compile_pip_dependencies
+      - cp -f ./python/requirements_compiled.txt /artifact-mount/
+    depends_on:
+      - forge
+    job_env: forge
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -25,5 +25,9 @@ if [ -d "/tmp/artifacts" ]; then
     find /tmp/artifacts -print || true
 fi
 
+echo "Creating var/ artifact directory..."
+docker run --rm -v /tmp/artifacts:/artifact-mount alpine:latest /bin/sh -c 'chown -R 2000 /artifact-mount/' || true
+
+
 RAYCI_CHECKOUT_DIR="$(pwd)"
 export RAYCI_CHECKOUT_DIR
diff --git a/ci/ci.sh b/ci/ci.sh
@@ -113,7 +113,7 @@ compile_pip_dependencies() {
   pip install --no-cache-dir numpy torch
 
   pip-compile --resolver=backtracking -q \
-     --pip-args --no-deps --strip-extras --no-annotate --no-header -o \
+     --pip-args --no-deps --strip-extras --no-header -o \
     "${WORKSPACE_DIR}/python/$TARGET" \
     "${WORKSPACE_DIR}/python/requirements.txt" \
     "${WORKSPACE_DIR}/python/requirements/lint-requirements.txt" \