Merge pull request #11 from aisingapore/0.2.0-fix

0.2.0 fix (part 2)

{{cookiecutter.repo_name}}/.gitlab-ci.yml

@@ -2,25 +2,35 @@ default:
   tags:
     - on-prem
 
-variables:
-  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-
-cache:
-  paths:
-    - .cache/pip
-
 stages:
   - test
-  - build
   - deploy-docs
+  - build
+
+test:conda-build:
+  stage: test
+  image:
+    name: continuumio/miniconda3:23.10.0-1
+  script:
+    - conda env create -f {{cookiecutter.repo_name}}-conda-env.yaml -p ./conda/{{cookiecutter.repo_name}}
+  rules:
+    - if: $CI_MERGE_REQUEST_IID
+      changes:
+        - {{cookiecutter.repo_name}}-conda-env.yaml
+    - if: $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_COMMIT_TAG
+      when: never
+  artifacts:
+    paths:
+      - ./conda
+  needs: []
 
 test:pylint-pytest:
   stage: test
   image:
-    name: continuumio/miniconda:4.7.12
+    name: continuumio/miniconda3:23.10.0-1
   before_script:
-    - conda env create -f {{cookiecutter.repo_name}}-conda-env.yaml
-    - source activate {{cookiecutter.repo_name}}
+    - source activate ./conda/{{cookiecutter.repo_name}}
     - pip install -r dev-requirements.txt
   script:
     - pylint src --fail-under=7.0 --ignore=tests --disable=W1202
@@ -33,14 +43,34 @@ test:pylint-pytest:
     - if: $CI_PIPELINE_SOURCE == "push"
     - if: $CI_COMMIT_TAG
       when: never
+  needs:
+    - test:conda-build
+
+pages:
+  stage: deploy-docs
+  image:
+    name: continuumio/miniconda3:23.10.0-1
+  before_script:
+    - source activate ./conda/{{cookiecutter.repo_name}}
+    - pip install -r docs-requirements.txt
+  script:
+    - sphinx-apidoc -f -o docs src
+    - sphinx-build -b html docs public
+  artifacts:
+    paths:
+    - public
+  only:
+    - main
+  needs:
+    - test:conda-build
 
 build:data-prep-image:
   stage: build
   image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [""]
   before_script:
-    - "[[ -z ${HARBOR_ROBOT_CREDS_JSON} ]] && exit 1"
+    - "[[ -z ${HARBOR_ROBOT_CREDS_JSON} ]] && echo 'HARBOR_ROBOT_CREDS_JSON variable is not set.' && exit 1"
   script:
     - mkdir -p /kaniko/.docker
     - cat $HARBOR_ROBOT_CREDS_JSON > /kaniko/.docker/config.json
@@ -58,14 +88,16 @@ build:data-prep-image:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_PIPELINE_SOURCE == "web" && $BUILD_ALL
     - if: $CI_PIPELINE_SOURCE == "web" && $BUILD_DATAPREP
+  needs:
+    - test:pylint-pytest
 
 build:model-training-image:
   stage: build
   image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [""]
   before_script:
-    - "[[ -z ${HARBOR_ROBOT_CREDS_JSON} ]] && exit 1"
+    - "[[ -z ${HARBOR_ROBOT_CREDS_JSON} ]] && echo 'HARBOR_ROBOT_CREDS_JSON variable is not set.' && exit 1"
   script:
     - mkdir -p /kaniko/.docker
     - cat $HARBOR_ROBOT_CREDS_JSON > /kaniko/.docker/config.json
@@ -83,33 +115,19 @@ build:model-training-image:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_PIPELINE_SOURCE == "web" && $BUILD_ALL
     - if: $CI_PIPELINE_SOURCE == "web" && $BUILD_MODEL
+  needs:
+    - test:pylint-pytest
 
 build:retag-images:
   stage: build
   image:
     name: gcr.io/go-containerregistry/crane:debug
     entrypoint: [""]
   before_script:
-    - "[[ -z ${HARBOR_ROBOT_CREDS_JSON} ]] && exit 1"
+    - "[[ -z ${HARBOR_ROBOT_CREDS_JSON} ]] && echo 'HARBOR_ROBOT_CREDS_JSON variable is not set.' && exit 1"
   script:
     - cat $HARBOR_ROBOT_CREDS_JSON > /root/.docker/config.json
     - crane tag {{cookiecutter.registry_project_path}}/data-prep:${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_TAG}
     - crane tag {{cookiecutter.registry_project_path}}/model-training:${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_TAG}
   rules:
     - if: $CI_COMMIT_TAG && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-
-pages:
-  stage: deploy-docs
-  image:
-    name: continuumio/miniconda:4.7.12
-  before_script:
-    - conda env create -f {{cookiecutter.repo_name}}-conda-env.yaml
-    - source activate {{cookiecutter.repo_name}}
-  script:
-    - sphinx-apidoc -f -o docs src
-    - sphinx-build -b html docs public
-  artifacts:
-    paths:
-    - public
-  only:
-    - main

{{cookiecutter.repo_name}}/aisg-context/guide-site/docs/guide-for-user/07-job-orchestration.md

@@ -63,6 +63,18 @@ provided in this template:
     $ docker push {{cookiecutter.registry_project_path}}/data-prep:0.1.0
     ```
 
+=== "VSCode Server Terminal"
+
+    ```bash
+    # Run `runai login` and `runai config project {{cookiecutter.proj_name}}` first if needed
+    # Run this in the base of your project repository, and change accordingly
+    $ khull kaniko --context $(pwd) \
+        --dockerfile $(pwd)/docker/{{cookiecutter.repo_name}}-data-prep.Dockerfile \
+        --destination {{cookiecutter.registry_project_path}}/data-prep:0.1.0 \
+        --cred-file /path/to/docker/config.json \
+        -v <pvc-name>:/path/to/pvc/mount
+    ```
+
 === "Run:ai YAML"
 
     ```bash
@@ -173,6 +185,18 @@ we need to build the Docker image to be used for it:
     $ docker push {{cookiecutter.registry_project_path}}/model-training:0.1.0
     ```
 
+=== "VSCode Server Terminal"
+
+    ```bash
+    # Run `runai login` and `runai config project {{cookiecutter.proj_name}}` first if needed
+    # Run this in the base of your project repository, and change accordingly
+    $ khull kaniko --context $(pwd) \
+        --dockerfile $(pwd)/docker/{{cookiecutter.repo_name}}-model-training.Dockerfile \
+        --destination {{cookiecutter.registry_project_path}}/model-training:0.1.0 \
+        --cred-file /path/to/docker/config.json \
+        -v <pvc-name>:/path/to/pvc/mount
+    ```
+
 === "Run:ai YAML"
 
     ```bash

{{cookiecutter.repo_name}}/aisg-context/guide-site/docs/guide-for-user/08-deployment.md

@@ -46,7 +46,7 @@ Run within your VSCode server with the MLFlow Client library:
     $ export MLFLOW_TRACKING_URI=<MLFLOW_TRACKING_URI>
     $ export MLFLOW_TRACKING_USERNAME=<MLFLOW_TRACKING_USERNAME>
     $ export MLFLOW_TRACKING_PASSWORD=<MLFLOW_TRACKING_PASSWORD>
-    $ python -c "import mlflow; mlflow.artifacts.download_artifact(artifact_uri='runs:/<MLFLOW_RUN_UUID>/', dst_path='models/<MLFLOW_RUN_UUID')"
+    $ python -c "import mlflow; mlflow.artifacts.download_artifacts(artifact_uri='runs:/<MLFLOW_RUN_UUID>/', dst_path='models/<MLFLOW_RUN_UUID')"
     Downloading artifacts: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00,  1.18it/s]
     ```
 
@@ -56,7 +56,7 @@ Run within your VSCode server with the MLFlow Client library:
     $ $Env:MLFLOW_TRACKING_URI=<MLFLOW_TRACKING_URI>
     $ $Env:MLFLOW_TRACKING_USERNAME=<MLFLOW_TRACKING_USERNAME>
     $ $Env:MLFLOW_TRACKING_PASSWORD=<MLFLOW_TRACKING_PASSWORD>
-    $ python -c "import mlflow; mlflow.artifacts.download_artifact(artifact_uri='runs:/<MLFLOW_RUN_UUID>/', dst_path='models/<MLFLOW_RUN_UUID')"
+    $ python -c "import mlflow; mlflow.artifacts.download_artifacts(artifact_uri='runs:/<MLFLOW_RUN_UUID>/', dst_path='models/<MLFLOW_RUN_UUID')"
     Downloading artifacts: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00,  1.18it/s]
     ```
 
@@ -110,7 +110,7 @@ following commands:
     $ export MLFLOW_TRACKING_URI=<MLFLOW_TRACKING_URI>
     $ export MLFLOW_TRACKING_USERNAME=<MLFLOW_TRACKING_USERNAME>
     $ export MLFLOW_TRACKING_PASSWORD=<MLFLOW_TRACKING_PASSWORD>
-    $ python -c "import mlflow; mlflow.artifacts.download_artifact(artifact_uri='runs:/$PRED_MODEL_UUID/', dst_path='models/$PRED_MODEL_UUID')"
+    $ python -c "import mlflow; mlflow.artifacts.download_artifacts(artifact_uri='runs:/$PRED_MODEL_UUID/', dst_path='models/$PRED_MODEL_UUID')"
     ```
 
 === "Windows PowerShell"
@@ -120,7 +120,7 @@ following commands:
     $ $Env:MLFLOW_TRACKING_URI=<MLFLOW_TRACKING_URI>
     $ $Env:MLFLOW_TRACKING_USERNAME=<MLFLOW_TRACKING_USERNAME>
     $ $Env:MLFLOW_TRACKING_PASSWORD=<MLFLOW_TRACKING_PASSWORD>
-    $ python -c "import mlflow; mlflow.artifacts.download_artifact(artifact_uri='runs:/$PRED_MODEL_UUID/', dst_path='models/$PRED_MODEL_UUID')"
+    $ python -c "import mlflow; mlflow.artifacts.download_artifacts(artifact_uri='runs:/$PRED_MODEL_UUID/', dst_path='models/$PRED_MODEL_UUID')"
     ```
 
 Executing the commands above will download the artifacts related to the

{{cookiecutter.repo_name}}/docs-requirements.txt

@@ -0,0 +1 @@
+sphinx==7.2.5

{{cookiecutter.repo_name}}/{{cookiecutter.repo_name}}-conda-env.yaml

@@ -23,4 +23,3 @@ dependencies:
     - pydantic==2.5.3
     - pydantic-settings==2.1.0
     - ipykernel==6.25.0
-    - sphinx==7.2.5