Training pipeline final tweaks (iusztinpaul#37)

* feat: Fix model added in a different experiment. Dev training setup. * chore: Rename callback to BestModelToModelRegistryCallback * feat: Improve Beam experience * docs: README
ukaserge · Nov 9, 2023 · e9f7e7f · e9f7e7f
1 parent b4a1d0e
commit e9f7e7f
Show file tree

Hide file tree

Showing 18 changed files with 149 additions and 85 deletions.
diff --git a/README.md b/README.md
@@ -59,12 +59,20 @@ Go to [Comet ML](https://www.comet.com/signup?utm_source=thepauls&utm_medium=par
 ### 2.4. Beam
 `cloud compute`
 
-Go to [Beam](https://www.beam.cloud?utm_source=thepauls&utm_medium=partner&utm_content=github) and follow their quick setup/get started tutorial. You must install their CLI and configure your credentials on your local machine.
+Go to [Beam](https://www.beam.cloud?utm_source=thepauls&utm_medium=partner&utm_content=github) and follow their quick setup/get started tutorial. You must create a FREE account, install their CLI and configure your credentials on your local machine.
 
-When using Poetry, we had issues locating the Beam CLI when using the Poetry virtual environment. To fix this, create a symlink using the following command - replace `<your-poetry-env-name>` with your Poetry env name:
+- [Introduction guide](https://docs.beam.cloud/getting-started/introduction?utm_source=thepauls&utm_medium=partner&utm_content=github)
+- [Installation guide](https://docs.beam.cloud/getting-started/installation?utm_source=thepauls&utm_medium=partner&utm_content=github)
+
+#### Troubleshooting
+
+When using Poetry, we had issues locating the Beam CLI when using it inside the Poetry virtual environment. To fix this, after installing Beam, create a symlink that points to Poetry's binaries, as follows:
  ```shell
- export POETRY_ENV_NAME=<your-poetry-env-name>
-  ln -s /usr/local/bin/beam ~/.cache/pypoetry/virtualenvs/${POETRY_ENV_NAME}/bin/beam
+  export COURSE_MODULE_PATH=<your-course-module-path> # e.g., modules/training_pipeline
+  cd $COURSE_MODULE_PATH
+  export POETRY_ENV_PATH=$(dirname $(dirname $(poetry run which python)))
+
+  ln -s /usr/local/bin/beam ${POETRY_ENV_PATH}/bin/beam
  ```
 
 

diff --git a/modules/financial_bot/README.md b/modules/financial_bot/README.md
@@ -100,14 +100,18 @@ Deploy the bot under a RESTful API using Beam in dev mode:
 make deploy_beam_dev
 ```
 
-Make a request to the bot calling the RESTful API:
+To test the deployment, make a request to the bot calling the RESTful API, as follows:
 ```shell
-export BEAM_DEPLOYMENT_ID=<BEAM_DEPLOYMENT_ID>
-export BEAM_AUTH_TOKEN=<BEAM_AUTH_TOKEN>
+export BEAM_DEPLOYMENT_ID=<BEAM_DEPLOYMENT_ID> # e.g., <xxxxx> from https://<xxxxx>.apps.beam.cloud
+export BEAM_AUTH_TOKEN=<BEAM_AUTH_TOKEN> # e.g., <xxxxx> from Authorization: Basic <xxxxx>
 
 make call_restful_api DEPLOYMENT_ID=${BEAM_DEPLOYMENT_ID} TOKEN=${BEAM_AUTH_TOKEN} 
 ```
 
+**Note:** To find out `BEAM_DEPLOYMENT_ID` and `BEAM_AUTH_TOKEN` navigate to your `financial_bot` or `financial_bot_dev` Beam app.
+
+**IMPORTANT:** After you finish testing your project, don't forget to stop your Beam deployment. 
+
 ## 3.3. Gradio UI
 
 Start the Gradio UI:

diff --git a/modules/financial_bot/poetry.lock b/modules/financial_bot/poetry.lock
diff --git a/modules/financial_bot/pyproject.toml b/modules/financial_bot/pyproject.toml
@@ -21,7 +21,7 @@ fire = "^0.5.0"
 comet-llm = "^1.1.0"
 bitsandbytes = "^0.41.1"
 torch = "2.0.1"
-beam-sdk = "0.14.6"
+beam-sdk = "0.15.9"
 gradio = "^3.48.0"
 unstructured = "^0.10.28"
 

diff --git a/modules/financial_bot/requirements.txt b/modules/financial_bot/requirements.txt
@@ -4,16 +4,20 @@ aiohttp==3.8.5 ; python_version >= "3.10" and python_version < "3.12"
 aiosignal==1.3.1 ; python_version >= "3.10" and python_version < "3.12"
 altair==5.1.2 ; python_version >= "3.10" and python_version < "3.12"
 anyio==4.0.0 ; python_version >= "3.10" and python_version < "3.12"
+astor==0.8.1 ; python_version >= "3.10" and python_version < "3.12"
 async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.12"
 attrs==23.1.0 ; python_version >= "3.10" and python_version < "3.12"
-beam-sdk==0.14.6 ; python_version >= "3.10" and python_version < "3.12"
+backoff==2.2.1 ; python_version >= "3.10" and python_version < "3.12"
+beam-sdk==0.15.9 ; python_version >= "3.10" and python_version < "3.12"
+beautifulsoup4==4.12.2 ; python_version >= "3.10" and python_version < "3.12"
 bitsandbytes==0.41.1 ; python_version >= "3.10" and python_version < "3.12"
 certifi==2023.7.22 ; python_version >= "3.10" and python_version < "3.12"
+chardet==5.2.0 ; python_version >= "3.10" and python_version < "3.12"
 charset-normalizer==3.2.0 ; python_version >= "3.10" and python_version < "3.12"
 click==8.1.7 ; python_version >= "3.10" and python_version < "3.12"
 colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.12" and platform_system == "Windows"
 comet-llm==1.3.0 ; python_version >= "3.10" and python_version < "3.12"
-comet-ml==3.33.10 ; python_version >= "3.10" and python_version < "3.12"
+comet-ml==3.35.1 ; python_version >= "3.10" and python_version < "3.12"
 configobj==5.0.8 ; python_version >= "3.10" and python_version < "3.12"
 contourpy==1.1.1 ; python_version >= "3.10" and python_version < "3.12"
 croniter==1.4.1 ; python_version >= "3.10" and python_version < "3.12"
@@ -22,11 +26,13 @@ dataclasses-json==0.5.14 ; python_version >= "3.10" and python_version < "3.12"
 decorator==5.1.1 ; python_version >= "3.10" and python_version < "3.12"
 dulwich==0.21.6 ; python_version >= "3.10" and python_version < "3.12"
 einops==0.6.1 ; python_version >= "3.10" and python_version < "3.12"
+emoji==2.8.0 ; python_version >= "3.10" and python_version < "3.12"
 everett[ini]==3.1.0 ; python_version >= "3.10" and python_version < "3.12"
 exceptiongroup==1.1.3 ; python_version >= "3.10" and python_version < "3.11"
 fastapi==0.103.0 ; python_version >= "3.10" and python_version < "3.12"
 ffmpy==0.3.1 ; python_version >= "3.10" and python_version < "3.12"
 filelock==3.12.3 ; python_version >= "3.10" and python_version < "3.12"
+filetype==1.2.0 ; python_version >= "3.10" and python_version < "3.12"
 fire==0.5.0 ; python_version >= "3.10" and python_version < "3.12"
 flatten-dict==0.4.2 ; python_version >= "3.10" and python_version < "3.12"
 fonttools==4.43.1 ; python_version >= "3.10" and python_version < "3.12"
@@ -49,11 +55,14 @@ idna==3.4 ; python_version >= "3.10" and python_version < "3.12"
 importlib-metadata==5.2.0 ; python_version >= "3.10" and python_version < "3.12"
 importlib-resources==6.1.0 ; python_version >= "3.10" and python_version < "3.12"
 jinja2==3.1.2 ; python_version >= "3.10" and python_version < "3.12"
+joblib==1.3.2 ; python_version >= "3.10" and python_version < "3.12"
 jsonschema-specifications==2023.7.1 ; python_version >= "3.10" and python_version < "3.12"
 jsonschema==4.19.0 ; python_version >= "3.10" and python_version < "3.12"
 kiwisolver==1.4.5 ; python_version >= "3.10" and python_version < "3.12"
 langchain==0.0.285 ; python_version >= "3.10" and python_version < "3.12"
+langdetect==1.0.9 ; python_version >= "3.10" and python_version < "3.12"
 langsmith==0.0.35 ; python_version >= "3.10" and python_version < "3.12"
+lxml==4.9.3 ; python_version >= "3.10" and python_version < "3.12"
 markdown-it-py==3.0.0 ; python_version >= "3.10" and python_version < "3.12"
 markupsafe==2.1.3 ; python_version >= "3.10" and python_version < "3.12"
 marshmallow-dataclass==8.6.0 ; python_version >= "3.10" and python_version < "3.12"
@@ -64,6 +73,7 @@ mpmath==1.3.0 ; python_version >= "3.10" and python_version < "3.12"
 multidict==6.0.4 ; python_version >= "3.10" and python_version < "3.12"
 mypy-extensions==1.0.0 ; python_version >= "3.10" and python_version < "3.12"
 networkx==3.1 ; python_version >= "3.10" and python_version < "3.12"
+nltk==3.8.1 ; python_version >= "3.10" and python_version < "3.12"
 numexpr==2.8.5 ; python_version >= "3.10" and python_version < "3.12"
 numpy==1.25.2 ; python_version >= "3.10" and python_version < "3.12"
 orjson==3.9.9 ; python_version >= "3.10" and python_version < "3.12"
@@ -80,10 +90,13 @@ pyparsing==3.1.1 ; python_version >= "3.10" and python_version < "3.12"
 python-box==6.1.0 ; python_version >= "3.10" and python_version < "3.12"
 python-dateutil==2.8.2 ; python_version >= "3.10" and python_version < "3.12"
 python-dotenv==1.0.0 ; python_version >= "3.10" and python_version < "3.12"
+python-iso639==2023.6.15 ; python_version >= "3.10" and python_version < "3.12"
+python-magic==0.4.27 ; python_version >= "3.10" and python_version < "3.12"
 python-multipart==0.0.6 ; python_version >= "3.10" and python_version < "3.12"
 pytz==2023.3.post1 ; python_version >= "3.10" and python_version < "3.12"
 pyyaml==6.0.1 ; python_version >= "3.10" and python_version < "3.12"
 qdrant-client==1.1.1 ; python_version >= "3.10" and python_version < "3.12"
+rapidfuzz==3.4.0 ; python_version >= "3.10" and python_version < "3.12"
 referencing==0.30.2 ; python_version >= "3.10" and python_version < "3.12"
 regex==2023.8.8 ; python_version >= "3.10" and python_version < "3.12"
 requests-toolbelt==1.0.0 ; python_version >= "3.10" and python_version < "3.12"
@@ -99,9 +112,11 @@ setuptools==68.2.0 ; python_version >= "3.10" and python_version < "3.12"
 simplejson==3.19.1 ; python_version >= "3.10" and python_version < "3.12"
 six==1.16.0 ; python_version >= "3.10" and python_version < "3.12"
 sniffio==1.3.0 ; python_version >= "3.10" and python_version < "3.12"
+soupsieve==2.5 ; python_version >= "3.10" and python_version < "3.12"
 sqlalchemy==2.0.20 ; python_version >= "3.10" and python_version < "3.12"
 starlette==0.27.0 ; python_version >= "3.10" and python_version < "3.12"
 sympy==1.12 ; python_version >= "3.10" and python_version < "3.12"
+tabulate==0.9.0 ; python_version >= "3.10" and python_version < "3.12"
 tenacity==8.2.3 ; python_version >= "3.10" and python_version < "3.12"
 termcolor==2.3.0 ; python_version >= "3.10" and python_version < "3.12"
 tokenizers==0.13.3 ; python_version >= "3.10" and python_version < "3.12"
@@ -116,6 +131,7 @@ types-urllib3==1.26.25.14 ; python_version >= "3.10" and python_version < "3.12"
 typing-extensions==4.7.1 ; python_version >= "3.10" and python_version < "3.12"
 typing-inspect==0.9.0 ; python_version >= "3.10" and python_version < "3.12"
 tzdata==2023.3 ; python_version >= "3.10" and python_version < "3.12"
+unstructured==0.10.28 ; python_version >= "3.10" and python_version < "3.12"
 urllib3==1.26.16 ; python_version >= "3.10" and python_version < "3.12"
 uvicorn==0.23.2 ; python_version >= "3.10" and python_version < "3.12"
 validators==0.20.0 ; python_version >= "3.10" and python_version < "3.12"

diff --git a/modules/training_pipeline/Makefile b/modules/training_pipeline/Makefile
@@ -31,6 +31,8 @@ export_requirements:
 	poetry export -f requirements.txt --output requirements.txt --without-hashes
 
 upload_dataset_to_beam:
+	@echo "Pushing data to the qa_dataset volume on Beam..."
+
 	beam volume upload qa_dataset dataset
 
 
@@ -46,12 +48,12 @@ train_local:
 
 	poetry run python -m tools.train_run --config_file configs/training_config.yaml --output_dir ./output --dataset_dir ./dataset
 
-dev_train_beam:
+dev_train_beam: export_requirements
 	@echo "Running training pipeline on Beam using the development config..."
 
 	BEAM_IGNORE_IMPORTS_OFF=true beam run ./tools/train_run.py:train -d '{"config_file": "configs/dev_training_config.yaml", "output_dir": "./output", "dataset_dir": "./qa_dataset/dataset", "env_file_path": ".env", "model_cache_dir": "./model_cache"}'
 
-train_beam:
+train_beam: export_requirements
 	@echo "Running training pipeline on Beam using the production config..."
 
 	BEAM_IGNORE_IMPORTS_OFF=true beam run ./tools/train_run.py:train -d '{"config_file": "configs/training_config.yaml", "output_dir": "./output", "dataset_dir": "./qa_dataset/dataset", "env_file_path": ".env", "model_cache_dir": "./model_cache"}'
@@ -69,15 +71,15 @@ infer_local:
 
 	poetry run python -m tools.inference_run --config_file configs/inference_config.yaml --dataset_dir ./dataset
 
-dev_infer_beam:
+dev_infer_beam: export_requirements
 	@echo "Running inference pipeline on Beam using the development config..."
 
-	BEAM_IGNORE_IMPORTS_OFF=true beam run ./tools/inference_run.py:infer -d '{"config_file": "configs/dev_inference_config.yaml", "dataset_dir": "./dataset/dataset", "env_file_path": "env", "model_cache_dir": "./model_cache"}'
+	BEAM_IGNORE_IMPORTS_OFF=true beam run ./tools/inference_run.py:infer -d '{"config_file": "configs/dev_inference_config.yaml", "dataset_dir": "./qa_dataset/dataset", "env_file_path": ".env", "model_cache_dir": "./model_cache"}'
 
-infer_beam:
+infer_beam: export_requirements
 	@echo "Running inference pipeline on Beam using the production config..."
 
-	BEAM_IGNORE_IMPORTS_OFF=true beam run ./tools/inference_run.py:infer -d '{"config_file": "configs/inference_config.yaml", "dataset_dir": "./dataset/dataset", "env_file_path": "env", "model_cache_dir": "./model_cache"}'
+	BEAM_IGNORE_IMPORTS_OFF=true beam run ./tools/inference_run.py:infer -d '{"config_file": "configs/inference_config.yaml", "dataset_dir": "./qa_dataset/dataset", "env_file_path": ".env", "model_cache_dir": "./model_cache"}'
 
 
 # === PEP8 ===

diff --git a/modules/training_pipeline/README.md b/modules/training_pipeline/README.md
@@ -51,12 +51,18 @@ cp .env.example .env
 ## 2.2. Beam
 `optional step in case you want to use Beam` 
 
--> [Create a Beam account & configure it.](https://www.beam.cloud?utm_source=thepauls&utm_medium=partner&utm_content=github)
+-> [Create a Beam account, install its CLI, and configure it.](https://www.beam.cloud?utm_source=thepauls&utm_medium=partner&utm_content=github)
 
-After you have to upload the dataset to a Beam volume:
+In addition to setting up Beam, you have to go to your [Beam account](https://www.beam.cloud?utm_source=thepauls&utm_medium=partner&utm_content=github) and create a volume, as follows:
+- go to the `Volumes` section
+- click create `New Volume` (in the top right corner)
+- choose `Volume Name = qa_dataset` and `Volume Type = Shared`
+
+After, run the following command to upload the Q&A dataset to the Beam volume you just created.
 ```shell
 make upload_dataset_to_beam
 ```
+Finally, check out that your [**qa_dataset** Beam volume](https://www.beam.cloud/dashboard/volumes/qa_dataset?utm_source=thepauls&utm_medium=partner&utm_content=github) contains the uploaded data. 
 
 # 3. Usage
 

diff --git a/modules/training_pipeline/configs/dev_inference_config.yaml b/modules/training_pipeline/configs/dev_inference_config.yaml
@@ -1,5 +1,5 @@
 peft_model:
-  id: iusztinpaul/fin-falcon-7b-lora:0.0.5
+  id: iusztinpaul/fin-falcon-7b-lora:1.0.5
 model:
   id: tiiuae/falcon-7b-instruct
   template_name: "falcon"

diff --git a/modules/training_pipeline/configs/dev_training_config.yaml b/modules/training_pipeline/configs/dev_training_config.yaml
@@ -1,25 +1,23 @@
 training:
   per_device_train_batch_size: 1
-  gradient_accumulation_steps: 4
+  gradient_accumulation_steps: 2
   per_device_eval_batch_size: 1
   eval_accumulation_steps: 1
   optim: paged_adamw_32bit
-  save_steps: 1
-  logging_steps: 1
+  save_steps: 2
+  logging_steps: 2
   learning_rate: 2.e-4
   fp16: True
   max_grad_norm: 0.3
   num_train_epochs: 1
   warmup_ratio: 0.03
   lr_scheduler_type: constant
   evaluation_strategy: steps
-  eval_steps: 1
+  eval_steps: 2
   report_to: comet_ml
   seed: 42
   load_best_model_at_end: True
 model:
   id: tiiuae/falcon-7b-instruct
   template: falcon
   max_seq_length: 1024
-setup:
-  debug: True
diff --git a/modules/training_pipeline/configs/inference_config.yaml b/modules/training_pipeline/configs/inference_config.yaml
@@ -1,5 +1,5 @@
 peft_model:
-  id: iusztinpaul/fin-falcon-7b-lora:0.0.5
+  id: iusztinpaul/fin-falcon-7b-lora:1.0.5
 model:
   id: tiiuae/falcon-7b-instruct
   template_name: "falcon"

diff --git a/modules/training_pipeline/configs/training_config.yaml b/modules/training_pipeline/configs/training_config.yaml
@@ -21,5 +21,3 @@ model:
   id: tiiuae/falcon-7b-instruct
   template: falcon
   max_seq_length: 1024
-setup:
-  debug: False
diff --git a/modules/training_pipeline/poetry.lock b/modules/training_pipeline/poetry.lock
diff --git a/modules/training_pipeline/pyproject.toml b/modules/training_pipeline/pyproject.toml
@@ -18,7 +18,7 @@ comet-ml = "3.35.1"
 python-dotenv = "^1.0.0"
 scipy = "^1.11.1"
 einops = "^0.6.1"
-beam-sdk = "^0.14.6"
+beam-sdk = "0.15.9"
 psutil = "^5.9.5"
 torch = "2.0.1"
 fire = "^0.5.0"