Update 2020 part 1

.circleci/config.yml

@@ -13,6 +13,13 @@ prepare_venv: &prepare_venv
       source venv/bin/activate
       pip install --upgrade pip
 
+prepare_tox: &prepare_tox
+  run:
+    name: Install tox
+    command: |
+      sudo pip install --upgrade pip
+      pip install --user tox
+
 fetch_data: &fetch_data
   run:
     name: Set script permissions and fetch data
@@ -22,47 +29,89 @@ fetch_data: &fetch_data
       ./scripts/fetch_kaggle_dataset.sh
 
 jobs:
-  test_regression_model:
-    <<: *defaults
+  test_regression_model_py36:
+    docker:
+      - image: circleci/python:3.6.9
+    working_directory: ~/project/packages/regression_model
     steps:
-      - checkout
-      - *prepare_venv
+      - checkout:
+          path: ~/project
       - run:
-          name: Install requirements
+          name: Run tests with Python 3.6
           command: |
-            . venv/bin/activate
-            pip install -r packages/regression_model/requirements.txt
-      - *fetch_data
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py36
+
+  test_regression_model_py37:
+    docker:
+      - image: circleci/python:3.7.6
+    working_directory: ~/project/packages/regression_model
+    steps:
+      - checkout:
+          path: ~/project
       - run:
-          name: Train model
+          name: Run tests with Python 3.7
           command: |
-            . venv/bin/activate
-            PYTHONPATH=./packages/regression_model python3 packages/regression_model/regression_model/train_pipeline.py
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py37
+
+  test_regression_model_py38:
+    docker:
+      - image: circleci/python:3.8.0
+    working_directory: ~/project/packages/regression_model
+    steps:
+      - checkout:
+          path: ~/project
       - run:
-          name: Run tests
+          name: Run tests with Python 3.8
           command: |
-            . venv/bin/activate
-            py.test -vv packages/regression_model/tests
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py38
 
-  test_ml_api:
-    <<: *defaults
+  test_ml_api_py36:
+    docker:
+      - image: circleci/python:3.6.9
+    working_directory: ~/project/packages/ml_api
     steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - py-deps-{{ checksum "packages/ml_api/requirements.txt" }}
+      - checkout:
+          path: ~/project
       - run:
-          name: Runnning tests
+          name: Run API tests with Python 3.6
           command: |
-            python3 -m venv venv
-            . venv/bin/activate
-            pip install --upgrade pip
-            pip install -r packages/ml_api/requirements.txt
-            py.test -vv packages/ml_api/tests -m "not differential"
-      - save_cache:
-          key: py-deps-{{ checksum "packages/ml_api/requirements.txt" }}
-          paths:
-            - "/venv"
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py36
+
+  test_ml_api_py37:
+    docker:
+      - image: circleci/python:3.7.6
+    working_directory: ~/project/packages/ml_api
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run API tests with Python 3.7
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py37
+
+  test_ml_api_py38:
+    docker:
+      - image: circleci/python:3.8.1
+    working_directory: ~/project/packages/ml_api
+    steps:
+      - checkout:
+          path: ~/project
+      - run:
+          name: Run API tests with Python 3.8
+          command: |
+            sudo pip install --upgrade pip
+            pip install --user tox
+            tox -e py38
 
   train_and_upload_regression_model:
     <<: *defaults
@@ -182,13 +231,20 @@ workflows:
   version: 2
   test-all:
     jobs:
-      - test_regression_model
-      - test_ml_api
+      - test_regression_model_py36
+      - test_regression_model_py37
+      - test_regression_model_py38
+      - test_ml_api_py36
+      - test_ml_api_py37
+      # - test_ml_api_py38 pending NN model update
       - section_9_differential_tests
       - train_and_upload_regression_model:
           requires:
-            - test_regression_model
-            - test_ml_api
+            - test_regression_model_py36
+            - test_regression_model_py37
+            - test_regression_model_py38
+            - test_ml_api_py36
+            - test_ml_api_py37
             - section_9_differential_tests
           filters:
             branches:
@@ -201,25 +257,25 @@ workflows:
       #       branches:
       #         only:
       #           - master
-      - section_11_build_and_push_to_heroku_docker:
-          requires:
-            - train_and_upload_regression_model
-          filters:
-            branches:
-              only:
-                - master
+#      - section_11_build_and_push_to_heroku_docker:
+#          requires:
+#            - train_and_upload_regression_model
+#          filters:
+#            branches:
+#              only:
+#                - master
       # - section_12_publish_docker_image_to_aws:
       #     requires:
       #       - train_and_upload_regression_model
       #     filters:
       #       branches:
       #         only:
       #           - master
-      - section_13_train_and_upload_neural_network_model:
-          requires:
-            - test_regression_model
-            - test_ml_api
-            - section_9_differential_tests
+#      - section_13_train_and_upload_neural_network_model:
+#          requires:
+#            - test_regression_model
+#            - test_ml_api
+#            - section_9_differential_tests
             # - train_and_upload_regression_model
           # filters:
           #   branches:

.gitignore

@@ -112,6 +112,9 @@ packages/regression_model/regression_model/datasets/*.zip
 packages/regression_model/regression_model/datasets/*.txt
 train.csv
 test.csv
+data_description.txt
+house-prices-advanced-regression-techniques.zip
+sample_submission.csv
 test_data_predictions.csv
 v2-plant-seedlings-dataset/
 v2-plant-seedlings-dataset.zip

packages/ml_api/VERSION

@@ -1 +1 @@
-0.2.1
+0.3.0

packages/ml_api/diff_test_requirements.txt

@@ -1,10 +1,13 @@
 --extra-index-url=${PIP_EXTRA_INDEX_URL}
 
 # api
-flask==1.0.2
+flask>=1.1.1,<1.2.0
 
 # schema validation
 marshmallow==2.17.0
 
 # Set this to the previous model version
-regression-model==0.1.0
+regression-model==2.0.19
+
+# temporarily necessary as we update sklearn
+joblib>=0.14.1,<0.15.0

packages/ml_api/requirements.txt

@@ -1,13 +1,13 @@
 --extra-index-url=${PIP_EXTRA_INDEX_URL}
 
 # api
-flask==1.0.2
+flask>=1.1.1,<1.2.0
 
 # schema validation
 marshmallow==2.17.0
 
 # Install from gemfury
-regression-model==1.0.0
+regression-model==2.0.20
 neural_network_model==0.1.1
 
 # Deployment

packages/ml_api/tox.ini

@@ -0,0 +1,32 @@
+[tox]
+envlist = py36, py37, py38
+skipsdist = True
+
+
+[testenv]
+install_command = pip install --pre {opts} {packages}
+deps =
+    -rrequirements.txt
+
+passenv =
+      PIP_EXTRA_INDEX_URL
+      KERAS_BACKEND
+
+setenv =
+  PYTHONPATH=.
+
+commands =
+     pytest \
+           -s \
+           -v \
+           -m "not differential" \
+           {posargs:tests}
+
+
+# content of pytest.ini
+[pytest]
+markers =
+    integration: mark a test as an integration test.
+	differential: mark a test as a differential test.
+filterwarnings =
+    ignore::DeprecationWarning

packages/regression_model/regression_model/VERSION

@@ -1 +1 @@
-1.0.0
+2.0.20

packages/regression_model/setup.py

@@ -10,14 +10,14 @@
 
 # Package meta-data.
 NAME = 'regression_model'
-DESCRIPTION = 'Train and deploy regression model.'
-URL = 'your github project'
-EMAIL = 'your_email@email.com'
-AUTHOR = 'Your name'
+DESCRIPTION = 'Regression model for using in the Train In Data online course "Deployment of Machine Learning Models".'
+URL = 'https://github.com/trainindata/deploying-machine-learning-models'
+EMAIL = 'christopher.samiullah@protonmail.com'
+AUTHOR = 'ChristopherGS'
 REQUIRES_PYTHON = '>=3.6.0'
 
 
-# What packages are required for this module to be executed?
+# Packages that are required for this module to be executed
 def list_reqs(fname='requirements.txt'):
     with open(fname) as fd:
         return fd.read().splitlines()
@@ -42,7 +42,7 @@ def list_reqs(fname='requirements.txt'):
 
 # Load the package's __version__.py module as a dictionary.
 ROOT_DIR = Path(__file__).resolve().parent
-PACKAGE_DIR = ROOT_DIR / NAME
+PACKAGE_DIR = ROOT_DIR / 'regression_model'
 about = {}
 with open(PACKAGE_DIR / 'VERSION') as f:
     _version = f.read().strip()
@@ -65,7 +65,7 @@ def list_reqs(fname='requirements.txt'):
     install_requires=list_reqs(),
     extras_require={},
     include_package_data=True,
-    license='MIT',
+    license='BSD 3',
     classifiers=[
         # Trove classifiers
         # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers

packages/regression_model/tox.ini

@@ -1,35 +1,25 @@
-# Tox is a generic virtualenv management and test command line tool. Its goal is to
-# standardize testing in Python. We will be using it extensively in this course.
-
-# Using Tox we can (on multiple operating systems):
-# + Eliminate PYTHONPATH challenges when running scripts/tests
-# + Eliminate virtualenv setup confusion
-# + Streamline steps such as model training, model publishing
-
 [tox]
-envlist = regression_model
-skipsdist = True
+envlist = py36, py37, py38
+
 
 [testenv]
-install_command = pip install {opts} {packages}
+install_command = pip install --pre {opts} {packages}
+whitelist_externals = unzip
 deps =
-	-rrequirements.txt
-
-setenv =
-  	PYTHONPATH=.
+    -rrequirements.txt
 
-commands =
-	python regression_model/train_pipeline.py
-	pytest tests/
-
-
-[testenv:install_locally]
-deps =
-	{[testenv]deps}
+passenv =
+      KAGGLE_USERNAME
+      KAGGLE_KEY
 
 setenv =
-  	PYTHONPATH=.
+  PYTHONPATH=.
 
 commands =
-	python regression_model/train_pipeline.py
-	python setup.py sdist bdist_wheel
+     kaggle competitions download -c house-prices-advanced-regression-techniques -p regression_model/datasets/
+     unzip -o regression_model/datasets/house-prices-advanced-regression-techniques.zip -d regression_model/datasets
+     python regression_model/train_pipeline.py
+     pytest \
+           -s \
+           -v \
+           {posargs:tests}