config.yml

version: '2.1'
orbs:
  node: circleci/node@5.1.0

defaults: &defaults
  docker:
    - image: cimg/python:3.11.1
  working_directory: ~/project

prepare_venv: &prepare_venv
  run:
    name: Create venv
    command: |
      python -m venv venv
      source venv/bin/activate
      pip install --upgrade pip

prepare_tox: &prepare_tox
  run:
    name: Install tox
    command: |
      pip install --user tox

fetch_data: &fetch_data
  run:
    name: Set script permissions and fetch data
    command: |
      source venv/bin/activate
      chmod +x ./scripts/fetch_kaggle_dataset.sh
      ./scripts/fetch_kaggle_dataset.sh

jobs:
  section_07_test_app:
    <<: *defaults
    working_directory: ~/project/section-07-ci-and-publishing/house-prices-api
    steps:
      - checkout:
          path: ~/project
      - *prepare_tox
      - run:
          name: Runnning app tests
          command: |
            tox

  section_07_deploy_app_to_railway:
    <<: *defaults
    steps:
      - checkout:
          path: ~/project/
      - node/install:
          node-version: '16.13'
      - run: node --version
      - run: npm i -g @railway/cli
      - run:
          name: Deploy to Railway App (You must set RAILWAY_TOKEN env var)
          command: |
            cd section-07-ci-and-publishing/house-prices-api && railway up --detach

  section_07_test_and_upload_regression_model:
    <<: *defaults
    working_directory: ~/project/section-07-ci-and-publishing/model-package
    steps:
      - checkout:
          path: ~/project
      - *prepare_tox
      - run:
          name: Fetch the data
          command: |
            tox -e fetch_data
      - run:
          name: Test the model
          command: |
            tox
      - run:
          name: Publish model to Gemfury
          command: |
            tox -e publish_model


  section_08_deploy_app_container_via_railway:
    <<: *defaults
    steps:
      - setup_remote_docker:
          # Supported versions: https://circleci.com/docs/2.0/building-docker-images/#docker-version
          version: 20.10.18
      - checkout:
          path: ~/project/
      - node/install:
          node-version: '16.13'
      - run: node --version
      - run: npm i -g @railway/cli
      - run:
          name: Build and run Dockerfile (see https://docs.railway.app/deploy/dockerfiles)
          command: |
            cd section-08-deploying-with-containers && railway up --detach

  test_regression_model_py37:
    docker:
      - image: circleci/python:3.7.6
    working_directory: ~/project/packages/regression_model
    steps:
      - checkout:
          path: ~/project
      - run:
          name: Run tests with Python 3.7
          command: |
            sudo pip install --upgrade pip
            pip install --user tox
            tox -e py37

  test_regression_model_py38:
    docker:
      - image: circleci/python:3.8.0
    working_directory: ~/project/packages/regression_model
    steps:
      - checkout:
          path: ~/project
      - run:
          name: Run tests with Python 3.8
          command: |
            sudo pip install --upgrade pip
            pip install --user tox
            tox -e py38

  test_ml_api_py37:
    docker:
      - image: circleci/python:3.7.6
    working_directory: ~/project/packages/ml_api
    steps:
      - checkout:
          path: ~/project
      - run:
          name: Run API tests with Python 3.7
          command: |
            sudo pip install --upgrade pip
            pip install --user tox
            tox -e py37

  test_ml_api_py38:
    docker:
      - image: circleci/python:3.8.1
    working_directory: ~/project/packages/ml_api
    steps:
      - checkout:
          path: ~/project
      - run:
          name: Run API tests with Python 3.8
          command: |
            sudo pip install --upgrade pip
            pip install --user tox
            tox -e py38

  train_and_upload_regression_model:
    <<: *defaults
    steps:
      - checkout
      - *prepare_venv
      - run: 
          name: Install requirements
          command: |
            . venv/bin/activate
            pip install -r packages/regression_model/requirements.txt
      - *fetch_data
      - run: 
          name: Train model
          command: |
            . venv/bin/activate
            PYTHONPATH=./packages/regression_model python3 packages/regression_model/regression_model/train_pipeline.py
      - run: 
          name: Publish model to Gemfury
          command: |
            . venv/bin/activate
            chmod +x ./scripts/publish_model.sh
            ./scripts/publish_model.sh ./packages/regression_model/

  section_9_differential_tests:
    <<: *defaults
    steps:
      - checkout
      - *prepare_venv
      - run:
          name: Capturing previous model predictions
          command: |
            . venv/bin/activate
            pip install -r packages/ml_api/diff_test_requirements.txt
            PYTHONPATH=./packages/ml_api python3 packages/ml_api/tests/capture_model_predictions.py
      - run:
          name: Runnning differential tests
          command: |
            . venv/bin/activate
            pip install -r packages/ml_api/requirements.txt
            py.test -vv packages/ml_api/tests -m differential

  section_11_build_and_push_to_heroku_docker:
    <<: *defaults
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run: docker login --username=$HEROKU_EMAIL --password=$HEROKU_API_KEY registry.heroku.com
      - run:
          name: Setup Heroku CLI
          command: |
            wget -qO- https://cli-assets.heroku.com/install-ubuntu.sh | sh
      - run: 
          name: Build and Push Image
          command: |
            make build-ml-api-heroku push-ml-api-heroku
      - run: 
          name: Release to Heroku
          command: |
            heroku container:release web --app $HEROKU_APP_NAME

  section_12_publish_docker_image_to_aws:
    <<: *defaults
    working_directory: ~/project/packages/ml_models
    steps:
      - checkout
      - setup_remote_docker
      - run:
          name: Publishing docker image to aws ECR
          command: |
            sudo pip install awscli
            eval $(aws ecr get-login --no-include-email --region us-east-1)
            make build-ml-api-aws tag-ml-api push-ml-api-aws
            aws ecs update-service --cluster ml-api-cluster --service custom-service --task-definition  first-run-task-definition --force-new-deployment

  section_13_train_and_upload_neural_network_model:
    docker:
      - image: circleci/python:3.6.4-stretch
    working_directory: ~/project
    steps:
      - checkout
      - *prepare_venv
      - run: 
          name: Install requirements
          command: |
            . venv/bin/activate
            pip install -r packages/neural_network_model/requirements.txt
      - run:
          name: Fetch Training data - 2GB
          command: |
            . venv/bin/activate
            chmod +x ./scripts/fetch_kaggle_large_dataset.sh
            ./scripts/fetch_kaggle_large_dataset.sh
      - run: 
          name: Train model
          command: |
            . venv/bin/activate
            PYTHONPATH=./packages/neural_network_model python3 packages/neural_network_model/neural_network_model/train_pipeline.py
      - run: 
          name: Publish model to Gemfury
          command: |
            . venv/bin/activate
            chmod +x ./scripts/publish_model.sh
            ./scripts/publish_model.sh ./packages/neural_network_model/


tags_only: &tags_only
  filters:
    branches:
      ignore: /.*/
    tags:
      only: /^.*/

workflows:
  version: 2
  deploy_pipeline:
    jobs:
      - section_07_test_app
      - section_07_deploy_app_to_railway:
          requires:
            - section_07_test_app
          filters:
            branches:
              only:
                - master
                - demo
      # upload after git tags are created
      - section_07_test_and_upload_regression_model:
          <<: *tags_only

      - section_08_deploy_app_container_via_railway:
         filters:
           branches:
             only:
               - master
               - demo

#  test-all:
#    jobs:
#      - test_regression_model_py36
#      - test_regression_model_py37
#      - test_regression_model_py38
#      - test_ml_api_py36
#      - test_ml_api_py37
#      # - test_ml_api_py38 pending NN model update
#      - section_9_differential_tests
#      - train_and_upload_regression_model:
#          requires:
#            - test_regression_model_py36
#            - test_regression_model_py37
#            - test_regression_model_py38
#            - test_ml_api_py36
#            - test_ml_api_py37
#            - section_9_differential_tests
#          filters:
#            branches:
#              only:
#                - master
#       - section_10_deploy_to_heroku:
#           requires:
#             - train_and_upload_regression_model
#           filters:
#             branches:
#               only:
#                 - master
#      - section_11_build_and_push_to_heroku_docker:
#          requires:
#            - train_and_upload_regression_model
#          filters:
#            branches:
#              only:
#                - master
#       - section_12_publish_docker_image_to_aws:
#           requires:
#             - train_and_upload_regression_model
#           filters:
#             branches:
#               only:
#                 - master
#      - section_13_train_and_upload_neural_network_model:
#          requires:
#            - test_regression_model
#            - test_ml_api
#            - section_9_differential_tests
#             - train_and_upload_regression_model
#           filters:
#             branches:
#               only:
#                 - master