feat: Implement CI/CD for the Streaming Pipeline (iusztinpaul#39)

* feat: Add push & pull streaming pipeline Docker image from ECR logic * feat: Wrap up CI pipeline * feat: Move CI to GitHub Actions * fix: GA file * fix: streaming pipeline CICD * feat: Implement deploy step on GA * refactor: Rename GA files * refactor: Rename GA files * fix: Missing AWS profile * fix: Missing ALPACA_API_SECRET * fix: Key issue * fix: Docker image name issue * feat: Add destory AWS SP GA * chore: Uncomment GA jobs * feat: Move CD only on manual trigger
garimamiet · Nov 16, 2023 · 520a038 · 520a038
1 parent e9f7e7f
commit 520a038
Show file tree

Hide file tree

Showing 16 changed files with 286 additions and 112 deletions.
diff --git a/.github/workflows/cd_streaming_pipeline.yaml b/.github/workflows/cd_streaming_pipeline.yaml
@@ -0,0 +1,87 @@
+name: Continuous Deployment (CD) for the Streaming Pipeline
+
+on: [workflow_dispatch]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REPOSITORY_NAME: streaming_pipeline
+
+jobs:
+  build_and_push:
+    name: Build and Push Docker Image to ECR
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Create ECR Repository
+        id: ecr-repository
+        uses: int128/create-ecr-repository-action@v1
+        with:
+          repository: ${{ env.REPOSITORY_NAME }}
+
+      - name: Build images & push to ECR
+        uses: docker/build-push-action@v4
+        env:
+          COMMIT_TAG: ${{ env.REPOSITORY_NAME }}:commit-${{ github.sha }}
+          LATEST_TAG: ${{ env.REPOSITORY_NAME }}:latest
+        with:
+          context: ./modules/streaming_pipeline
+          file: ./modules/streaming_pipeline/deploy/Dockerfile
+          target: release
+          tags: |
+            ${{ steps.login-ecr.outputs.registry }}/${{ env.COMMIT_TAG }}
+            ${{ steps.login-ecr.outputs.registry }}/${{ env.LATEST_TAG }}
+          push: true
+
+  deploy:
+    name: Deploy & start the Docker image on an AWS EC2 Instance
+    runs-on: ubuntu-latest
+    needs: build_and_push
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Install GNU Make
+      id: install_make
+      run: sudo apt-get update && sudo apt-get install -y make
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_REGION }}
+
+    - name: Remove current deployment
+      id: undeploy_aws
+      working-directory: ./modules/streaming_pipeline
+      run: make undeploy_aws
+
+    - name: Deploy to AWS
+      id: deploy_aws
+      working-directory: ./modules/streaming_pipeline
+      run: make deploy_aws
+      env:
+        ALPACA_API_KEY: ${{ secrets.ALPACA_API_KEY }}
+        ALPACA_API_SECRET: ${{ secrets.ALPACA_API_SECRET }}
+        QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}
+        QDRANT_URL: ${{ secrets.QDRANT_URL }}
+        AWS_REGION: ${{ secrets.AWS_REGION }}
+        AWS_ECR_REPO_NAME: ${{ env.REPOSITORY_NAME }}
diff --git a/.github/workflows/pep8_financial_bot.yaml → .github/workflows/ci_financial_bot.yaml b/.github/workflows/pep8_financial_bot.yaml → .github/workflows/ci_financial_bot.yaml
@@ -1,4 +1,4 @@
-name: pep8_financial_bot
+name: Continuous Integration (CI) for the Financial Bot
 
 on: [push]
 

diff --git a/...ub/workflows/pep8_streaming_pipeline.yaml → .github/workflows/ci_streaming_pipeline.yaml b/...ub/workflows/pep8_streaming_pipeline.yaml → .github/workflows/ci_streaming_pipeline.yaml
@@ -1,4 +1,4 @@
-name: pep8_streaming_pipeline
+name: Continuous Integration (CI) for the Streaming Pipeline
 
 on: [push]
 

diff --git a/...hub/workflows/pep8_training_pipeline.yaml → .github/workflows/ci_training_pipeline.yaml b/...hub/workflows/pep8_training_pipeline.yaml → .github/workflows/ci_training_pipeline.yaml
@@ -1,4 +1,4 @@
-name: pep8_training_pipeline
+name: Continuous Integration (CI) for the Training Pipeline
 
 on: [push]
 

diff --git a/.github/workflows/destroy_aws_streaming_pipeline.yaml b/.github/workflows/destroy_aws_streaming_pipeline.yaml
@@ -0,0 +1,35 @@
+name: Destroy the AWS infrastructure for the Streaming Pipeline
+
+on: [workflow_dispatch]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REPOSITORY_NAME: streaming_pipeline
+
+jobs:
+  deploy:
+    name: Destroy AWS infrastructure for the Streaming Pipeline
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Install GNU Make
+      id: install_make
+      run: sudo apt-get update && sudo apt-get install -y make
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_REGION }}
+
+    - name: Destroy AWS infrastructure
+      id: undeploy_aws
+      working-directory: ./modules/streaming_pipeline
+      run: make undeploy_aws
diff --git a/.gitignore b/.gitignore
@@ -185,3 +185,4 @@ set_env_variables.sh
 logs/
 .DS_Store
 gradio_cached_examples
+trust_policy.json
diff --git a/modules/streaming_pipeline/.env.example b/modules/streaming_pipeline/.env.example
@@ -1,5 +1,9 @@
-ALPACA_API_KEY=<YOUR_ALPACA_API_KEY>
-ALPACA_API_SECRET=<YOUR_ALPACA_API_SECRET>
+export ALPACA_API_KEY=<YOUR_ALPACA_API_KEY>
+export ALPACA_API_SECRET=<YOUR_ALPACA_API_SECRET>
 
-QDRANT_API_KEY=<YOUR_QDRANT_API_KEY>
-QDRANT_URL=<YOUR_QDRANT_URL>
+export QDRANT_API_KEY=<YOUR_QDRANT_API_KEY>
+export QDRANT_URL=<YOUR_QDRANT_URL>
+
+export AWS_ECR_REPO_NAME=streaming_pipeline
+export AWS_REGION=eu-central-1
+export AWS_PROFILE=default
diff --git a/modules/streaming_pipeline/Makefile b/modules/streaming_pipeline/Makefile
@@ -1,7 +1,10 @@
-### Install ###
+# Export env variables from .env file if it exists.
+ifneq (,$(wildcard .env))
+include .env
+export
+endif
 
-install-debian:
-	sudo apt-get install jq
+### Install ###
 
 install:
 	@echo "Installing streaming pipeline..."
@@ -45,9 +48,9 @@ search:
 build:
 	@echo "Build docker image"
 
-	docker build -t streaming_pipeline -f deploy/Dockerfile .
+	docker build -t streaming_pipeline:latest -f deploy/Dockerfile .
 
-run_docker: 
+run_real_time_docker: 
 	@echo "Run docker image"
 
 	docker run --rm \
@@ -57,7 +60,7 @@ run_docker:
 		-e QDRANT_API_KEY=${QDRANT_API_KEY} \
 		-e QDRANT_URL=${QDRANT_URL} \
 		--name streaming_pipeline \
-		streaming_pipeline
+		streaming_pipeline:latest
 
 run_docker_dev:
 	@echo "Run docker image"
@@ -66,7 +69,7 @@ run_docker_dev:
 		--env-file .env \
 		-e DEBUG=true \
 		--name streaming_pipeline \
-		streaming_pipeline
+		streaming_pipeline:latest
 
 
 ### Deploy AWS ###
@@ -84,63 +87,6 @@ undeploy_aws:
 	bash deploy/terminate_ec2.sh
 
 
-### Deploy AWS [Waxctl] ###
-
-NAME := streaming_pipeline_test
-
-# generate a tar file with project files to send to AWS EC2 instance
-deployment-files:
-	if [ -d test ]; then rm -rf test; fi
-
-	tar \
-	--exclude __pycache__ \
-	--exclude .ruff_cache \
-	--exclude logs \
-	--exclude .beamignore \
-	--exclude .env.example \
-	--exclude .env \
-	--exclude Makefile \
-	--exclude *.tar \
-	--exclude .DS_Store \
-	--exclude setup_ec2.sh \
-	--exclude README.md \
-	--exclude requirements.txt \
-	--exclude poetry.lock \
-	--exclude user-data \
-	--exclude tools \
-	--warning=no-file-changed \
-	-cvzf project-files.tar -C . .
-
-	mkdir test; cp project-files.tar test; cd test; tar -xvzf project-files.tar 
-
-deploy-waxctl: deployment-files
-	waxctl aws deploy project-files.tar \
-		--python-file-name streaming_pipeline/run.py \
-		--requirements-file-name requirements.txt \
-		--name ${NAME} \
-		--system-setup-file-name ./setup_ec2.sh \
-		--instance-type t2.small \
-		--region eu-central-1 \
-		--save-cloud-config \
-		--debug \
-		-E ALPACA_API_KEY=${ALPACA_API_KEY},ALPACA_API_SECRET=${ALPACA_API_SECRET},QDRANT_API_KEY=${QDRANT_API_KEY},QDRANT_URL=${QDRANT_URL}
-
-info:
-	waxctl aws ls --verbose --name ${NAME}
-
-undeploy-waxctl:
-	waxctl aws delete --name ${NAME} --yes
-
-clean_aws:
-	# https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_manage_delete.html
-	aws iam remove-role-from-instance-profile --instance-profile-name Waxctl-EC2-eu-central-1-streaming_pipeline-InstanceProfile --role-name Waxctl-EC2-eu-central-1-streaming_pipeline-Role
-	aws iam detach-role-policy --role-name Waxctl-EC2-eu-central-1-streaming_pipeline-Role --policy-arn arn:aws:iam::994231256807:policy/Waxctl-EC2-eu-central-1-streaming_pipeline-Policy
-	aws iam delete-role --role-name Waxctl-EC2-eu-central-1-streaming_pipeline-Role
-	aws iam delete-policy --policy-arn arn:aws:iam::994231256807:policy/Waxctl-EC2-eu-central-1-streaming_pipeline-Policy
-	aws iam delete-instance-profile --instance-profile-name Waxctl-EC2-eu-central-1-streaming_pipeline-InstanceProfile
-
-
-
 ### PEP 8 ###
 # Be sure to install the dev dependencies first #
 

diff --git a/modules/streaming_pipeline/README.md b/modules/streaming_pipeline/README.md
@@ -119,7 +119,7 @@ make build
 
 Run the streaming pipeline in `real-time` mode inside the Docker image:
 ```shell
-source .env && make run_docker
+make run_real_time_docker
 ```
 
 
@@ -143,6 +143,10 @@ To remove the EC2 machine, run:
 make undeploy_aws
 ```
 
+Connect to your EC2 machine and run:
+```
+cat /var/log/cloud-init-output.log
+```
 
 ## 3.4. PEP8 Linting & Formatting
 

diff --git a/modules/streaming_pipeline/deploy/Dockerfile b/modules/streaming_pipeline/deploy/Dockerfile
@@ -1,5 +1,5 @@
 # Use the specified image as the base
-FROM python:3.10-slim-bullseye as base
+FROM python:3.10-slim-bullseye as release
 
 # Set environment variables for Poetry
 ENV POETRY_VERSION=1.6.1

diff --git a/modules/streaming_pipeline/deploy/create_user_data.sh b/modules/streaming_pipeline/deploy/create_user_data.sh
@@ -1,11 +1,27 @@
 #!/bin/bash
 
-# Source the environment variables
-source .env
+# Source the environment variables if .env file exists
+if [ -f ".env" ]; then
+    source .env
+else
+    echo ".env file does not exist."
+fi
+
+# Query the ECR registry URI.
+export DOCKER_IMAGE_ECR_REGISTRY_URI=$(aws ecr describe-repositories --repository-names ${AWS_ECR_REPO_NAME} --query "repositories[?repositoryName==\`${AWS_ECR_REPO_NAME}\`].repositoryUri" --output text --region $AWS_REGION)
+if [ -z "$DOCKER_IMAGE_ECR_REGISTRY_URI" ]; then
+    echo "DOCKER_IMAGE_ECR_REGISTRY_URI is not set. Most probably because the AWS_ECR_REPO_NAME=${AWS_ECR_REPO_NAME} ECR repository does not exist. Exiting script."
+    exit 1
+fi
+
+echo "DOCKER_IMAGE_ECR_REGISTRY_URI=${DOCKER_IMAGE_ECR_REGISTRY_URI}"
 
 # Extract all variables from the template
 variables=$(grep -oP '\$\{\K[^}]*' deploy/user_data_template.sh)
 
+# Define your list of strings
+allowed_variables=("DOCKER_IMAGE_ECR_REGISTRY_URI")
+
 # Flag to indicate if all variables are set
 all_set=true