feat: Implement deploy step on GA

.github/workflows/release_streaming_pipeline.yaml

@@ -0,0 +1,92 @@
+name: Release Streaming Pipeline
+
+on: [push, workflow_dispatch] # Or any other event that triggers this workflow
+# on:
+#   push:
+#     branches:
+#       - main
+#   workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  REPOSITORY_NAME: streaming_pipeline
+
+jobs:
+  build_and_push:
+    name: Build and Push Docker Image to ECR
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Create ECR Repository
+        id: ecr-repository
+        uses: int128/create-ecr-repository-action@v1
+        with:
+          repository: ${{ env.REPOSITORY_NAME }}
+
+      - name: Build images & push to ECR
+        uses: docker/build-push-action@v4
+        env:
+          COMMIT_TAG: ${{ env.REPOSITORY_NAME }}:commit-${{ github.sha }}
+          LATEST_TAG: ${{ env.REPOSITORY_NAME }}:latest
+        with:
+          context: ./modules/streaming_pipeline
+          file: ./modules/streaming_pipeline/deploy/Dockerfile
+          target: release
+          tags: |
+            ${{ steps.login-ecr.outputs.registry }}/${{ env.COMMIT_TAG }}
+            ${{ steps.login-ecr.outputs.registry }}/${{ env.LATEST_TAG }}
+          push: true
+
+  deploy:
+    name: Deploy & start the Docker image on an AWS EC2 Instance
+    runs-on: ubuntu-latest
+    needs: build_and_push
+
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+
+    - name: Install GNU Make
+      id: install_make
+      run: sudo apt-get update && sudo apt-get install -y make
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ env.AWS_REGION }}
+
+    - name: Remove current deployment
+      id: undeploy_aws
+      working-directory: ./modules/streaming_pipeline
+      run: make undeploy_aws
+
+    - name: Deploy to AWS
+      id: deploy_aws
+      working-directory: ./modules/streaming_pipeline
+      run: make deploy_aws
+      env:
+        ALPACA_API_KEY: ${{ secrets.ALPACA_API_KEY }}
+        ALPACA_SECRET_KEY: ${{ secrets.ALPACA_SECRET_KEY }}
+        QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}
+        QDRANT_URL: ${{ secrets.QDRANT_URL }}
+        AWS_REGION: ${{ secrets.AWS_REGION }}
+        AWS_ECR_REPO_NAME: ${{ env.REPOSITORY_NAME }}

modules/streaming_pipeline/Makefile

@@ -90,63 +90,6 @@ undeploy_aws:
 	bash deploy/terminate_ec2.sh
 
 
-### Deploy AWS [Waxctl] ###
-
-NAME := streaming_pipeline_test
-
-# generate a tar file with project files to send to AWS EC2 instance
-deployment-files:
-	if [ -d test ]; then rm -rf test; fi
-
-	tar \
-	--exclude __pycache__ \
-	--exclude .ruff_cache \
-	--exclude logs \
-	--exclude .beamignore \
-	--exclude .env.example \
-	--exclude .env \
-	--exclude Makefile \
-	--exclude *.tar \
-	--exclude .DS_Store \
-	--exclude setup_ec2.sh \
-	--exclude README.md \
-	--exclude requirements.txt \
-	--exclude poetry.lock \
-	--exclude user-data \
-	--exclude tools \
-	--warning=no-file-changed \
-	-cvzf project-files.tar -C . .
-
-	mkdir test; cp project-files.tar test; cd test; tar -xvzf project-files.tar 
-
-deploy-waxctl: deployment-files
-	waxctl aws deploy project-files.tar \
-		--python-file-name streaming_pipeline/run.py \
-		--requirements-file-name requirements.txt \
-		--name ${NAME} \
-		--system-setup-file-name ./setup_ec2.sh \
-		--instance-type t2.small \
-		--region eu-central-1 \
-		--save-cloud-config \
-		--debug \
-		-E ALPACA_API_KEY=${ALPACA_API_KEY},ALPACA_API_SECRET=${ALPACA_API_SECRET},QDRANT_API_KEY=${QDRANT_API_KEY},QDRANT_URL=${QDRANT_URL}
-
-info:
-	waxctl aws ls --verbose --name ${NAME}
-
-undeploy-waxctl:
-	waxctl aws delete --name ${NAME} --yes
-
-clean_aws:
-	# https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_manage_delete.html
-	aws iam remove-role-from-instance-profile --instance-profile-name Waxctl-EC2-eu-central-1-streaming_pipeline-InstanceProfile --role-name Waxctl-EC2-eu-central-1-streaming_pipeline-Role
-	aws iam detach-role-policy --role-name Waxctl-EC2-eu-central-1-streaming_pipeline-Role --policy-arn arn:aws:iam::994231256807:policy/Waxctl-EC2-eu-central-1-streaming_pipeline-Policy
-	aws iam delete-role --role-name Waxctl-EC2-eu-central-1-streaming_pipeline-Role
-	aws iam delete-policy --policy-arn arn:aws:iam::994231256807:policy/Waxctl-EC2-eu-central-1-streaming_pipeline-Policy
-	aws iam delete-instance-profile --instance-profile-name Waxctl-EC2-eu-central-1-streaming_pipeline-InstanceProfile
-
-
-
 ### PEP 8 ###
 # Be sure to install the dev dependencies first #
 

modules/streaming_pipeline/deploy/create_user_data.sh

@@ -1,10 +1,18 @@
 #!/bin/bash
 
-# Source the environment variables
-source .env
+# Source the environment variables if .env file exists
+if [ -f ".env" ]; then
+    source .env
+else
+    echo ".env file does not exist."
+fi
 
 # Query the ECR registry URI.
 export ECR_REGISTRY_URI=$(aws ecr describe-repositories --repository-names ${AWS_ECR_REPO_NAME} --query "repositories[?repositoryName==\`${AWS_ECR_REPO_NAME}\`].repositoryUri" --output text --profile $AWS_PROFILE --region $AWS_REGION)
+if [ -z "$ECR_REGISTRY_URI" ]; then
+    echo "ECR_REGISTRY_URI is not set. Most probably because the AWS_ECR_REPO_NAME=${AWS_ECR_REPO_NAME} ECR repository does not exist. Exiting script."
+    exit 1
+fi
 
 # Extract all variables from the template
 variables=$(grep -oP '\$\{\K[^}]*' deploy/user_data_template.sh)

modules/streaming_pipeline/deploy/user_data_template.sh

@@ -31,7 +31,7 @@ echo "Sleeping for 60 seconds to allow the instance to fully initialize..."
 sleep 60
 
 # Authenticate Docker to the ECR registry.
-aws ecr get-login-password --region "eu-central-1" | docker login --username AWS --password-stdin "994231256807.dkr.ecr.eu-central-1.amazonaws.com"
+aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${ECR_REGISTRY_URI}
 
 # Pull Docker image from ECR.
 echo "Pulling Docker image from ECR: ${ECR_REGISTRY_URI}/${AWS_ECR_REPO_NAME}:latest"