Module 5

.github/workflows/module-1-advanced.yaml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Print pods
         run: |
-          kubectl wait --for=condition=available --timeout=90s deployment/deployments-app-web
+          kubectl wait --for=condition=available --timeout=180s deployment/deployments-app-web
 
       - name: Print pods
         run: |

.github/workflows/module-5.yaml

@@ -1,53 +1,62 @@
 name: Module 5
 
 on:
-  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+  pull_request:
+    branches:
+      - main
+    # paths:
+    #   - 'module-5/**'
 
 jobs:
-  build:
+
+  docker-builds:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
-      - name: Checkout 
-        uses: actions/checkout@v2
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build app streamlit
-        uses: docker/build-push-action@v2
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push app-streamlit
+        uses: docker/build-push-action@v6
         with:
-          context: week-5/
-          file: week-5/Dockerfile
+          context: module-5/
           push: true
           target: app-streamlit
-          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/app-streamlit:latest
-          cache-from: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/app-streamlit:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/app-streamlit:buildcache,mode=max
+          tags: ghcr.io/kyryl-opens-ml/app-streamlit:latest
 
-      - name: Build app fastapi
-        uses: docker/build-push-action@v2
+      - name: Build and push app-fastapi
+        uses: docker/build-push-action@v6
         with:
-          context: week-5/
-          file: week-5/Dockerfile
+          context: module-5/
           push: true
           target: app-fastapi
-          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/app-fastapi:latest
-          cache-from: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/app-fastapi:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/app-fastapi:buildcache,mode=max
+          tags: ghcr.io/kyryl-opens-ml/app-fastapi:latest
+
+      - name: Build and push app-pytriton
+        uses: docker/build-push-action@v6
+        with:
+          context: module-5/
+          push: true
+          target: app-pytriton
+          tags: ghcr.io/kyryl-opens-ml/app-pytriton:latest
 
-      - name: Build app seldon
-        uses: docker/build-push-action@v2
+      - name: Build and push app-kserve
+        uses: docker/build-push-action@v6
         with:
-          context: week-5/
-          file: week-5/Dockerfile
+          context: module-5/
           push: true
-          target: app-seldon
-          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/app-seldon:latest
-          cache-from: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/app-seldon:buildcache
-          cache-to: type=registry,ref=${{ secrets.DOCKER_HUB_USERNAME }}/app-seldon:buildcache,mode=max
+          target: app-kserve
+          tags: ghcr.io/kyryl-opens-ml/app-kserve:latest          

module-4/requirements.txt

@@ -1,3 +1,3 @@
 kfp==2.8.0
-apache-airflow==2.9.3
+apache-airflow==2.10.0
 apache-airflow-providers-cncf-kubernetes==8.3.3

module-5/.gitignore

@@ -0,0 +1 @@
+.lock-file

module-5/Dockerfile

@@ -1,18 +1,14 @@
-# FROM huggingface/transformers-pytorch-gpu:4.22.1 as base
-FROM huggingface/transformers-pytorch-gpu:4.35.2 as base
+FROM python:3.11 as base
 
 WORKDIR /app
 
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-RUN pip install pip --upgrade 
 COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt
 
 
-RUN ln -s /usr/bin/python3 /usr/bin/python
-
 ENV PYTHONPATH /app
 COPY . . 
 
@@ -27,30 +23,9 @@ CMD streamlit run --server.address 0.0.0.0 --server.port 8080 serving/ui_app.py
 FROM base AS app-fastapi
 CMD uvicorn --host 0.0.0.0 --port 8080 --workers 4 serving.fast_api:app 
 
-
-FROM base AS app-seldon
-
-# Port for GRPC
-EXPOSE 5000
-# Port for REST
-EXPOSE 9000
-
-# Define environment variables
-ENV MODEL_NAME SeldonAPI
-ENV SERVICE_TYPE MODEL
-# COPY /app/serving/seldon_api.py /app/SeldonAPI.py
-COPY serving/seldon_api.py /app/SeldonAPI.py
-
-RUN chown -R 8888 /app
-RUN mkdir /.cache
-RUN chmod 777 /.cache
-RUN mkdir /.config
-RUN chmod 777 /.config
-
-CMD exec seldon-core-microservice $MODEL_NAME --service-type $SERVICE_TYPE
+FROM base AS app-pytriton
+CMD python serving/pytriton_serving.py
 
 
 FROM base AS app-kserve
-ENV WANDB_API_KEY=cb86168a2e8db7edb905da69307450f5e7867d66
-RUN pip install protobuf==3.20.*
 ENTRYPOINT ["python", "serving/kserve_api.py"]

module-5/Makefile

@@ -2,7 +2,7 @@ build_all:
 	docker build -f Dockerfile -t all:latest --target app-seldon .    
 
 build_app_streamlit:
-	docker build -f Dockerfile -t app-streamlit:latest --target app-streamlit .  
+	docker build -f Dockerfile -t app-streamlit:latest --target app-streamlit . 
 
 run_app_streamlit: build_app_streamlit
 	docker run -it -p 8081:8080 -e WANDB_API_KEY=${WANDB_API_KEY} app-streamlit:latest
@@ -13,26 +13,24 @@ build_fast_api:
 run_fast_api: build_fast_api
 	docker run -it -p 8081:8080 -e WANDB_API_KEY=${WANDB_API_KEY} app-fastapi:latest
 
-build_app_seldon:
-	docker build -f Dockerfile -t app-seldon:latest --target app-seldon .  
-
-run_app_seldon: build_app_seldon
-	docker run -it -p 8081:8080 -e WANDB_API_KEY=${WANDB_API_KEY} app-seldon:latest
-
-run_dev: build_all
-	docker run -it --net=host -v $PWD:/dev_data -e WANDB_API_KEY=${WANDB_API_KEY} all:latest /bin/bash
+build_pytriton:
+	docker build -f Dockerfile -t app-pytriton:latest --target app-pytriton .  
 
-format:
-	black --line-length 120 serving tests
-	isort -rc serving tests
-
-lint:
-	flake8 --max-line-length 120 serving tests
+run_pytriton: build_pytriton
+	docker run -it -p 8001:8001 -p 8000:8000 -p 8002:8002 -e WANDB_API_KEY=${WANDB_API_KEY} app-pytriton:latest
 
 build_kserve:
 	docker build -f Dockerfile -t app-kserve:latest --target app-kserve .
 
-run_kserve:
+run_kserve: build_kserve
 	docker run -e PORT=8080 -e WANDB_API_KEY=${WANDB_API_KEY} -p 8081:8080 app-kserve:latest 
 
 
+build_app_seldon:
+	docker build -f Dockerfile -t app-seldon:latest --target app-seldon .  
+
+run_app_seldon: build_app_seldon
+	docker run -it -p 8081:8080 -e WANDB_API_KEY=${WANDB_API_KEY} app-seldon:latest
+
+
+

module-5/PRACTICE.md

@@ -1,11 +1,10 @@
-# Practice 
-
-*** 
+# Practice
 
+***
 
 # H9: API serving
 
-## Reading list: 
+## Reading list:
 
 - [CS 329S Lecture 8. Model Deployment](https://docs.google.com/document/d/1hNuW6bqWYZjlwpit_8W1cu7kllb-jTfy3Liof1GJWug/edit#heading=h.kp1fg79091xd)
 - [Machine Learning Systems Design](https://docs.google.com/presentation/d/1U_zKs19VLJKnGE02JDRnzxJ8lgeVF22WSZ_GrA646fY/edit#slide=id.p)
@@ -16,22 +15,20 @@
 - [Gradio Quickstart](https://www.gradio.app/guides/quickstart) 
 - [Top 6 Kubernetes Deployment Strategies and How to Choose](https://codefresh.io/learn/kubernetes-deployment/top-6-kubernetes-deployment-strategies-and-how-to-choose/)
 
-
 ## Task:
 
 - PR1: Write a Streamlit UI for serving your model, with tests and CI integration.
 - PR2: Write a Gradio UI for serving your model, with tests and CI integration.
 - PR3: Write a FastAPI server for your model, with tests and CI integration.
 - PR4: Write a Kubernetes deployment YAML (Deployment, Service) for your model's API.
 - PR5: Write a Kubernetes deployment YAML (Deployment, Service) for your model's UI (Streamlit, Gradio).
-- Google doc update with a model serving plan for your ML model. 
+- Google doc update with a model serving plan for your ML model.
 
-## Criteria: 
+## Criteria:
 
-- 5 PRs merged 
+- 5 PRs merged
 - Serving plan in the google doc.
 
-
 # H10: Inference servers
 
 ## Reading list:
@@ -52,17 +49,15 @@
 
 ## Task:
 
-
 - PR1: Write code for Seldon API deployment of your model, including tests.
 - PR2: Write code for KServe API integration with your model, including tests.
 - PR3: Write code for Triton Inference Server deployment, incorporating tests.
 - PR4: Write code for Ray deployment, complete with tests.
-- PR5: Write code for LLM deployment using TGI, vLLM, and LoRAX.
-- PR6: Write code for LLM deployment with ModalLab.
+- PR5 (optional): Write code for LLM deployment using TGI, vLLM, and LoRAX.
+- PR6 (optional): Write code for LLM deployment with ModalLab.
 - Update the Google document on model serving, outlining options and comparisons between custom servers and inference servers. Decide and explain which solution you will use and why.
 
-
 ## Criteria:
 
-- 6 PRs merged 
-- Serving comparisons and conclusion in the google doc.
+- 6 PRs merged
+- Serving comparisons and conclusion in the google doc.