refactor(all): update for course september 2024 (#37)

guillaume-chervet · Sep 8, 2024 · 7b782c6 · 7b782c6
1 parent f4b6cb0
commit 7b782c6
Show file tree

Hide file tree

Showing 12 changed files with 208 additions and 79 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -18,6 +18,9 @@ on:
       image_file:
         required: true
         type: string
+      docker_registry:
+        required: true
+        type: string
     secrets:
       DOCKER_USERNAME:
         required: true
@@ -26,13 +29,17 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     environment: MLOpsPython
     steps:
     - uses: actions/checkout@v3
 
     - name: Log in to Docker Hub
       uses: docker/login-action@v2
       with:
+        registry: ${{ inputs.docker_registry }}
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
@@ -50,7 +57,7 @@ jobs:
         file : ${{ inputs.image_file }}
         build-args: ${{ inputs.image_build_args }}
         push: true
-        tags: ${{ inputs.image_name }}:latest,${{ inputs.image_name }}:${{ inputs.image_version }}
+        tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:latest,${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }}
         labels: ${{ steps.meta.outputs.labels }}
 
     - name: Build and push Docker image
@@ -61,6 +68,6 @@ jobs:
         file : ${{ inputs.image_file }}
         build-args: ${{ inputs.image_build_args }}
         push: true
-        tags: ${{ inputs.image_name }}:${{ inputs.image_version }}
+        tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }}
         labels: ${{ steps.meta.outputs.labels }}
 
diff --git a/.github/workflows/docker_with_model.yml b/.github/workflows/docker_with_model.yml
@@ -27,16 +27,27 @@ on:
       workspace_name:
         required: true
         type: string
+      location:
+        required: true
+        type: string
+      docker_registry:
+        required: true
+        type: string
     secrets:
       DOCKER_USERNAME:
         required: true
       DOCKER_PASSWORD:
         required: true
       AZURE_CREDENTIALS:
         required: true
+      AZURE_SUBSCRIPTION_ID:
+        required: true
 jobs:
   build:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     environment: MLOpsPython
     steps:
     - uses: actions/checkout@v3
@@ -64,19 +75,25 @@ jobs:
 
     - name: download model
       run: |
-        az extension add -n ml
         MODEL_VERSION=${{ inputs.model_version }}
         RESOURCE_GROUP=${{ inputs.resource_group }}
         WORKSPACE_NAME=${{ inputs.workspace_name }}
+        AZURE_LOCATION=${{ inputs.location }}
+        DOWNLOAD_PATH=production/api/core/model
+        cd train
+        chmod +x ./run_download_model.sh
+        echo './run_download_model.sh "$MODEL_VERSION" "$RESOURCE_GROUP" "$WORKSPACE_NAME" "$AZURE_LOCATION" "${{ secrets.AZURE_SUBSCRIPTION_ID }}" "$DOWNLOAD_PATH"'
+        ./run_download_model.sh "$MODEL_VERSION" "$RESOURCE_GROUP" "$WORKSPACE_NAME" "$AZURE_LOCATION" "${{ secrets.AZURE_SUBSCRIPTION_ID }}" "$DOWNLOAD_PATH"
         cd production/api/core/model
-        az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group $RESOURCE_GROUP --workspace-name $WORKSPACE_NAME
+        # az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group $RESOURCE_GROUP --workspace-name $WORKSPACE_NAME
         # find files recursively and copy them to the current directory root
         find ./ -name '*.keras' -exec cp "{}" ./  \;
         ls 
         rm -r ./cats-dogs-others
     - name: Log in to Docker Hub
       uses: docker/login-action@v2
       with:
+        registry: ${{ inputs.docker_registry }}
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
@@ -94,7 +111,7 @@ jobs:
         file : ${{ inputs.image_file }}
         build-args: ${{ inputs.image_build_args }}
         push: true
-        tags: ${{ inputs.image_name }}:latest,${{ inputs.image_name }}:${{ inputs.image_version }}
+        tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:latest,${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }}
         labels: ${{ steps.meta.outputs.labels }}
 
     - name: Build and push Docker image
@@ -105,6 +122,6 @@ jobs:
         file : ${{ inputs.image_file }}
         build-args: ${{ inputs.image_build_args }}
         push: true
-        tags: ${{ inputs.image_name }}:${{ inputs.image_version }}
+        tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }}
         labels: ${{ steps.meta.outputs.labels }}
 
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -10,17 +10,20 @@ env:
   PNPM_VERSION: 8.5.1
   PYTHON_VERSION: 3.10.13
   NODE_VERSION: 18
-  AZURE_RESOURCE_GROUP_NAME: "azure-ml-yolw"
+  AZURE_RESOURCE_GROUP_NAME: "azure-ml-gui"
   AZURE_LOCATION: "northeurope"
-  AZURE_ML_WORKSPACE_NAME: "cats-dogs-yolw"
-  AZURE_WEBAPP_NAME: "cats-dogs-yolw"
+  AZURE_ML_WORKSPACE_NAME: "cats-dogs-gui"
+  AZURE_WEBAPP_NAME: "cats-dogs-gui"
   DELETE_WEBAPP: "false"
-  DOCKER_API_IMAGE_NAME: "guillaumechervetlunique/mlopspython-api"
-  DOCKER_WEBAPP_IMAGE_NAME: "guillaumechervetlunique/mlopspython-webapp"
+  DOCKER_API_IMAGE_NAME: "mlopspython-api"
+  DOCKER_REPOSITORY: ${{ github.repository_owner }}
+  DOCKER_WEBAPP_IMAGE_NAME: "mlopspython-webapp"
+  DOCKER_REGISTRY: "ghcr.io"
 
 permissions:
   id-token: write
   contents: write
+  packages: write
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -149,7 +152,9 @@ jobs:
       EXPERIMENT_ID: ${{ steps.train.outputs.EXPERIMENT_ID }}
       AZURE_RESOURCE_GROUP_NAME: ${{ env.AZURE_RESOURCE_GROUP_NAME }}
       AZURE_ML_WORKSPACE_NAME: ${{ env.AZURE_ML_WORKSPACE_NAME }}
-      DOCKER_API_IMAGE_NAME: ${{ env.DOCKER_API_IMAGE_NAME }}
+      DOCKER_API_IMAGE_NAME: ${{ env.DOCKER_REPOSITORY }}/${{ env.DOCKER_API_IMAGE_NAME }}
+      AZURE_LOCATION: ${{ env.AZURE_LOCATION }}
+      DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }}
     steps:
     - uses: actions/checkout@v3
       with:
@@ -185,19 +190,23 @@ jobs:
     - name: download model
       id: train
       run: |
-          az extension add -n ml
           MODEL_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt model_version)
-          echo "MODEL_VERSION=$MODEL_VERSION" >> $GITHUB_OUTPUT
+          echo "MODEL_VERSION=$MODEL_VERSION" >> $GITHUB_OUTPUT          
           INTEGRATION_DATASET_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt integration_dataset_version)
           echo "INTEGRATION_DATASET_VERSION=$INTEGRATION_DATASET_VERSION" >> $GITHUB_OUTPUT
           EXPERIMENT_ID=$(python bin/retrieve_output.py ./train/train_output.txt experiment_id)
           echo "EXPERIMENT_ID=$EXPERIMENT_ID" >> $GITHUB_OUTPUT
-        
           mkdir model
           rm -r model
           mkdir model
           cd model
-          az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME }} --workspace-name ${{ env.AZURE_ML_WORKSPACE_NAME }}
+          DOWNLOAD_PATH=$PWD
+          cd ..
+          cd train
+          chmod +x ./run_download_model.sh
+          ./run_download_model.sh $MODEL_VERSION ${{ env.AZURE_RESOURCE_GROUP_NAME }} ${{ env.AZURE_ML_WORKSPACE_NAME }} ${{ env.AZURE_LOCATION }} ${{ secrets.AZURE_SUBSCRIPTION_ID }} $DOWNLOAD_PATH
+          cd ..
+          cd model
           # find files recursively and copy them to the current directory root
           find ./ -name '*.keras' -exec cp "{}" ./  \;
           find ./ -name '*.json' -exec cp "{}" ./  \;
@@ -248,10 +257,13 @@ jobs:
       model_version: ${{ needs.train.outputs.MODEL_VERSION }}
       resource_group: ${{ needs.train.outputs.AZURE_RESOURCE_GROUP_NAME }}
       workspace_name: ${{ needs.train.outputs.AZURE_ML_WORKSPACE_NAME }}
+      location: ${{ needs.train.outputs.AZURE_LOCATION }}
+      docker_registry: ${{ needs.train.outputs.DOCKER_REGISTRY }}
     secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      DOCKER_USERNAME: ${{ github.actor }}
+      DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
   deploy:
     environment: MLOpsPython
     runs-on: ubuntu-latest
@@ -264,7 +276,7 @@ jobs:
     - name: Deploy container
       run: |
         # https://learn.microsoft.com/en-us/cli/azure/container?view=azure-cli-latest#az-container-create()
-        az container create --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --dns-name-label ${{ env.AZURE_WEBAPP_NAME }} --image docker.io/${{ env.DOCKER_API_IMAGE_NAME }}:${{ needs.tags.outputs.new_version }} --ports 5000
+        az container create --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --dns-name-label ${{ env.AZURE_WEBAPP_NAME }} --image ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_REPOSITORY }}/${{ env.DOCKER_API_IMAGE_NAME }}:${{ needs.tags.outputs.new_version }} --ports 5000
   integration_tests:
     environment: MLOpsPython
     runs-on: ubuntu-latest
@@ -306,7 +318,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: MLOpsPython
     outputs:
-      DOCKER_WEBAPP_IMAGE_NAME: ${{ env.DOCKER_WEBAPP_IMAGE_NAME }}
+      DOCKER_WEBAPP_IMAGE_NAME: ${{ env.DOCKER_REPOSITORY }}/${{ env.DOCKER_WEBAPP_IMAGE_NAME }}
     steps:
     - uses: actions/checkout@v3
     - uses: dorny/paths-filter@v2
@@ -330,17 +342,18 @@ jobs:
         name: Publish webapp
         path: ./production/webapp/build
   build_docker_webapp:
-    needs: [webapp_unit_tests, tags]
+    needs: [webapp_unit_tests, tags, train]
     uses: ./.github/workflows/docker.yml
     with:
       image_name: ${{ needs.webapp_unit_tests.outputs.DOCKER_WEBAPP_IMAGE_NAME }}
       image_version: ${{ needs.tags.outputs.new_version }}
       image_build_args: ""
       image_context: ./production/webapp
       image_file: "./production/webapp/Dockerfile"
+      docker_registry: ${{ needs.train.outputs.DOCKER_REGISTRY }}
     secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      DOCKER_USERNAME: ${{ github.actor }}
+      DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
 
   change_log:
     runs-on: ubuntu-latest
@@ -349,7 +362,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          token: ${{ secrets.GIT_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
       - name: Commit and push
         if: github.ref == 'refs/heads/main'

diff --git a/README.md b/README.md
@@ -20,9 +20,9 @@ git clone https://www.github.com/guillaume-chervet/MLOpsPython
 cd MLOpsPython
 chmod +x Makefile
 # If you have poetry installed
-./Makefile poetry 0.2.0
+./Makefile poetry 0.13.0
 # Else degraded mode
-./Makefile pip 0.2.0
+./Makefile pip 0.13.0
 
 cd production
 docker-compose up

diff --git a/poetry.lock b/poetry.lock
diff --git a/train/azureml_run_download_model.py b/train/azureml_run_download_model.py
@@ -0,0 +1,44 @@
+import argparse
+
+from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential
+
+from azure.ai.ml import MLClient
+
+
+parser = argparse.ArgumentParser("train")
+parser.add_argument("--subscription_id", type=str)
+parser.add_argument("--resource_group_name", type=str)
+parser.add_argument("--workspace_name", type=str)
+parser.add_argument("--location", type=str)
+parser.add_argument("--version", type=str, default="1")
+parser.add_argument("--download_path", type=str, default="./")
+
+args = parser.parse_args()
+subscription_id = args.subscription_id
+resource_group_name = args.resource_group_name
+workspace_name = args.workspace_name
+location = args.location
+version = args.version
+download_path = args.download_path
+
+try:
+    credential = DefaultAzureCredential()
+    # Check if given credential can get token successfully.
+    credential.get_token("https://management.azure.com/.default")
+except Exception as ex:
+    print(ex)
+    # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work
+    credential = InteractiveBrowserCredential()
+
+
+# Get a handle to workspace
+ml_client = MLClient(
+    credential=credential,
+    subscription_id=subscription_id,
+    resource_group_name=resource_group_name,
+    workspace_name=workspace_name,
+)
+
+model_name = 'cats-dogs-others'
+
+ml_client.models.download(name=model_name, version=version, download_path=download_path)