Skip to content
Python Continuous Integration

refactor(all): update for course september 2024 #97

Sign in to view logs

refactor(all): update for course september 2024

refactor(all): update for course september 2024 #97

Workflow file for this run

.github/workflows/main.yml at dca0950
name: Python Continuous Integration
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
env:
PNPM_VERSION: 8.5.1
PYTHON_VERSION: 3.10.13
NODE_VERSION: 18
AZURE_RESOURCE_GROUP_NAME: "azure-ml-gui"
AZURE_LOCATION: "northeurope"
AZURE_ML_WORKSPACE_NAME: "cats-dogs-gui"
AZURE_WEBAPP_NAME: "cats-dogs-gui"
DELETE_WEBAPP: "false"
DOCKER_API_IMAGE_NAME: "guillaumechervetlunique/mlopspython-api"
DOCKER_WEBAPP_IMAGE_NAME: "guillaumechervetlunique/mlopspython-webapp"
permissions:
id-token: write
contents: write
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter_python
with:
filters: |
packages:
- 'packages/**'
production:
- 'production/**'
train:
- 'train/**'
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install Poetry dependencies
run: |
python -m pip install --upgrade pip
pip install --user poetry
- name: Format with Black
run: |
poetry install
poetry run black train
poetry run black train --check
- name: Lint with flake8
run: |
poetry install
poetry run flake8 .
packages_unit_tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install --user poetry
- name: Run unit tests Packages Extraction
working-directory: packages/extraction
run: |
poetry install
poetry run coverage run -m unittest tests.extraction_tests
poetry run coverage report
- name: Run unit tests Packages Inference
working-directory: packages/inference
run: |
poetry install
#poetry run coverage run -m unittest tests.inference_tests
#poetry run coverage report
train_unit_tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install --user poetry
- name: Build packages
run: |
chmod +x ./Makefile
./Makefile
- name: Run unit tests Extraction
working-directory: train/extraction
run: |
poetry lock
poetry install
#poetry run coverage run -m unittest tests.extraction_test
#poetry run coverage report
- name: Run unit tests Label Split Data
working-directory: train/label_split_data
run: |
poetry lock
poetry install
poetry run coverage run -m unittest tests.label_split_data_test
poetry run coverage report
#- name: Run unit tests Train
# working-directory: train/train
# run: |
# poetry install --dev
# poetry run python -m unittest tests.train_test
#- name: Run unit tests Evaluate
# working-directory: train/test
# run: |
# poetry install --dev
# poetry run python -m unittest tests.evaluate_test
tags:
runs-on: ubuntu-latest
needs: [train_unit_tests, packages_unit_tests]
outputs:
new_version: ${{ steps.tag.outputs.new_version }}
steps:
- uses: actions/checkout@v3
- name: Bump version and push tag
id: tag_version
if: github.ref == 'refs/heads/main'
uses: mathieudutour/github-tag-action@v6.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Add tag to output step for main branch
id: tag
run: |
if [ '${{ github.ref }}' = 'refs/heads/main' ]; then
echo "new_version=${{ steps.tag_version.outputs.new_version }}" >> $GITHUB_OUTPUT
else
echo "new_version=pr-${{ github.event.number }}-${{ github.run_number }}" >> $GITHUB_OUTPUT
fi
train:
runs-on: ubuntu-latest
environment: MLOpsPython
needs: tags
outputs:
MODEL_VERSION: ${{ steps.train.outputs.MODEL_VERSION }}
INTEGRATION_DATASET_VERSION: ${{ steps.train.outputs.INTEGRATION_DATASET_VERSION }}
EXPERIMENT_ID: ${{ steps.train.outputs.EXPERIMENT_ID }}
AZURE_RESOURCE_GROUP_NAME: ${{ env.AZURE_RESOURCE_GROUP_NAME }}
AZURE_ML_WORKSPACE_NAME: ${{ env.AZURE_ML_WORKSPACE_NAME }}
DOCKER_API_IMAGE_NAME: ${{ env.DOCKER_API_IMAGE_NAME }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install --user poetry
- name: Build packages
run: |
chmod +x ./Makefile
./Makefile
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
- name: Run Train Pipeline
run: |
chmod +x ./setup_AzureML.sh
./setup_AzureML.sh ${{ secrets.AZURE_SUBSCRIPTION_ID }} ${{ env.AZURE_RESOURCE_GROUP_NAME }} ${{ env.AZURE_ML_WORKSPACE_NAME }} ${{ env.AZURE_LOCATION }}
chmod +x ./run_AzureML.sh
./run_AzureML.sh ${{ secrets.AZURE_SUBSCRIPTION_ID }} \
${{ env.AZURE_RESOURCE_GROUP_NAME }} \
${{ env.AZURE_ML_WORKSPACE_NAME }} \
${{ env.AZURE_LOCATION }} \
"{\"git_head_ref\":\"${{ github.head_ref }}\",\"git\":\"${{ github.head_ref }}.${{ github.sha }}\",\"version\":\"${{ needs.tags.outputs.new_version }}\",\"triggering_actor\":\"${{github.triggering_actor}}\"}" > train_output.txt
cat train_output.txt
working-directory: train
- name: download model
id: train
run: |
az extension add -n ml
echo "here"
MODEL_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt model_version)
echo "here1"
echo "MODEL_VERSION=$MODEL_VERSION" >> $GITHUB_OUTPUT
INTEGRATION_DATASET_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt integration_dataset_version)
echo "here2"
echo "INTEGRATION_DATASET_VERSION=$INTEGRATION_DATASET_VERSION" >> $GITHUB_OUTPUT
EXPERIMENT_ID=$(python bin/retrieve_output.py ./train/train_output.txt experiment_id)
echo "EXPERIMENT_ID=$EXPERIMENT_ID" >> $GITHUB_OUTPUT
mkdir model
rm -r model
mkdir model
cd model
echo "here3"
az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME }} --workspace-name ${{ env.AZURE_ML_WORKSPACE_NAME }}
echo "here4"
# find files recursively and copy them to the current directory root
find ./ -name '*.keras' -exec cp "{}" ./ \;
find ./ -name '*.json' -exec cp "{}" ./ \;
rm -r ./cats-dogs-others
- name: Upload Model Build Artifact
uses: actions/upload-artifact@v2
with:
name: Publish model
path: ./model
- name: release
uses: actions/create-release@v1
id: create_release
if: github.ref == 'refs/heads/main'
with:
draft: false
prerelease: false
body: |
${{ needs.tags.outputs.changelog }}
release_name: "v${{ needs.tags.outputs.new_version }}"
tag_name: "v${{ needs.tags.outputs.new_version }}"
env:
GITHUB_TOKEN: ${{ github.token }}
- name: Zip Release
uses: TheDoctor0/zip-release@0.6.0
with:
filename: mlopspython_model.zip
path: .
directory: ./model
- name: upload artifact
if: github.ref == 'refs/heads/main'
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ github.token }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./model/mlopspython_model.zip
asset_name: mlopspython_model.zip
asset_content_type: application/zip
build_docker_api:
needs: [train, tags]
uses: ./.github/workflows/docker_with_model.yml
with:
image_name: ${{ needs.train.outputs.DOCKER_API_IMAGE_NAME }}
image_version: ${{ needs.tags.outputs.new_version }}
image_build_args: ""
image_context: ./production/api
image_file: "./production/api/Dockerfile"
model_version: ${{ needs.train.outputs.MODEL_VERSION }}
resource_group: ${{ needs.train.outputs.AZURE_RESOURCE_GROUP_NAME }}
workspace_name: ${{ needs.train.outputs.AZURE_ML_WORKSPACE_NAME }}
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
deploy:
environment: MLOpsPython
runs-on: ubuntu-latest
needs: [tags, build_docker_api, build_docker_webapp]
steps:
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
- name: Deploy container
run: |
# https://learn.microsoft.com/en-us/cli/azure/container?view=azure-cli-latest#az-container-create()
az container create --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --dns-name-label ${{ env.AZURE_WEBAPP_NAME }} --image docker.io/${{ env.DOCKER_API_IMAGE_NAME }}:${{ needs.tags.outputs.new_version }} --ports 5000
integration_tests:
environment: MLOpsPython
runs-on: ubuntu-latest
needs: [deploy, train]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ env.PYTHON_VERSION }}
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install --user poetry
- name: azure login
uses: azure/login@v1
with:
creds: ${{secrets.AZURE_CREDENTIALS}}
- name: Run Integration Test
working-directory: production/integration
run: |
poetry install
az extension add -n ml
poetry run python azureml_run_pipeline.py \
--subscription_id ${{ secrets.AZURE_SUBSCRIPTION_ID }} \
--resource_group_name ${{ env.AZURE_RESOURCE_GROUP_NAME }} \
--workspace_name ${{ env.AZURE_ML_WORKSPACE_NAME }} \
--experiment_id ${{ needs.train.outputs.EXPERIMENT_ID }} \
--integration_dataset_name "cats-dogs-others-integration" \
--integration_dataset_version ${{ needs.train.outputs.INTEGRATION_DATASET_VERSION }} \
--url "http://${{ env.AZURE_WEBAPP_NAME }}.${{ env.AZURE_LOCATION }}.azurecontainer.io:5000/upload" > integration_output.txt
cat integration_output.txt
- name: Delete Environment
run: |
if [ "${{ env.DELETE_WEBAPP }}" = "true" ]; then
az container delete --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --yes
fi
webapp_unit_tests:
runs-on: ubuntu-latest
environment: MLOpsPython
outputs:
DOCKER_WEBAPP_IMAGE_NAME: ${{ env.DOCKER_WEBAPP_IMAGE_NAME }}
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter_webapp
with:
filters: |
webapp:
- 'webapp/**'
- uses: actions/setup-node@v2
with:
node-version: '18'
- name: Install dependencies > UnitTest > Build
working-directory: production/webapp
run: |
npm ci
npm run coverage
npm run build
- name: Upload WebApp Build Artifact
uses: actions/upload-artifact@v2
with:
name: Publish webapp
path: ./production/webapp/build
build_docker_webapp:
needs: [webapp_unit_tests, tags]
uses: ./.github/workflows/docker.yml
with:
image_name: ${{ needs.webapp_unit_tests.outputs.DOCKER_WEBAPP_IMAGE_NAME }}
image_version: ${{ needs.tags.outputs.new_version }}
image_build_args: ""
image_context: ./production/webapp
image_file: "./production/webapp/Dockerfile"
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
change_log:
runs-on: ubuntu-latest
needs: [ integration_tests ]
environment: MLOpsPython
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.GIT_TOKEN }}
fetch-depth: 0
- name: Commit and push
if: github.ref == 'refs/heads/main'
run: |
git config --global user.name "GitHub"
git config --global user.email "github-action@bot.com"
PROJECT_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit"
chmod +x ./bin/generate-changelog.sh
./bin/generate-changelog.sh "$PROJECT_URL"
git add ./CHANGELOG.md
git commit -m "[skip ci] Generate changelog to version ${{ steps.tag.outputs.new_version }}"
git push -f --set-upstream origin "HEAD:main" --follow-tags