From 7b782c6fe5181953f08beb0f7cc8668899f63ec7 Mon Sep 17 00:00:00 2001 From: Guillaume Chervet Date: Sun, 8 Sep 2024 14:17:26 +0200 Subject: [PATCH] refactor(all): update for course september 2024 (#37) --- .github/workflows/docker.yml | 11 +++++- .github/workflows/docker_with_model.yml | 25 +++++++++++-- .github/workflows/main.yml | 49 ++++++++++++++++--------- README.md | 4 +- poetry.lock | 45 ++++++++++++----------- train/azureml_run_download_model.py | 44 ++++++++++++++++++++++ train/run_download_model.sh | 14 +++++++ train/setup_AzureML.sh | 2 +- train/test/test.py | 7 ++++ train/train/command.py | 2 +- train/train/explore.ipynb | 42 +++++++++++++++------ train/train/train.py | 42 ++++++++++++--------- 12 files changed, 208 insertions(+), 79 deletions(-) create mode 100644 train/azureml_run_download_model.py create mode 100644 train/run_download_model.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 941e7e6f..6c2e9138 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,6 +18,9 @@ on: image_file: required: true type: string + docker_registry: + required: true + type: string secrets: DOCKER_USERNAME: required: true @@ -26,6 +29,9 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: read + packages: write environment: MLOpsPython steps: - uses: actions/checkout@v3 @@ -33,6 +39,7 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: + registry: ${{ inputs.docker_registry }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -50,7 +57,7 @@ jobs: file : ${{ inputs.image_file }} build-args: ${{ inputs.image_build_args }} push: true - tags: ${{ inputs.image_name }}:latest,${{ inputs.image_name }}:${{ inputs.image_version }} + tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:latest,${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }} labels: ${{ steps.meta.outputs.labels }} - name: Build and push Docker image @@ -61,6 +68,6 @@ jobs: file : ${{ inputs.image_file }} build-args: ${{ inputs.image_build_args }} push: true - tags: ${{ inputs.image_name }}:${{ inputs.image_version }} + tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker_with_model.yml b/.github/workflows/docker_with_model.yml index 248a8fa4..5bacdaee 100644 --- a/.github/workflows/docker_with_model.yml +++ b/.github/workflows/docker_with_model.yml @@ -27,6 +27,12 @@ on: workspace_name: required: true type: string + location: + required: true + type: string + docker_registry: + required: true + type: string secrets: DOCKER_USERNAME: required: true @@ -34,9 +40,14 @@ on: required: true AZURE_CREDENTIALS: required: true + AZURE_SUBSCRIPTION_ID: + required: true jobs: build: runs-on: ubuntu-latest + permissions: + contents: read + packages: write environment: MLOpsPython steps: - uses: actions/checkout@v3 @@ -64,12 +75,17 @@ jobs: - name: download model run: | - az extension add -n ml MODEL_VERSION=${{ inputs.model_version }} RESOURCE_GROUP=${{ inputs.resource_group }} WORKSPACE_NAME=${{ inputs.workspace_name }} + AZURE_LOCATION=${{ inputs.location }} + DOWNLOAD_PATH=production/api/core/model + cd train + chmod +x ./run_download_model.sh + echo './run_download_model.sh "$MODEL_VERSION" "$RESOURCE_GROUP" "$WORKSPACE_NAME" "$AZURE_LOCATION" "${{ secrets.AZURE_SUBSCRIPTION_ID }}" "$DOWNLOAD_PATH"' + ./run_download_model.sh "$MODEL_VERSION" "$RESOURCE_GROUP" "$WORKSPACE_NAME" "$AZURE_LOCATION" "${{ secrets.AZURE_SUBSCRIPTION_ID }}" "$DOWNLOAD_PATH" cd production/api/core/model - az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group $RESOURCE_GROUP --workspace-name $WORKSPACE_NAME + # az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group $RESOURCE_GROUP --workspace-name $WORKSPACE_NAME # find files recursively and copy them to the current directory root find ./ -name '*.keras' -exec cp "{}" ./ \; ls @@ -77,6 +93,7 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@v2 with: + registry: ${{ inputs.docker_registry }} username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} @@ -94,7 +111,7 @@ jobs: file : ${{ inputs.image_file }} build-args: ${{ inputs.image_build_args }} push: true - tags: ${{ inputs.image_name }}:latest,${{ inputs.image_name }}:${{ inputs.image_version }} + tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:latest,${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }} labels: ${{ steps.meta.outputs.labels }} - name: Build and push Docker image @@ -105,6 +122,6 @@ jobs: file : ${{ inputs.image_file }} build-args: ${{ inputs.image_build_args }} push: true - tags: ${{ inputs.image_name }}:${{ inputs.image_version }} + tags: ${{ inputs.docker_registry }}/${{ inputs.image_name }}:${{ inputs.image_version }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c4c4cfe..96daf823 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,17 +10,20 @@ env: PNPM_VERSION: 8.5.1 PYTHON_VERSION: 3.10.13 NODE_VERSION: 18 - AZURE_RESOURCE_GROUP_NAME: "azure-ml-yolw" + AZURE_RESOURCE_GROUP_NAME: "azure-ml-gui" AZURE_LOCATION: "northeurope" - AZURE_ML_WORKSPACE_NAME: "cats-dogs-yolw" - AZURE_WEBAPP_NAME: "cats-dogs-yolw" + AZURE_ML_WORKSPACE_NAME: "cats-dogs-gui" + AZURE_WEBAPP_NAME: "cats-dogs-gui" DELETE_WEBAPP: "false" - DOCKER_API_IMAGE_NAME: "guillaumechervetlunique/mlopspython-api" - DOCKER_WEBAPP_IMAGE_NAME: "guillaumechervetlunique/mlopspython-webapp" + DOCKER_API_IMAGE_NAME: "mlopspython-api" + DOCKER_REPOSITORY: ${{ github.repository_owner }} + DOCKER_WEBAPP_IMAGE_NAME: "mlopspython-webapp" + DOCKER_REGISTRY: "ghcr.io" permissions: id-token: write contents: write + packages: write jobs: lint: runs-on: ubuntu-latest @@ -149,7 +152,9 @@ jobs: EXPERIMENT_ID: ${{ steps.train.outputs.EXPERIMENT_ID }} AZURE_RESOURCE_GROUP_NAME: ${{ env.AZURE_RESOURCE_GROUP_NAME }} AZURE_ML_WORKSPACE_NAME: ${{ env.AZURE_ML_WORKSPACE_NAME }} - DOCKER_API_IMAGE_NAME: ${{ env.DOCKER_API_IMAGE_NAME }} + DOCKER_API_IMAGE_NAME: ${{ env.DOCKER_REPOSITORY }}/${{ env.DOCKER_API_IMAGE_NAME }} + AZURE_LOCATION: ${{ env.AZURE_LOCATION }} + DOCKER_REGISTRY: ${{ env.DOCKER_REGISTRY }} steps: - uses: actions/checkout@v3 with: @@ -185,19 +190,23 @@ jobs: - name: download model id: train run: | - az extension add -n ml MODEL_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt model_version) - echo "MODEL_VERSION=$MODEL_VERSION" >> $GITHUB_OUTPUT + echo "MODEL_VERSION=$MODEL_VERSION" >> $GITHUB_OUTPUT INTEGRATION_DATASET_VERSION=$(python bin/retrieve_output.py ./train/train_output.txt integration_dataset_version) echo "INTEGRATION_DATASET_VERSION=$INTEGRATION_DATASET_VERSION" >> $GITHUB_OUTPUT EXPERIMENT_ID=$(python bin/retrieve_output.py ./train/train_output.txt experiment_id) echo "EXPERIMENT_ID=$EXPERIMENT_ID" >> $GITHUB_OUTPUT - mkdir model rm -r model mkdir model cd model - az ml model download --name cats-dogs-others --version $MODEL_VERSION --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME }} --workspace-name ${{ env.AZURE_ML_WORKSPACE_NAME }} + DOWNLOAD_PATH=$PWD + cd .. + cd train + chmod +x ./run_download_model.sh + ./run_download_model.sh $MODEL_VERSION ${{ env.AZURE_RESOURCE_GROUP_NAME }} ${{ env.AZURE_ML_WORKSPACE_NAME }} ${{ env.AZURE_LOCATION }} ${{ secrets.AZURE_SUBSCRIPTION_ID }} $DOWNLOAD_PATH + cd .. + cd model # find files recursively and copy them to the current directory root find ./ -name '*.keras' -exec cp "{}" ./ \; find ./ -name '*.json' -exec cp "{}" ./ \; @@ -248,10 +257,13 @@ jobs: model_version: ${{ needs.train.outputs.MODEL_VERSION }} resource_group: ${{ needs.train.outputs.AZURE_RESOURCE_GROUP_NAME }} workspace_name: ${{ needs.train.outputs.AZURE_ML_WORKSPACE_NAME }} + location: ${{ needs.train.outputs.AZURE_LOCATION }} + docker_registry: ${{ needs.train.outputs.DOCKER_REGISTRY }} secrets: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ github.actor }} + DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} deploy: environment: MLOpsPython runs-on: ubuntu-latest @@ -264,7 +276,7 @@ jobs: - name: Deploy container run: | # https://learn.microsoft.com/en-us/cli/azure/container?view=azure-cli-latest#az-container-create() - az container create --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --dns-name-label ${{ env.AZURE_WEBAPP_NAME }} --image docker.io/${{ env.DOCKER_API_IMAGE_NAME }}:${{ needs.tags.outputs.new_version }} --ports 5000 + az container create --resource-group ${{ env.AZURE_RESOURCE_GROUP_NAME}} --name ${{ env.AZURE_WEBAPP_NAME }} --dns-name-label ${{ env.AZURE_WEBAPP_NAME }} --image ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_REPOSITORY }}/${{ env.DOCKER_API_IMAGE_NAME }}:${{ needs.tags.outputs.new_version }} --ports 5000 integration_tests: environment: MLOpsPython runs-on: ubuntu-latest @@ -306,7 +318,7 @@ jobs: runs-on: ubuntu-latest environment: MLOpsPython outputs: - DOCKER_WEBAPP_IMAGE_NAME: ${{ env.DOCKER_WEBAPP_IMAGE_NAME }} + DOCKER_WEBAPP_IMAGE_NAME: ${{ env.DOCKER_REPOSITORY }}/${{ env.DOCKER_WEBAPP_IMAGE_NAME }} steps: - uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 @@ -330,7 +342,7 @@ jobs: name: Publish webapp path: ./production/webapp/build build_docker_webapp: - needs: [webapp_unit_tests, tags] + needs: [webapp_unit_tests, tags, train] uses: ./.github/workflows/docker.yml with: image_name: ${{ needs.webapp_unit_tests.outputs.DOCKER_WEBAPP_IMAGE_NAME }} @@ -338,9 +350,10 @@ jobs: image_build_args: "" image_context: ./production/webapp image_file: "./production/webapp/Dockerfile" + docker_registry: ${{ needs.train.outputs.DOCKER_REGISTRY }} secrets: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ github.actor }} + DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} change_log: runs-on: ubuntu-latest @@ -349,7 +362,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - token: ${{ secrets.GIT_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 - name: Commit and push if: github.ref == 'refs/heads/main' diff --git a/README.md b/README.md index e6f19c52..907eb367 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ git clone https://www.github.com/guillaume-chervet/MLOpsPython cd MLOpsPython chmod +x Makefile # If you have poetry installed -./Makefile poetry 0.2.0 +./Makefile poetry 0.13.0 # Else degraded mode -./Makefile pip 0.2.0 +./Makefile pip 0.13.0 cd production docker-compose up diff --git a/poetry.lock b/poetry.lock index 0b210602..6572a52b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -111,50 +111,51 @@ files = [ [[package]] name = "packaging" -version = "23.2" +version = "24.1" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, - {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] name = "pathspec" -version = "0.11.2" +version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, - {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] [[package]] name = "platformdirs" -version = "3.11.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, - {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, ] [package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] [[package]] name = "pycodestyle" -version = "2.11.0" +version = "2.11.1" description = "Python style guide checker" optional = false python-versions = ">=3.8" files = [ - {file = "pycodestyle-2.11.0-py2.py3-none-any.whl", hash = "sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8"}, - {file = "pycodestyle-2.11.0.tar.gz", hash = "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0"}, + {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, + {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, ] [[package]] @@ -181,16 +182,16 @@ files = [ [[package]] name = "typing-extensions" -version = "4.8.0" +version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.11" -content-hash = "d05f09d871f41cb4de8f46259a42faf41746b466fd5fd3b07a432acdc19b8ce0" +content-hash = "b491eecf6c55ae33c436fdc16027f1e22d8ea6aee51a71bd15480ef749307801" diff --git a/train/azureml_run_download_model.py b/train/azureml_run_download_model.py new file mode 100644 index 00000000..fbfc4eed --- /dev/null +++ b/train/azureml_run_download_model.py @@ -0,0 +1,44 @@ +import argparse + +from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential + +from azure.ai.ml import MLClient + + +parser = argparse.ArgumentParser("train") +parser.add_argument("--subscription_id", type=str) +parser.add_argument("--resource_group_name", type=str) +parser.add_argument("--workspace_name", type=str) +parser.add_argument("--location", type=str) +parser.add_argument("--version", type=str, default="1") +parser.add_argument("--download_path", type=str, default="./") + +args = parser.parse_args() +subscription_id = args.subscription_id +resource_group_name = args.resource_group_name +workspace_name = args.workspace_name +location = args.location +version = args.version +download_path = args.download_path + +try: + credential = DefaultAzureCredential() + # Check if given credential can get token successfully. + credential.get_token("https://management.azure.com/.default") +except Exception as ex: + print(ex) + # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work + credential = InteractiveBrowserCredential() + + +# Get a handle to workspace +ml_client = MLClient( + credential=credential, + subscription_id=subscription_id, + resource_group_name=resource_group_name, + workspace_name=workspace_name, +) + +model_name = 'cats-dogs-others' + +ml_client.models.download(name=model_name, version=version, download_path=download_path) \ No newline at end of file diff --git a/train/run_download_model.sh b/train/run_download_model.sh new file mode 100644 index 00000000..359bbefa --- /dev/null +++ b/train/run_download_model.sh @@ -0,0 +1,14 @@ +MODEL_VERSION=$1 +AZURE_RESOURCE_GROUP_NAME=$2 +AZURE_ML_WORKSPACE_NAME=$3 +AZURE_LOCATION=$4 +AZURE_SUBSCRIPTION_ID=$5 +DOWNLOAD_PATH=$6 + +poetry run python azureml_run_download_model.py \ + --subscription_id "$AZURE_SUBSCRIPTION_ID" \ + --resource_group_name "$AZURE_RESOURCE_GROUP_NAME" \ + --workspace_name "$AZURE_ML_WORKSPACE_NAME" \ + --location "$AZURE_LOCATION" \ + --version "$MODEL_VERSION" \ + --download_path "$DOWNLOAD_PATH" diff --git a/train/setup_AzureML.sh b/train/setup_AzureML.sh index 36360534..c270846d 100644 --- a/train/setup_AzureML.sh +++ b/train/setup_AzureML.sh @@ -8,7 +8,7 @@ AZURE_LOCATION=$4 #az account list-locations -o table # Initialize Azure ML Workspace az account set --subscription "$AZURE_SUBSCRIPTION_ID" az group create --name "$AZURE_RESOURCE_GROUP_NAME" --location "$AZURE_LOCATION" -az extension add -n ml +az extension add -n ml --allow-preview false --yes # shellcheck disable=SC2086 az ml workspace create -n "$AZURE_ML_WORKSPACE_NAME" -g $AZURE_RESOURCE_GROUP_NAME diff --git a/train/test/test.py b/train/test/test.py index 5840f92d..c80dae8e 100644 --- a/train/test/test.py +++ b/train/test/test.py @@ -16,6 +16,7 @@ def execute_model_and_generate_integration_test_data( model_output_directory: Path, integration_output_directory: Path, ): + logger = logging.getLogger("test") model_path = input_model_directory / model_name model = Inference(logging, str(model_path)) @@ -25,7 +26,11 @@ def execute_model_and_generate_integration_test_data( for path in tests_directory.glob("**/*"): if path.is_dir(): continue + # keep only files with .png extension + if path.suffix != ".png": + continue model_result = model.execute(str(path)) + logger.info(f"Prediction for {path}: {model_result}") prediction = model_result["prediction"] prediction_truth = path.parent.name.lower().replace("s", "") @@ -38,6 +43,8 @@ def execute_model_and_generate_integration_test_data( "prediction_truth": prediction_truth, "values": model_result["values"], } + logger.info(f"Result for {path}: ") + logger.info(result) results.append(result) statistics["total"] = statistics["ok"] + statistics["ko"] diff --git a/train/train/command.py b/train/train/command.py index 1871b2c1..51a5f71f 100644 --- a/train/train/command.py +++ b/train/train/command.py @@ -18,7 +18,7 @@ batch_size = 64 -epochs = 20 +epochs = 10 params = { "batch_size": batch_size, "epochs": epochs, diff --git a/train/train/explore.ipynb b/train/train/explore.ipynb index 6d922bd7..58d7a7d6 100644 --- a/train/train/explore.ipynb +++ b/train/train/explore.ipynb @@ -507,8 +507,13 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-12T09:42:02.532827Z", + "start_time": "2024-04-12T09:42:02.522276Z" + } + }, + "source": "print(toto)", "outputs": [ { "name": "stdout", @@ -518,27 +523,42 @@ ] } ], - "source": [] + "execution_count": 3 }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2024-04-12T09:41:36.920818Z", + "start_time": "2024-04-12T09:41:36.894980Z" + } + }, + "source": "toto=\"toto\"\n", "outputs": [], - "source": [ - "toto=\"toto\"" - ] + "execution_count": 1 }, { "cell_type": "code", - "execution_count": null, "metadata": { "pycharm": { "is_executing": true + }, + "ExecuteTime": { + "end_time": "2024-04-12T09:41:36.943211Z", + "start_time": "2024-04-12T09:41:36.926840Z" } }, - "outputs": [], - "source": [] + "source": "print(toto)", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "toto\n" + ] + } + ], + "execution_count": 2 }, { "cell_type": "code", diff --git a/train/train/train.py b/train/train/train.py index f3b6abf4..4d908956 100644 --- a/train/train/train.py +++ b/train/train/train.py @@ -1,5 +1,6 @@ # https://machinelearningmastery.com/how-to-develop-a-convolutional-neural-network-to-classify-photos-of-dogs-and-cats/ from dataclasses import dataclass + # vgg16 model used for transfer learning on the dogs and cats dataset from pathlib import Path @@ -22,21 +23,15 @@ def define_model(): layer.trainable = False # add new classifier layers output = model.layers[-1].output - #drop1 = keras.layers.Dropout(0.2)(output) flat1 = Flatten()(output) - class1 = Dense(64, activation="relu", kernel_initializer="he_uniform")(flat1) - #class2 = Dense(42, activation="relu", kernel_initializer="he_uniform")(class1) - output = Dense(3, activation="sigmoid")(class1) + output = Dense(3, activation="sigmoid")(flat1) # define new model model = Model(inputs=model.inputs, outputs=output) - # compile model - #opt = SGD(lr=0.0002, momentum=0.8) - #model.compile( - # optimizer=opt, loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=["accuracy"] - #) - model.compile(optimizer='adam', - loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), - metrics=['accuracy']) + model.compile( + optimizer="adam", + loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), + metrics=["accuracy"], + ) return model @@ -58,6 +53,7 @@ def summarize_diagnostics(history, output_directory: Path): pyplot.close() return plot_filepath + @dataclass class ModelResult: evaluate_accuracy_percentage: int @@ -66,7 +62,9 @@ class ModelResult: # run the test harness for evaluating a model -def run_test_harness(input_directory: Path, output_directory: Path, batch_size=64, epochs=7) -> ModelResult: +def run_test_harness( + input_directory: Path, output_directory: Path, batch_size=64, epochs=7 +) -> ModelResult: Path(output_directory).mkdir(parents=True, exist_ok=True) # define model model = define_model() @@ -79,15 +77,20 @@ def run_test_harness(input_directory: Path, output_directory: Path, batch_size=6 str(input_directory / "train"), class_mode="binary", batch_size=batch_size, - target_size=(224, 224) + target_size=(224, 224), ) validation_it = datagen.flow_from_directory( str(input_directory / "evaluate"), class_mode="binary", batch_size=batch_size, - target_size=(224, 224) + target_size=(224, 224), ) # fit model + model_path = output_directory / "final_model.keras" + callback_model_checkpoint = keras.callbacks.ModelCheckpoint( + filepath=model_path, + save_best_only=True) + callback_early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience = 3) history = model.fit_generator( train_it, steps_per_epoch=len(train_it), @@ -95,16 +98,19 @@ def run_test_harness(input_directory: Path, output_directory: Path, batch_size=6 validation_steps=len(validation_it), epochs=epochs, verbose=1, + callbacks=[callback_model_checkpoint, callback_early_stopping], ) # test model evaluate_it = datagen.flow_from_directory( - str(input_directory / "test"), class_mode="binary", batch_size=batch_size, target_size=(224, 224) + str(input_directory / "test"), + class_mode="binary", + batch_size=batch_size, + target_size=(224, 224), ) _, acc = model.evaluate_generator(evaluate_it, steps=len(evaluate_it), verbose=1) evaluate_accuracy_percentage = acc * 100.0 print("> %.3f" % (evaluate_accuracy_percentage)) # learning curves summary_image_path = summarize_diagnostics(history, output_directory) - model_path = output_directory / "final_model.keras" - model.save(str(model_path)) + #model.save(str(model_path)) return ModelResult(evaluate_accuracy_percentage, summary_image_path, model_path)