Merge pull request #22 from aisingapore/0.3.0-gcp-backports

[0.3.0] GCP backporting

README.md

@@ -26,20 +26,41 @@ the near future.
 
 ## Usage
 
+### Creating new repository
+
 To use the template and create a repository, you would need to install
 the `cookiecutter` CLI, say within a virtual environment and pass the
 URL of this template as an argument, like such:
 
 ```bash
-$ pip install cookiecutter
+$ pip install "cookiecutter>=2.2"
 $ cookiecutter https://github.com/aisingapore/kapitan-hull
 ```
 
-You will then be prompted to provide inputs.These inputs will be used to
-populate different parts of the repository to be generated by
+If you want to run a specific version of Kapitan Hull for compatibility 
+reasons, you can specify the `-c` parameter for the specific tag/branch
+we have:
+
+```bash
+$ cookiecutter https://github.com/aisingapore/kapitan-hull -c <tag>
+```
+
+You will then be prompted to provide inputs. These inputs will be used 
+to populate different parts of the repository to be generated by
 `cookiecutter`.
 
-> **Note**: Your cookiecutter version must be at least ***2.2.0***.
+### Updating your repository
+
+If you want to update the repository with updated utilities, 
+***push all your changes in your repository first***. After that, you 
+can run this command in the repository:
+
+```bash
+# Add `-c <tag>` for the specific tag/branch
+$ cookiecutter --replay-file cookiecutter.json \
+    https://github.com/aisingapore/kapitan-hull \
+    -o .. -f
+```
 
 ### Input Parameters
 

hooks/pre_gen_project.py

@@ -19,7 +19,7 @@
         "regex": r"^[a-z](?:_?[a-z0-9]+)*$"},
     "registry_project_path": {
         "user_input": "{{cookiecutter.registry_project_path}}",
-        "regex": r"^registry\.aisingapore\.net[\-\/\w\d]+(?:[a-z0-9]+)$"},
+        "regex": r"^(https?:\/\/)?([a-zA-Z0-9.-]+(:[a-zA-Z0-9.-]+)?@)?([a-zA-Z0-9.-]+)(:[0-9]+)?\/([a-zA-Z0-9._-]+\/)*([a-zA-Z0-9._-]+)(:[a-zA-Z0-9._-]+)?$"},
     "author_name": {
         "user_input": "{{cookiecutter.author_name}}",
         "regex": r"^[a-zA-Z](?:_?[a-zA-Z0-9]+)*$"}
@@ -53,7 +53,7 @@ def check_input_regex(cookie_input_key, cookie_input_val):
                 % (cookie_input_key, cookie_input_val["user_input"]))
 
         if cookie_input_key == "registry_project_path":
-            ERROR_MSG_LIST.append("ERROR: %s - '%s' is not a valid Harbor path."
+            ERROR_MSG_LIST.append("ERROR: %s - '%s' is not a valid container registry path."
                 % (cookie_input_key, cookie_input_val["user_input"]))
 
         if cookie_input_key == "author_name":

{{cookiecutter.repo_name}}/README.md

@@ -1,7 +1,13 @@
 # {{cookiecutter.project_name}}
 
+{% if cookiecutter.platform == 'gcp' -%}
+![AI Singapore's Kapitan Hull EPTG GCP Run:ai Banner](./aisg-context/guide-site/docs/kapitan-hull-eptg-gcp-runai-banner.png)
+
+{% elif cookiecutter.platform == 'onprem' -%}
 ![AI Singapore's Kapitan Hull EPTG Onprem Run:ai Banner](./aisg-context/guide-site/docs/kapitan-hull-eptg-onprem-runai-banner.png)
 
+{% endif -%}
+
 _{{cookiecutter.description}}_
 
 __A project generated using AI Singpaore's Kapitan Hull, an end-to-end 
@@ -32,6 +38,25 @@ Inputs provided to `cookiecutter` for the generation of this template:
 [ccutter]: https://cookiecutter.readthedocs.io/en/stable/
 [kh-readme]: https://github.com/aisingapore/kapitan-hull/blob/main/README.md
 
+## Regenerating/Updating from Kapitan Hull
+
+> It is ***strongly recommended*** to push any changes you made to git 
+> before running the command to revert back the changes you do not want, 
+> especially the model code and the files in the `conf` folder.
+
+If you need to regenerate the scripts from Kapitan Hull to update the 
+inputs or to a new version of Kapitan Hull, you can update the 
+`cookiecutter.json` with the new inputs and run this command 
+within this repository:
+
+```bash
+$ cookiecutter --replay-file cookiecutter.json \
+    https://github.com/aisingapore/kapitan-hull -o .. -f
+```
+
+Add the `-c` parameter to specify the tag/branch you wish to use 
+instead.
+
 ## End-to-end Guide
 
 This repository contains a myriad of boilerplate codes and configuration

{{cookiecutter.repo_name}}/aisg-context/guide-site/docs/guide-for-user/03-mlops-components-platform.md

@@ -53,15 +53,10 @@ See [here][sdk-auth] for more information on authorising your SDK.
 A simple command to authorise access:
 
 ```bash
+# For authorisation with user account
 $ gcloud auth login
-```
-
-To register `gcloud` for Docker so you can push to Google Container 
-Registry:
-
-```bash
-$ gcloud auth configure-docker \
-    asia-southeast1-docker.pkg.dev
+# For authorisation with service account
+$ gcloud auth login --cred-file=/path/to/service-account-key.json
 ```
 
 With your user account, you should have access to the following GCP
@@ -73,7 +68,8 @@ products/services:
 
 [gcp-sdk]: https://cloud.google.com/sdk
 [sdk-auth]: https://cloud.google.com/sdk/docs/authorizing
-{%- set kubeplat = 'GKE' -%}
+{%- set kubeplat = 'GKE' %}
+
 {% endif -%}
 
 ## Kubernetes
@@ -130,7 +126,8 @@ the same set of credentials that you use for your GitLab account.
 
 [aisg-rke]: https://rancher.aisingapore.net
 [rancher]: https://www.rancher.com
-{%- set kubeplat = 'Rancher' -%}
+{%- set kubeplat = 'Rancher' %}
+
 {% endif -%}
 
 {% if cookiecutter.orchestrator == 'runai' -%}
@@ -144,7 +141,7 @@ the same set of credentials that you use for your GitLab account.
 {% elif cookiecutter.orchestrator == "noorch" -%}
     {%- set vs_orch = " " -%}
     {%- set and_orch = " " -%}
-{% endif %}
+{% endif -%}
 
 ### Kubernetes VS {{kubeplat}}{{vs_orch}}
 
@@ -186,7 +183,13 @@ Run:AI.
 The diagram below showcases the some of the components that this guide
 will cover as well as how each of them relate to each other.
 
-[![AISG's End-to-end MLOps Workflow & Components Diagram](assets/images/aisg-e2e-mlops-workflow-components_jul2023.png)](assets/aisg-e2e-mlops-workflow-components_jul2023.html)
+{% if cookiecutter.platform == 'gcp' -%}
+[![AISG's End-to-end MLOps Workflow & Components Diagram for GCP Run:ai](assets/images/aisg-e2e-mlops-gcp-runai-workflow-components_jul2023.png)](assets/aisg-e2e-mlops-gcp-runai-workflow-components_jul2023.html)
+
+{% elif cookiecutter.platform == 'onprem' -%}
+[![AISG's End-to-end MLOps Workflow & Components Diagram for Onprem Run:ai](assets/images/aisg-e2e-mlops-onprem-runai-workflow-components_jul2023.png)](assets/aisg-e2e-mlops-onprem-runai-workflow-components_jul2023.html)
+
+{% endif -%}
 
 !!! note
     Click on the image above for an interactive view of the diagram.
@@ -398,7 +401,7 @@ Host gitlab.aisingapore.net
 
     - [GitLab Docs - Use SSH keys to communicate with GitLab](https://docs.gitlab.com/ee/user/ssh.html)
 
-{%- if cookiecutter.orchestrator == 'runai' -%}
+{% if cookiecutter.orchestrator == 'runai' -%}
 ## Run:AI
 
 Run:AI is an enterprise orchestration and cluster management platform
@@ -590,8 +593,7 @@ verification code and paste it into the terminal.
 {%- elif cookiecutter.orchestrator == 'polyaxon' -%}
 {%- elif cookiecutter.orchestrator == 'none' -%}
 {% endif %}
-
-{%- if cookiecutter.platform == 'onprem' -%}
+{% if cookiecutter.platform == 'onprem' %}
 ## Harbor
 
 AI Singapore uses a self-hosted Harbor as the on-premise container image
@@ -757,10 +759,10 @@ created by administrators. These accounts are usually created for
 automated workflows that require access to ECS. Configuring them for the
 CLI works the same as configuring a regular user account.
 
-{%- elif cookiecutter.platform == 'gcp' -%}
-## Google Container Registry
+{%- elif cookiecutter.platform == 'gcp' %}
+## Google Artifact Registry
 
-AI Singapore's emphases on reproducibility and portability of
+AI Singapore's emphasis on reproducibility and portability of
 workflows and accompanying environments translates to heavy usage of
 containerisation. Throughout this guide, we will be building Docker
 images necessary for setting up development environments, jobs for
@@ -773,13 +775,54 @@ like so:
 ```bash
 $ gcloud container images list --repository={{cookiecutter.registry_project_path}}
 ```
-You will be pushing the Docker images to the aforementioned repository.
+
+To push or pull images to/from Artifact Registry, you would need to
+authenticate with the Google Cloud project that the registry is
+associated with. You can do so by running the following command:
+
+```bash
+$ gcloud auth configure-docker asia-southeast1-docker.pkg.dev
+```
+
+The command above will populate your Docker configuration file with
+the intended Artifact Registry Docker host. Host names Google
+Artifact Registry ends with `-docker.pkg.dev`.
 
 ??? info "Reference Link(s)"
 
     - [`gcloud` Reference - `gcloud container images list`](https://cloud.google.com/sdk/gcloud/reference/container/images/list)
+    - [Google Cloud Artifact Registry Docs - Set up authentication for Docker](https://cloud.google.com/artifact-registry/docs/docker/authentication)
     - [Artifact Registry Guide - Pushing & Pulling Images](https://cloud.google.com/artifact-registry/docs/docker/pushing-and-pulling)
-{% endif %}
+
+## Google Cloud Storage (GCS)
+
+In the context of a Google Cloud infrastructure environment, there are
+two main storage mediums:
+
+1. Google Cloud Filestore for managed network file storage (NFS)
+2. Google Cloud Storage (GCS) for object storage
+
+The usage of NFS storage is mainly observable through Persistent
+Volumes (PVs) or virtual machine disks. There is however little to
+nothing for end-users to configure the NFS storage as most of the setup
+will be done by AI Singapore's Platforms team.
+
+As for GCS, one would be provided with access to one or more GCS buckets
+through the provided user or service account. Upon authorisation, one
+may list the contents of a bucket like so:
+
+!!! note inline end
+    `GCP_PROJECT_ID` and `GCP_PROJECT_ID` are provided by the MLOps
+    team. The team is reachable at `mlops@aisingapore.org`.
+
+```bash
+$ gsutil ls -p <GCP_PROJECT_ID> gs://<GCS_BUCKET_NAME>
+```
+
+??? info "Reference Link(s)"
+
+    - [IBM Blog - Object vs. File vs. Block Storage: What’s the Difference?](https://www.ibm.com/blog/object-vs-file-vs-block-storage)
+{%- endif %}
 
 ## MLflow
 
@@ -835,7 +878,9 @@ MLflow Tracking server.
     ```bash
     $ conda create -n mlflow-test python=3.11.7
     $ conda activate mlflow-test
-    $ pip install boto3==1.34.17 mlflow==2.9.2
+    $ pip install mlflow==2.9.2
+    # Install boto3 or google-cloud-storage packages if 
+    # custom object storage is used
     $ export MLFLOW_TRACKING_USERNAME=<MLFLOW_TRACKING_USERNAME>
     $ export MLFLOW_TRACKING_PASSWORD=<MLFLOW_TRACKING_PASSWORD>
     $ python src/mlflow_test.py <MLFLOW_TRACKING_URI> <NAME_OF_DEFAULT_MLFLOW_EXPERIMENT>
@@ -846,7 +891,9 @@ MLflow Tracking server.
     ```powershell
     $ conda create -n mlflow-test python=3.11.7
     $ conda activate mlflow-test
-    $ pip install boto3==1.34.17 mlflow==2.9.2
+    $ pip install mlflow==2.9.2
+    # Install boto3 or google-cloud-storage packages if 
+    # custom object storage is used
     $ $MLFLOW_TRACKING_USERNAME=<MLFLOW_TRACKING_USERNAME>
     $ $MLFLOW_TRACKING_PASSWORD=<MLFLOW_TRACKING_PASSWORD>
     $ python src/mlflow_test.py <MLFLOW_TRACKING_URI> <NAME_OF_DEFAULT_MLFLOW_EXPERIMENT>

{{cookiecutter.repo_name}}/aisg-context/guide-site/docs/guide-for-user/05-virtual-env.md

@@ -84,7 +84,7 @@ server to detect the `conda` environments that you would have created.
     ```bash
     $ conda activate /<NAME_OF_DATA_SOURCE>/workspaces/<YOUR_HYPHENATED_NAME>/conda_envs/{{cookiecutter.repo_name}}
     $ conda list | grep "ipykernel"
-    ipykernel  X.X.X  pypi_0  pypi
+    ipykernel  6.25.0  pypi_0  pypi
     ```
 
 - Now enter `Ctrl + Shift + P` again and execute 
@@ -106,6 +106,10 @@ server to detect the `conda` environments that you would have created.
 
 ## Jupyter Kernel for JupyterLab
 
+!!! attention
+    If you're using VSCode and not JupyterLab, this section can be 
+    skipped.
+
 The same with the VSCode server, the JupyterLab server would not by 
 default detect `conda` environments. You would have to specify to the 
 JupyterLab installation the `ipython` kernel existing within your 
@@ -124,7 +128,7 @@ JupyterLab installation the `ipython` kernel existing within your
     ```bash
     $ conda activate /<NAME_OF_DATA_SOURCE>/workspaces/<YOUR_HYPHENATED_NAME>/conda_envs/{{cookiecutter.repo_name}}
     $ conda list | grep "ipykernel"
-    ipykernel  6.9.2  pypi_0  pypi
+    ipykernel  6.25.0  pypi_0  pypi
     ```
 
 - Within the `conda` environment, execute the following:

{{cookiecutter.repo_name}}/aisg-context/guide-site/docs/guide-for-user/07-job-orchestration.md

@@ -72,6 +72,9 @@ provided in this template:
     $ khull kaniko --context $(pwd) \
         --dockerfile $(pwd)/docker/{{cookiecutter.repo_name}}-data-prep.Dockerfile \
         --destination {{cookiecutter.registry_project_path}}/data-prep:0.1.0 \
+{%- if cookiecutter.platform == 'gcp' %}
+        --gcp \
+{%- endif %}
         --cred-file /path/to/docker/config.json \
         -v <pvc-name>:/path/to/pvc/mount
     ```
@@ -207,6 +210,9 @@ the Docker image to be used for it:
     $ khull kaniko --context $(pwd) \
         --dockerfile $(pwd)/docker/{{cookiecutter.repo_name}}-model-training.Dockerfile \
         --destination {{cookiecutter.registry_project_path}}/model-training:0.1.0 \
+{%- if cookiecutter.platform == 'gcp' %}
+        --gcp \
+{%- endif %}
         --cred-file /path/to/docker/config.json \
         -v <pvc-name>:/path/to/pvc/mount
     ```

{{cookiecutter.repo_name}}/aisg-context/guide-site/docs/index.md

@@ -1,7 +1,13 @@
 # {{cookiecutter.project_name}}
 
+{% if cookiecutter.platform == 'gcp' -%}
+![AI Singapore's Kapitan Hull EPTG GCP Run:ai Banner](./kapitan-hull-eptg-gcp-runai-banner.png)
+
+{% elif cookiecutter.platform == 'onprem' -%}
 ![AI Singapore's Kapitan Hull EPTG Onprem Run:ai Banner](./kapitan-hull-eptg-onprem-runai-banner.png)
 
+{% endif -%}
+
 _{{cookiecutter.description}}_
 
 __A project generated using AI Singpaore's Kapitan Hull, an end-to-end 

{{cookiecutter.repo_name}}/cookiecutter.json

@@ -0,0 +1,14 @@
+{
+    "cookiecutter": {
+        "project_name": "{{cookiecutter.project_name}}",
+        "description": "{{cookiecutter.description}}",
+        "repo_name": "{{cookiecutter.repo_name}}",
+        "src_package_name": "{{cookiecutter.src_package_name}}",
+        "src_package_name_short": "{{cookiecutter.src_package_name_short}}",
+        "platform": "{{cookiecutter.platform}}",
+        "orchestrator": "{{cookiecutter.orchestrator}}",
+        "proj_name": "{{cookiecutter.proj_name}}",
+        "registry_project_path": "{{cookiecutter.registry_project_path}}",
+        "author_name": "{{cookiecutter.author_name}}"
+    }
+}

{{cookiecutter.repo_name}}/src/{{cookiecutter.src_package_name}}/general_utils.py

@@ -39,7 +39,7 @@ def setup_logging(
         logger.info("Logging config file is not found. Basic config is being used.")
 
 
-def mlflow_init(args, setup_mlflow=False, autolog=False):
+def mlflow_init(args, run_name='train-model', setup_mlflow=False, autolog=False):
     """Initialise MLflow connection.
 
     Parameters
@@ -73,8 +73,6 @@ def mlflow_init(args, setup_mlflow=False, autolog=False):
             if autolog:
                 mlflow.autolog()
 
-            run_name = "train-model"
-
             if "MLFLOW_HPTUNING_TAG" in os.environ: run_name += "-hp"
 
             run_name += "-{:.0f}".format(time.time())

{{cookiecutter.repo_name}}/src/{{cookiecutter.src_package_name}}_fastapi/config.py

@@ -11,7 +11,7 @@ class Settings(pydantic_settings.BaseSettings):
 
     USE_CUDA: bool = False
     USE_MPS: bool = False
-    PRED_MODEL_UUID: str
+    PRED_MODEL_UUID: str = "test"
     PRED_MODEL_PATH: str