From 22576e694ee5885cb08da65d6e497cb3b6644e84 Mon Sep 17 00:00:00 2001 From: agunapal Date: Thu, 4 May 2023 20:36:46 +0000 Subject: [PATCH 1/5] Added k8s mnist example using minikube --- kubernetes/examples/mnist/MNIST.md | 110 ++++++++++++++++++++++ kubernetes/examples/mnist/deployment.yaml | 29 ++++++ kubernetes/examples/mnist/service.yaml | 19 ++++ 3 files changed, 158 insertions(+) create mode 100644 kubernetes/examples/mnist/MNIST.md create mode 100644 kubernetes/examples/mnist/deployment.yaml create mode 100644 kubernetes/examples/mnist/service.yaml diff --git a/kubernetes/examples/mnist/MNIST.md b/kubernetes/examples/mnist/MNIST.md new file mode 100644 index 0000000000..b5b02f436a --- /dev/null +++ b/kubernetes/examples/mnist/MNIST.md @@ -0,0 +1,110 @@ +# Digit recognition model with MNIST dataset using a Kubernetes cluster + +In this example, we show how to use a pre-trained custom MNIST model to performing real time Digit recognition with TorchServe. +We will be serving the model using a Kubernetes cluster deployed using [minikube](https://minikube.sigs.k8s.io/docs/start/). + +The inference service would return the digit inferred by the model in the input image. + +We used the following pytorch example to train the basic MNIST model for digit recognition : +https://github.com/pytorch/examples/tree/master/mnist + +## Serve an MNIST model on TorchServe docker container + +Run the commands given in following steps from the parent directory of the root of the repository. For example, if you cloned the repository into /home/my_path/serve, run the steps from /home/my_path/serve + + ### Create a torch model archive using the torch-model-archiver utility to archive the above files. + + ```bash + torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler examples/image_classifier/mnist/mnist_handler.py + ``` + + ### Move .mar file into model_store directory + + ```bash + mkdir model_store + mv mnist.mar model_store/ + ``` + + ### Start kubernetes cluster + + We start the cluster mounting the location of `serve` to `/host` + + The following command works if torchserve is under $HOME/serve + ```bash + minikube start --mount-string="$HOME/serve:/host" --mount + ``` + + ### Deploy the cluster + + In this example, we are launching a cluster with a single pod. + We are exposing ports 8080 and 8081 + We are also mapping the the `model_store` directory created on host to + `/home/model-server/model-store` on the container + + ```bash + kubectl apply -f kubernetes/examples/mnist/deployment.yaml + ``` + + Make sure the pod is running + + ```bash + $ kubectl get pods + NAME READY STATUS RESTARTS AGE + ts-def-5c95fdfd57-m446t 1/1 Running 0 58m + + ``` + + ### Create a Service + We create a service to send inference request to the pod. + We are using `NodePort` so that the cluster can be accessed by the outside world. + + ```bash + kubectl apply -f kubernetes/examples/mnist/service.yaml + ``` + + Verify the service is running + + ```bash + kubectl get svc + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + ts-def NodePort 10.109.14.120 8080:30160/TCP,8081:30302/TCP 59m + + ``` + + ### Make cluster accessible by localhost + + We use kubectl port-forward to make the cluster accessible from the local machine. This will run in the background. Make sure to kill the process when the test is done. + + ```bash + kubectl port-forward svc/ts-def 8080:8080 8081:8081 & + ``` + + ### Register the model on TorchServe using the above model archive file + + ```bash + curl -X POST "localhost:8081/models?model_name=mnist&url=mnist.mar&initial_workers=4" + ``` + + If this succeeeds, you will see a message like below + + ```bash + { + "status": "Model \"mnist\" Version: 1.0 registered with 4 initial workers" + } + ``` + + ### Run digit recognition inference + + ```bash + curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png + ``` + + The output in this case will be a `0` + + + ### Delete the cluster + + ```bash + minikube stop + minikube delete + ``` diff --git a/kubernetes/examples/mnist/deployment.yaml b/kubernetes/examples/mnist/deployment.yaml new file mode 100644 index 0000000000..9fa6cf5f4a --- /dev/null +++ b/kubernetes/examples/mnist/deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ts-def + labels: + app: ts-def +spec: + replicas: 1 + selector: + matchLabels: + app: ts-def + template: + metadata: + labels: + app: ts-def + spec: + volumes: + - name: model-store + hostPath: + path: /host/model_store + containers: + - name: torchserve + image: pytorch/torchserve:latest-cpu + ports: + - containerPort: 8080 + - containerPort: 8081 + volumeMounts: + - name: model-store + mountPath: /home/model-server/model-store diff --git a/kubernetes/examples/mnist/service.yaml b/kubernetes/examples/mnist/service.yaml new file mode 100644 index 0000000000..c14b328e0d --- /dev/null +++ b/kubernetes/examples/mnist/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: ts-def + labels: + run: ts-def +spec: + type: NodePort + selector: + app: ts-def + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + name: inference + - protocol: TCP + port: 8081 + targetPort: 8081 + name: management From 6325abf29ed028be88ce903bcc0f7e68458e7e45 Mon Sep 17 00:00:00 2001 From: agunapal Date: Thu, 4 May 2023 23:40:20 +0000 Subject: [PATCH 2/5] spellcheck addition --- ts_scripts/spellcheck_conf/wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ts_scripts/spellcheck_conf/wordlist.txt b/ts_scripts/spellcheck_conf/wordlist.txt index ba81059e97..301888fc4f 100644 --- a/ts_scripts/spellcheck_conf/wordlist.txt +++ b/ts_scripts/spellcheck_conf/wordlist.txt @@ -1043,3 +1043,4 @@ QueueTime WorkerLoadTime WorkerName WorkerThreadTime +minikube From 43190fa477cb0aff8911bb745c53b951df402afe Mon Sep 17 00:00:00 2001 From: agunapal Date: Mon, 24 Jul 2023 21:21:42 +0000 Subject: [PATCH 3/5] remove bash --- kubernetes/examples/mnist/MNIST.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/kubernetes/examples/mnist/MNIST.md b/kubernetes/examples/mnist/MNIST.md index b5b02f436a..2d42e8219b 100644 --- a/kubernetes/examples/mnist/MNIST.md +++ b/kubernetes/examples/mnist/MNIST.md @@ -14,13 +14,13 @@ Run the commands given in following steps from the parent directory of the root ### Create a torch model archive using the torch-model-archiver utility to archive the above files. - ```bash + ``` torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler examples/image_classifier/mnist/mnist_handler.py ``` ### Move .mar file into model_store directory - ```bash + ``` mkdir model_store mv mnist.mar model_store/ ``` @@ -30,7 +30,7 @@ Run the commands given in following steps from the parent directory of the root We start the cluster mounting the location of `serve` to `/host` The following command works if torchserve is under $HOME/serve - ```bash + ``` minikube start --mount-string="$HOME/serve:/host" --mount ``` @@ -41,13 +41,13 @@ Run the commands given in following steps from the parent directory of the root We are also mapping the the `model_store` directory created on host to `/home/model-server/model-store` on the container - ```bash + ``` kubectl apply -f kubernetes/examples/mnist/deployment.yaml ``` Make sure the pod is running - ```bash + ``` $ kubectl get pods NAME READY STATUS RESTARTS AGE ts-def-5c95fdfd57-m446t 1/1 Running 0 58m @@ -58,13 +58,13 @@ Run the commands given in following steps from the parent directory of the root We create a service to send inference request to the pod. We are using `NodePort` so that the cluster can be accessed by the outside world. - ```bash + ``` kubectl apply -f kubernetes/examples/mnist/service.yaml ``` Verify the service is running - ```bash + ``` kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ts-def NodePort 10.109.14.120 8080:30160/TCP,8081:30302/TCP 59m @@ -75,19 +75,19 @@ Run the commands given in following steps from the parent directory of the root We use kubectl port-forward to make the cluster accessible from the local machine. This will run in the background. Make sure to kill the process when the test is done. - ```bash + ``` kubectl port-forward svc/ts-def 8080:8080 8081:8081 & ``` ### Register the model on TorchServe using the above model archive file - ```bash + ``` curl -X POST "localhost:8081/models?model_name=mnist&url=mnist.mar&initial_workers=4" ``` If this succeeeds, you will see a message like below - ```bash + ``` { "status": "Model \"mnist\" Version: 1.0 registered with 4 initial workers" } @@ -95,7 +95,7 @@ Run the commands given in following steps from the parent directory of the root ### Run digit recognition inference - ```bash + ``` curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png ``` @@ -104,7 +104,7 @@ Run the commands given in following steps from the parent directory of the root ### Delete the cluster - ```bash + ``` minikube stop minikube delete ``` From a2bf2519d3d52cd6b0a5c684540d2f640eb00f1f Mon Sep 17 00:00:00 2001 From: agunapal Date: Mon, 24 Jul 2023 21:23:29 +0000 Subject: [PATCH 4/5] remove bash --- kubernetes/examples/mnist/MNIST.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kubernetes/examples/mnist/MNIST.md b/kubernetes/examples/mnist/MNIST.md index 2d42e8219b..4f08a2427f 100644 --- a/kubernetes/examples/mnist/MNIST.md +++ b/kubernetes/examples/mnist/MNIST.md @@ -14,16 +14,16 @@ Run the commands given in following steps from the parent directory of the root ### Create a torch model archive using the torch-model-archiver utility to archive the above files. - ``` - torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler examples/image_classifier/mnist/mnist_handler.py - ``` + ``` + torch-model-archiver --model-name mnist --version 1.0 --model-file examples/image_classifier/mnist/mnist.py --serialized-file examples/image_classifier/mnist/mnist_cnn.pt --handler examples/image_classifier/mnist/mnist_handler.py + ``` ### Move .mar file into model_store directory - ``` - mkdir model_store - mv mnist.mar model_store/ - ``` + ``` + mkdir model_store + mv mnist.mar model_store/ + ``` ### Start kubernetes cluster From f62679fd025d1b6136b78718ace02c8f80c34501 Mon Sep 17 00:00:00 2001 From: agunapal Date: Mon, 24 Jul 2023 21:28:59 +0000 Subject: [PATCH 5/5] formatting --- kubernetes/examples/mnist/MNIST.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/kubernetes/examples/mnist/MNIST.md b/kubernetes/examples/mnist/MNIST.md index 4f08a2427f..0a55b9ca23 100644 --- a/kubernetes/examples/mnist/MNIST.md +++ b/kubernetes/examples/mnist/MNIST.md @@ -48,7 +48,10 @@ Run the commands given in following steps from the parent directory of the root Make sure the pod is running ``` - $ kubectl get pods + kubectl get pods + ``` + shows the output + ``` NAME READY STATUS RESTARTS AGE ts-def-5c95fdfd57-m446t 1/1 Running 0 58m @@ -66,6 +69,10 @@ Run the commands given in following steps from the parent directory of the root ``` kubectl get svc + ``` + shows the output + ``` + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ts-def NodePort 10.109.14.120 8080:30160/TCP,8081:30302/TCP 59m @@ -95,9 +102,9 @@ Run the commands given in following steps from the parent directory of the root ### Run digit recognition inference - ``` - curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png - ``` + ``` + curl http://127.0.0.1:8080/predictions/mnist -T examples/image_classifier/mnist/test_data/0.png + ``` The output in this case will be a `0`