update triton to 21.08 (allowing for content-endocded payloads) (#3589)

* update triton to 21.08 * experiment with content-encoding * executor: escape html only when no encoding is present * undo force no compression * remove forgotten fmt.Println * Update client.go add in-code comment * typo fix * fix whitespace
SeldonIO · Sep 15, 2021 · 8988f5c · 8988f5c
1 parent e9a7baa
commit 8988f5c
Show file tree

Hide file tree

Showing 15 changed files with 56 additions and 28 deletions.
diff --git a/components/alibi-explain-server/Makefile b/components/alibi-explain-server/Makefile
@@ -210,7 +210,7 @@ test_models/triton/cifar10/tf_cifar10:
 
 
 anchor_images_triton_model: test_models/triton/cifar10/tf_cifar10
-	docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -p5001:5001 -v ${PWD}/test_models/triton/tf_cifar10:/models nvcr.io/nvidia/tritonserver:20.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --http-port=9000 --grpc-port=5001
+	docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -p5001:5001 -v ${PWD}/test_models/triton/tf_cifar10:/models nvcr.io/nvidia/tritonserver:21.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --http-port=9000 --grpc-port=5001
 
 anchor_images_triton_predict:
 	curl -H "Content-Type: application/json" http://0.0.0.0:9000/v2/models/cifar10/infer -d '@tests/data/truck-v2.json'
@@ -226,5 +226,3 @@ anchor_images_triton_docker:
 
 anchor_images_triton_explain:
 	curl -d @tests/data/truck-v2.json  -X POST http://localhost:8080/v2/models/cifar10/explain    -H "Content-Type: application/json"
-
-
diff --git a/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb b/components/drift-detection/nvidia-triton-cifar10/cifar10_drift.ipynb
@@ -210,7 +210,7 @@
     "    - metadata: {}\n",
     "      spec:\n",
     "        containers:\n",
-    "        - image: nvcr.io/nvidia/tritonserver:20.08-py3\n",
+    "        - image: nvcr.io/nvidia/tritonserver:21.08-py3\n",
     "          name: cifar10\n",
     "          resources:\n",
     "            limits:\n",

diff --git a/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb b/components/outlier-detection/nvidia-triton-cifar10/cifar10_outlier.ipynb
@@ -217,7 +217,7 @@
     "    - metadata: {}\n",
     "      spec:\n",
     "        containers:\n",
-    "        - image: nvcr.io/nvidia/tritonserver:20.08-py3\n",
+    "        - image: nvcr.io/nvidia/tritonserver:21.08-py3\n",
     "          name: cifar10\n",
     "          resources:\n",
     "            limits:\n",

diff --git a/examples/models/lightgbm_custom_server/iris.ipynb b/examples/models/lightgbm_custom_server/iris.ipynb
@@ -163,7 +163,7 @@
     "  TRITON_SERVER:\n",
     "    protocols:\n",
     "      kfserving:\n",
-    "        defaultImageVersion: \"20.08-py3\"\n",
+    "        defaultImageVersion: \"21.08-py3\"\n",
     "        image: nvcr.io/nvidia/tritonserver\n",
     "  TEMPO_SERVER:\n",
     "    protocols:\n",

diff --git a/executor/api/payload/bytes.go b/executor/api/payload/bytes.go
@@ -1,8 +1,9 @@
 package payload
 
 type BytesPayload struct {
-	Msg         []byte
-	ContentType string
+	Msg             []byte
+	ContentType     string
+	ContentEncoding string
 }
 
 func (s *BytesPayload) GetPayload() interface{} {
@@ -13,6 +14,10 @@ func (s *BytesPayload) GetContentType() string {
 	return s.ContentType
 }
 
+func (s *BytesPayload) GetContentEncoding() string {
+	return s.ContentEncoding
+}
+
 func (s *BytesPayload) GetBytes() ([]byte, error) {
 	return s.Msg, nil
 }
diff --git a/executor/api/payload/payload.go b/executor/api/payload/payload.go
@@ -3,5 +3,6 @@ package payload
 type SeldonPayload interface {
 	GetPayload() interface{}
 	GetContentType() string
+	GetContentEncoding() string
 	GetBytes() ([]byte, error)
 }
diff --git a/executor/api/payload/proto.go b/executor/api/payload/proto.go
@@ -18,6 +18,10 @@ func (s *ProtoPayload) GetContentType() string {
 	return APPLICATION_TYPE_PROTOBUF
 }
 
+func (s *ProtoPayload) GetContentEncoding() string {
+	return ""
+}
+
 func (s *ProtoPayload) GetBytes() ([]byte, error) {
 	data, err := proto.Marshal(s.Msg)
 	if err != nil {

diff --git a/executor/api/rest/client.go b/executor/api/rest/client.go
@@ -63,10 +63,20 @@ func (smc *JSONRestClient) Marshall(w io.Writer, msg payload.SeldonPayload) erro
 		return invalidPayload("couldn't convert to []byte")
 	}
 
-	var escaped bytes.Buffer
+	var err error
+	// When "Content-Encoding" header is not empty it means that payload is compressed.
+	// We do not want to modify it in this situation. This change was added during update of Triton
+	// image from 20.08 to 21.08 as new version allowed for gzip-encoded payloads.
+	// Related PR: https://github.com/SeldonIO/seldon-core/pull/3589
+	// More on this header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
+	if msg.GetContentEncoding() != "" {
+		_, err = w.Write(payload)
+	} else {
+		var escaped bytes.Buffer
 
-	json.HTMLEscape(&escaped, payload)
-	_, err := escaped.WriteTo(w)
+		json.HTMLEscape(&escaped, payload)
+		_, err = escaped.WriteTo(w)
+	}
 
 	return err
 }
@@ -164,21 +174,24 @@ func (smc *JSONRestClient) addHeaders(req *http.Request, m map[string][]string)
 	}
 }
 
-func (smc *JSONRestClient) doHttp(ctx context.Context, modelName string, method string, url *url.URL, msg []byte, meta map[string][]string, contentType string) ([]byte, string, error) {
+func (smc *JSONRestClient) doHttp(ctx context.Context, modelName string, method string, url *url.URL, msg []byte, meta map[string][]string, contentType string, contentEncoding string) ([]byte, string, string, error) {
 	smc.Log.V(1).Info("Calling HTTP", "URL", url)
 
 	var req *http.Request
 	var err error
 	if msg != nil {
 		req, err = http.NewRequest("POST", url.String(), bytes.NewBuffer(msg))
 		if err != nil {
-			return nil, "", err
+			return nil, "", "", err
 		}
 		req.Header.Set(http2.ContentType, contentType)
+		if contentEncoding != "" {
+			req.Header.Set("Content-Encoding", contentEncoding)
+		}
 	} else {
 		req, err = http.NewRequest("GET", url.String(), nil)
 		if err != nil {
-			return nil, "", err
+			return nil, "", "", err
 		}
 	}
 
@@ -205,24 +218,25 @@ func (smc *JSONRestClient) doHttp(ctx context.Context, modelName string, method
 
 	response, err := client.Do(req)
 	if err != nil {
-		return nil, "", err
+		return nil, "", "", err
 	}
 
 	//Read response
 	b, err := ioutil.ReadAll(response.Body)
 	if err != nil {
-		return nil, "", err
+		return nil, "", "", err
 	}
 	defer response.Body.Close()
 
 	contentTypeResponse := response.Header.Get(http2.ContentType)
+	contentEncodingResponse := response.Header.Get("Content-Encoding")
 
 	if response.StatusCode != http.StatusOK {
 		smc.Log.Info("httpPost failed", "response code", response.StatusCode)
 		err = &httpStatusError{StatusCode: response.StatusCode, Url: url}
 	}
 
-	return b, contentTypeResponse, err
+	return b, contentTypeResponse, contentEncodingResponse, err
 }
 
 func (smc *JSONRestClient) modifyMethod(method string, modelName string) string {
@@ -271,12 +285,14 @@ func (smc *JSONRestClient) call(ctx context.Context, modelName string, method st
 	}
 	var bytes []byte
 	var contentType = ContentTypeJSON
+	var contentEncoding = ""
 	if req != nil {
 		bytes = req.GetPayload().([]byte)
 		contentType = req.GetContentType()
+		contentEncoding = req.GetContentEncoding()
 	}
-	sm, contentType, err := smc.doHttp(ctx, modelName, method, &url, bytes, meta, contentType)
-	res := payload.BytesPayload{Msg: sm, ContentType: contentType}
+	sm, contentType, contentEncoding, err := smc.doHttp(ctx, modelName, method, &url, bytes, meta, contentType, contentEncoding)
+	res := payload.BytesPayload{Msg: sm, ContentType: contentType, ContentEncoding: contentEncoding}
 	return &res, err
 }
 

diff --git a/executor/api/rest/server.go b/executor/api/rest/server.go
@@ -4,11 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	http2 "github.com/cloudevents/sdk-go/pkg/bindings/http"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 
+	http2 "github.com/cloudevents/sdk-go/pkg/bindings/http"
+	"time"
+
 	"github.com/go-logr/logr"
 	"github.com/gorilla/mux"
 	"github.com/opentracing/opentracing-go"
@@ -22,7 +24,6 @@ import (
 	"github.com/seldonio/seldon-core/executor/predictor"
 	v1 "github.com/seldonio/seldon-core/operator/apis/machinelearning.seldon.io/v1"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
-	"time"
 )
 
 type SeldonRestApi struct {
@@ -76,6 +77,10 @@ func (r *SeldonRestApi) CreateHttpServer(port int) *http.Server {
 
 func (r *SeldonRestApi) respondWithSuccess(w http.ResponseWriter, code int, payload payload.SeldonPayload) {
 	w.Header().Set("Content-Type", payload.GetContentType())
+	contentEncoding := payload.GetContentEncoding()
+	if contentEncoding != "" {
+		w.Header().Set("Content-Encoding", contentEncoding)
+	}
 	w.WriteHeader(code)
 
 	err := r.Client.Marshall(w, payload)

diff --git a/executor/samples/local/kfserving/Makefile b/executor/samples/local/kfserving/Makefile
@@ -13,7 +13,7 @@ run_executor:
 
 
 run_triton_simple:
-	docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -p5001:5001 -v ${PWD}/triton-inference-server/docs/examples/model_repository:/models nvcr.io/nvidia/tritonserver:20.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --http-port=9000 --grpc-port=5001
+	docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -p5001:5001 -v ${PWD}/triton-inference-server/docs/examples/model_repository:/models nvcr.io/nvidia/tritonserver:21.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --http-port=9000 --grpc-port=5001
 
 curl_rest_triton:
 	curl -v -d '{"inputs":[{"name":"INPUT0","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]}]}'    -X POST http://0.0.0.0:9000/v2/models/simple/infer    -H "Content-Type: application/json"

diff --git a/helm-charts/seldon-core-operator/README.md b/helm-charts/seldon-core-operator/README.md
@@ -104,7 +104,7 @@ helm install seldon-core-operator seldonio/seldon-core-operator --namespace seld
 | predictor_servers.TENSORFLOW_SERVER.protocols.seldon.image | string | `"seldonio/tfserving-proxy"` |  |
 | predictor_servers.TENSORFLOW_SERVER.protocols.tensorflow.defaultImageVersion | string | `"2.1.0"` |  |
 | predictor_servers.TENSORFLOW_SERVER.protocols.tensorflow.image | string | `"tensorflow/serving"` |  |
-| predictor_servers.TRITON_SERVER.protocols.kfserving.defaultImageVersion | string | `"20.08-py3"` |  |
+| predictor_servers.TRITON_SERVER.protocols.kfserving.defaultImageVersion | string | `"21.08-py3"` |  |
 | predictor_servers.TRITON_SERVER.protocols.kfserving.image | string | `"nvcr.io/nvidia/tritonserver"` |  |
 | predictor_servers.XGBOOST_SERVER.protocols.kfserving.defaultImageVersion | string | `"0.3.2"` |  |
 | predictor_servers.XGBOOST_SERVER.protocols.kfserving.image | string | `"seldonio/mlserver"` |  |

diff --git a/helm-charts/seldon-core-operator/values.yaml b/helm-charts/seldon-core-operator/values.yaml
@@ -144,7 +144,7 @@ predictor_servers:
   TRITON_SERVER:
     protocols:
       kfserving:
-        defaultImageVersion: "20.08-py3"
+        defaultImageVersion: "21.08-py3"
         image: nvcr.io/nvidia/tritonserver
   TEMPO_SERVER:
     protocols:
@@ -201,4 +201,3 @@ engine:
 # Explainer image
 explainer:
   image: seldonio/alibiexplainer:1.11.0-dev
-
diff --git a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go
@@ -159,7 +159,7 @@ var configs = map[string]string{
           "protocols" : {
             "kfserving": {
               "image": "nvcr.io/nvidia/tritonserver",
-              "defaultImageVersion": "20.08-py3"
+              "defaultImageVersion": "21.08-py3"
               }
             }
         },

diff --git a/operator/config/manager/configmap.yaml b/operator/config/manager/configmap.yaml
@@ -69,7 +69,7 @@ data:
           "protocols" : {
             "kfserving": {
               "image": "nvcr.io/nvidia/tritonserver",
-              "defaultImageVersion": "20.08-py3"
+              "defaultImageVersion": "21.08-py3"
               }
             }
         },

diff --git a/operator/controllers/suite_test.go b/operator/controllers/suite_test.go
@@ -117,7 +117,7 @@ var configs = map[string]string{
           "protocols" : {
             "kfserving": {
               "image": "nvcr.io/nvidia/tritonserver",
-              "defaultImageVersion": "20.08-py3"
+              "defaultImageVersion": "21.08-py3"
               }
             }
         }