Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Python wrapper update and openvino example #342

Merged
merged 6 commits into from
Dec 13, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
622 changes: 81 additions & 541 deletions examples/models/openvino/openvino-squeezenet.ipynb

Large diffs are not rendered by default.

54 changes: 40 additions & 14 deletions examples/models/tfserving-mnist/tfserving-mnist.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,31 @@
"metadata": {},
"outputs": [
{
"name": "stderr",
"name": "stdout",
"output_type": "stream",
"text": [
"/home/clive/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
" from ._conv import register_converters as _register_converters\n"
"'../../../proto/tensorflow/tensorflow' -> './tensorflow'\r\n",
"'../../../proto/tensorflow/tensorflow/core' -> './tensorflow/core'\r\n",
"'../../../proto/tensorflow/tensorflow/core/framework' -> './tensorflow/core/framework'\r\n",
"'../../../proto/tensorflow/tensorflow/core/framework/types.proto' -> './tensorflow/core/framework/types.proto'\r\n",
"'../../../proto/tensorflow/tensorflow/core/framework/resource_handle.proto' -> './tensorflow/core/framework/resource_handle.proto'\r\n",
"'../../../proto/tensorflow/tensorflow/core/framework/tensor_shape.proto' -> './tensorflow/core/framework/tensor_shape.proto'\r\n",
"'../../../proto/tensorflow/tensorflow/core/framework/tensor.proto' -> './tensorflow/core/framework/tensor.proto'\r\n"
]
}
],
"source": [
"!mkdir -p proto\n",
"!cp ../../../proto/prediction.proto ./proto\n",
"!cp -vr ../../../proto/tensorflow/tensorflow .\n",
"!python -m grpc.tools.protoc -I./ --python_out=./ --grpc_python_out=./ ./proto/prediction.proto"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import requests\n",
Expand All @@ -42,12 +59,17 @@
"from tensorflow.examples.tutorials.mnist import input_data\n",
"import sys\n",
"sys.path.append(\"../../../notebooks\")\n",
"from visualizer import get_graph"
"from visualizer import get_graph\n",
"from proto import prediction_pb2\n",
"from proto import prediction_pb2_grpc\n",
"import grpc\n",
"import tensorflow as tf\n",
"from tensorflow.core.framework.tensor_pb2 import TensorProto"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -88,6 +110,17 @@
" print(response.status_code)\n",
" print(response.text)\n",
"\n",
"def grpc_request_internal(data,endpoint=\"localhost:5000\"):\n",
" datadef = prediction_pb2.DefaultData(\n",
" tftensor=tf.make_tensor_proto(data)\n",
" )\n",
"\n",
" request = prediction_pb2.SeldonMessage(data = datadef)\n",
" channel = grpc.insecure_channel(endpoint)\n",
" stub = prediction_pb2_grpc.ModelStub(channel)\n",
" response = stub.Predict(request=request)\n",
" return response\n",
"\n",
"\n",
"def gen_mnist_data(mnist):\n",
" batch_xs, batch_ys = mnist.train.next_batch(1)\n",
Expand All @@ -99,38 +132,31 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From <ipython-input-2-8029beac92e7>:7: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
"WARNING:tensorflow:From <ipython-input-5-d2f9601b3465>:7: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
"WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Please write your own downloading logic.\n",
"WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Please use urllib or similar directly.\n",
"Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
"WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Please use tf.data to implement this functionality.\n",
"Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
"Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
"WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Please use tf.data to implement this functionality.\n",
"Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
"WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Please use tf.one_hot on tensors.\n",
"Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
"Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
"Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
"Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
"WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"spec": {
"containers": [
{
"image": "seldonio/tfserving-proxy:0.1",
"image": "{{ .Values.tfserving_proxy.image }}",
"name": "tfserving-proxy"
},
{
Expand Down Expand Up @@ -72,7 +72,7 @@
}],
"graph": {
"name": "tfserving-proxy",
"endpoint": { "type" : "REST" },
"endpoint": { "type" : "GRPC" },
"type": "MODEL",
"children": [],
"parameters":
Expand Down
7 changes: 4 additions & 3 deletions helm-charts/seldon-openvino/values.yaml
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
openvino:
image: ie-serving-py:0.1
image: intelaipg/openvino-model-server:0.2
model:
path: /opt/ml/squeezenet
name: squeezenet1.1
input: data
output: prob
port: 8000
port: 8001
model_volume: hostPath

tfserving_proxy:
image: seldonio/tfserving-proxy:0.2
2 changes: 1 addition & 1 deletion integrations/tfserving/.s2i/environment
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
MODEL_NAME=TfServingProxy
API_TYPE=REST
API_TYPE=GRPC
SERVICE_TYPE=MODEL
PERSISTENCE=0
4 changes: 2 additions & 2 deletions integrations/tfserving/Makefile
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
IMAGE_VERSION=0.1
IMAGE_VERSION=0.2
IMAGE_NAME = docker.io/seldonio/tfserving-proxy

SELDON_CORE_DIR=../../..

.PHONY: build
build:
s2i build . seldonio/seldon-core-s2i-python3:0.3 $(IMAGE_NAME):$(IMAGE_VERSION)
s2i build . seldonio/seldon-core-s2i-python36:0.4-SNAPSHOT $(IMAGE_NAME):$(IMAGE_VERSION)

push_to_dockerhub:
docker push $(IMAGE_NAME):$(IMAGE_VERSION)
Expand Down
46 changes: 45 additions & 1 deletion integrations/tfserving/TfServingProxy.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@
from tensorflow.python.saved_model import signature_constants
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
from seldon_core.microservice import get_data_from_proto, array_to_grpc_datadef
from seldon_core.model_microservice import get_class_names
from seldon_core.proto import prediction_pb2, prediction_pb2_grpc

import requests
import json
import numpy as np

class TensorflowServerError(Exception):

Expand Down Expand Up @@ -36,14 +40,54 @@ def __init__(self,rest_endpoint=None,grpc_endpoint=None,model_name=None,signatur
self.signature_name = signature_name
self.model_input = model_input
self.model_output = model_output


# if we have a TFTensor message we got directly without converting the message otherwise we go the usual route
def predict_grpc(self,request):
print("Predict grpc called")
default_data_type = request.data.WhichOneof("data_oneof")
print(default_data_type)
if default_data_type == "tftensor":
tfrequest = predict_pb2.PredictRequest()
tfrequest.model_spec.name = self.model_name
tfrequest.model_spec.signature_name = self.signature_name
tfrequest.inputs[self.model_input].CopyFrom(request.data.tftensor)
result = self.stub.Predict(tfrequest)
print(result)
datadef = prediction_pb2.DefaultData(
tftensor=result.outputs[self.model_output]
)
return prediction_pb2.SeldonMessage(data=datadef)

else:
features = get_data_from_proto(request)
datadef = request.data
data_type = request.WhichOneof("data_oneof")
predictions = self.predict(features, datadef.names)

predictions = np.array(predictions)
if len(predictions.shape) > 1:
class_names = get_class_names(
self, predictions.shape[1])
else:
class_names = []

if data_type == "data":
default_data_type = request.data.WhichOneof("data_oneof")
else:
default_data_type = "tensor"
data = array_to_grpc_datadef(
predictions, class_names, default_data_type)
return prediction_pb2.SeldonMessage(data=data)



def predict(self,X,features_names):
if self.grpc:
request = predict_pb2.PredictRequest()
request.model_spec.name = self.model_name
request.model_spec.signature_name = self.signature_name
request.inputs[self.model_input].CopyFrom(tf.contrib.util.make_tensor_proto(X.tolist(), shape=X.shape))
print(request)
result = self.stub.Predict(request)
print(result)
response = numpy.array(result.outputs[self.model_output].float_val)
Expand Down
Loading