From ae6828cf3f337e49b0efa546b4c78cda8cad6490 Mon Sep 17 00:00:00 2001 From: Ankush Agarwal Date: Wed, 7 Mar 2018 09:27:38 -0800 Subject: [PATCH 1/4] Create a end-to-end kubeflow example using seq2seq model (3/n) * Create a simple tornado server to serve the model * TODO: Create a docker image for the server and deploy on kubeflow Related to https://github.com/kubeflow/examples/issues/11 --- github_issue_summarization/README.md | 2 +- .../notebooks/server.py | 54 +++++++++++++++++++ .../serving_the_model.md | 19 +++++++ 3 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 github_issue_summarization/notebooks/server.py create mode 100644 github_issue_summarization/serving_the_model.md diff --git a/github_issue_summarization/README.md b/github_issue_summarization/README.md index 122581dfc..1e8b84151 100644 --- a/github_issue_summarization/README.md +++ b/github_issue_summarization/README.md @@ -21,7 +21,7 @@ By the end of this tutorial, you should learn how to: datasets * Train a Sequence-to-Sequence model using TensorFlow on the cluster using GPUs -* Serve the model using TensorFlow Serving +* Serve the model a Tornado Server ## Steps: diff --git a/github_issue_summarization/notebooks/server.py b/github_issue_summarization/notebooks/server.py new file mode 100644 index 000000000..f007af22d --- /dev/null +++ b/github_issue_summarization/notebooks/server.py @@ -0,0 +1,54 @@ +from __future__ import print_function + +import logging + +import tornado.web +from tornado import gen +from tornado.options import define, options, parse_command_line +from keras.models import load_model +import dill as dpickle +from seq2seq_utils import Seq2Seq_Inference + +define("port", default=8888, help="run on the given port", type=int) +define("instances_key", default='instances', help="requested instances json object key") + + +class PredictHandler(tornado.web.RequestHandler): + @gen.coroutine + def post(self): + request_key = self.settings['request_key'] + request_data = tornado.escape.json_decode(self.request.body) + model = self.settings['model'] + predictions = [model.generate_issue_title(body)[1] for body in request_data[request_key]] + self.write(dict(predictions=predictions)) + + +class IndexHandler(tornado.web.RequestHandler): + def get(self): + self.write('Hello World') + + +def main(): + parse_command_line() + with open('body_pp.dpkl', 'rb') as f: + body_pp = dpickle.load(f) + with open('title_pp.dpkl', 'rb') as f: + title_pp = dpickle.load(f) + model = Seq2Seq_Inference(encoder_preprocessor=body_pp, + decoder_preprocessor=title_pp, + seq2seq_model=load_model('seq2seq_model_tutorial.h5')) + app = tornado.web.Application( + [ + (r"/predict", PredictHandler), + (r"/", IndexHandler), + ], + xsrf_cookies=False, + request_key=options.instances_key, + model=model) + app.listen(options.port) + logging.info('running at http://localhost:%s' % options.port) + tornado.ioloop.IOLoop.current().start() + + +if __name__ == "__main__": + main() diff --git a/github_issue_summarization/serving_the_model.md b/github_issue_summarization/serving_the_model.md new file mode 100644 index 000000000..528b416f1 --- /dev/null +++ b/github_issue_summarization/serving_the_model.md @@ -0,0 +1,19 @@ +# Serving the model + +We are going to use a simple tornado server to serve the model. The [server.py](notebooks/server.py) contains the server code. + +Start the server using `python server.py --port=8888`. + +## Sample request + +``` +curl -X POST -H 'Content-Type: application/json' -d '{"instances": ["issue overview add a new property to disable detection of image stream files those ended with -is.yml from target directory. expected behaviour by default cube should not process image stream files if user does not set it. current behaviour cube always try to execute -is.yml files which can cause some problems in most of cases, for example if you are using kuberentes instead of openshift or if you use together fabric8 maven plugin with cube"]}' http://localhost:8888/predict +``` + +## Sample response + +``` +{"predictions": ["add a new property to disable detection"]} +``` + +Next: [Teardown](teardown.md) From 5f741ed851db91e10c5df8e9e842ac8447d605b1 Mon Sep 17 00:00:00 2001 From: Ankush Agarwal Date: Wed, 7 Mar 2018 09:32:59 -0800 Subject: [PATCH 2/4] README update --- github_issue_summarization/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/github_issue_summarization/README.md b/github_issue_summarization/README.md index 1e8b84151..83e4c146d 100644 --- a/github_issue_summarization/README.md +++ b/github_issue_summarization/README.md @@ -21,10 +21,11 @@ By the end of this tutorial, you should learn how to: datasets * Train a Sequence-to-Sequence model using TensorFlow on the cluster using GPUs -* Serve the model a Tornado Server +* Serve the model using a Tornado Server ## Steps: 1. [Setup a Kubeflow cluster](setup_a_kubeflow_cluster.md) 1. [Training the model](training_the_model.md) +1. [Serving the model](serving_the_model.md) 1. [Teardown](teardown.md) From 910a15d2581e2df636d547ff1c305acc4b8848bf Mon Sep 17 00:00:00 2001 From: Ankush Agarwal Date: Wed, 7 Mar 2018 15:32:50 -0800 Subject: [PATCH 3/4] Add comment --- github_issue_summarization/serving_the_model.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/github_issue_summarization/serving_the_model.md b/github_issue_summarization/serving_the_model.md index 528b416f1..54ee3716c 100644 --- a/github_issue_summarization/serving_the_model.md +++ b/github_issue_summarization/serving_the_model.md @@ -4,6 +4,8 @@ We are going to use a simple tornado server to serve the model. The [server.py]( Start the server using `python server.py --port=8888`. +> The model is written in Keras and when exported as a TensorFlow model seems to be incompatible with TensorFlow Serving. So we're using our own webserver to serve this model. + ## Sample request ``` From ae774e9658739900972b749c6357f00883ed164b Mon Sep 17 00:00:00 2001 From: Ankush Agarwal Date: Wed, 7 Mar 2018 15:34:30 -0800 Subject: [PATCH 4/4] Link to issue --- github_issue_summarization/serving_the_model.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_issue_summarization/serving_the_model.md b/github_issue_summarization/serving_the_model.md index 54ee3716c..3b6dc4d7f 100644 --- a/github_issue_summarization/serving_the_model.md +++ b/github_issue_summarization/serving_the_model.md @@ -4,7 +4,7 @@ We are going to use a simple tornado server to serve the model. The [server.py]( Start the server using `python server.py --port=8888`. -> The model is written in Keras and when exported as a TensorFlow model seems to be incompatible with TensorFlow Serving. So we're using our own webserver to serve this model. +> The model is written in Keras and when exported as a TensorFlow model seems to be incompatible with TensorFlow Serving. So we're using our own webserver to serve this model. More details [here](https://github.com/kubeflow/examples/issues/11#issuecomment-371005885). ## Sample request