From 6f2bf4aef5cdd4ac561054e0016495b9e3638481 Mon Sep 17 00:00:00 2001 From: RafalSkolasinski Date: Mon, 10 Oct 2022 09:49:51 +0100 Subject: [PATCH] Add batch examples to docs (#481) * improve tritonclient rendering in docs * add basic batch examples --- .../contents/examples/batch-examples.md | 10 + docs/source/contents/examples/index.md | 5 +- .../examples/tritonclient-examples.md | 2 +- samples/batch-inputs/iris-input.txt | 100 ++ samples/batch-inputs/tfsimple-input.txt | 100 ++ samples/k8s-batch-examples.ipynb | 940 ++++++++++++++++++ samples/k8s-batch-examples.md | 630 ++++++++++++ samples/k8s-tritonclient.ipynb | 62 +- samples/k8s-tritonclient.md | 30 +- 9 files changed, 1819 insertions(+), 60 deletions(-) create mode 100644 docs/source/contents/examples/batch-examples.md create mode 100644 samples/batch-inputs/iris-input.txt create mode 100644 samples/batch-inputs/tfsimple-input.txt create mode 100644 samples/k8s-batch-examples.ipynb create mode 100644 samples/k8s-batch-examples.md diff --git a/docs/source/contents/examples/batch-examples.md b/docs/source/contents/examples/batch-examples.md new file mode 100644 index 0000000000..d30ee7fe17 --- /dev/null +++ b/docs/source/contents/examples/batch-examples.md @@ -0,0 +1,10 @@ +# Batch Inference Examples + +Requires `mlserver` to be installed. +```bash +pip install mlserver +``` + +```{include} ../../../../samples/k8s-batch-examples.md +:relative-images: +``` diff --git a/docs/source/contents/examples/index.md b/docs/source/contents/examples/index.md index 55e8d4d046..69e3bda459 100644 --- a/docs/source/contents/examples/index.md +++ b/docs/source/contents/examples/index.md @@ -7,11 +7,12 @@ This section will provide some examples to allow operations with Seldon to be te * [Kubernetes examples](k8s-examples.md) * [Pipeline examples](pipeline-examples.md) * [Explainer examples](explainer-examples.md) - + ## Making inference requests - * [Inference examples](inference.md) + * [Inference examples](inference.md) * [Tritonclient examples](tritonclient-examples.md) + * [Batch Inference examples](batch-examples.md) ## Advanced Examples diff --git a/docs/source/contents/examples/tritonclient-examples.md b/docs/source/contents/examples/tritonclient-examples.md index a6fe3b975f..d7be635ad6 100644 --- a/docs/source/contents/examples/tritonclient-examples.md +++ b/docs/source/contents/examples/tritonclient-examples.md @@ -1,7 +1,7 @@ # Tritonclient Examples with Seldon Core V2 To install `tritonclient` -``` +```bash pip install tritonclient[all] ``` diff --git a/samples/batch-inputs/iris-input.txt b/samples/batch-inputs/iris-input.txt new file mode 100644 index 0000000000..558a72113a --- /dev/null +++ b/samples/batch-inputs/iris-input.txt @@ -0,0 +1,100 @@ +{"inputs":[{"name":"predict","data":[0.38606369295833043,0.006894049558299753,0.6104082981607108,0.3958954239450676],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7223678219956075,0.608521741883582,0.8596266157372878,0.20041864827775757],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8659159480026418,0.2383384971368594,0.7743518759043038,0.8748919374334038],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9152051336644693,0.27993540543455764,0.6904393306346505,0.875081841850771],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6393741808897597,0.28776930847943205,0.8179214541861932,0.2018912125929685],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9789057670606082,0.4191508463498963,0.2559642168572732,0.8999249523370249],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8610541473062108,0.907574545740293,0.06499375833777932,0.08078813963652032],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5372892847566128,0.9976312929760024,0.9553473408487858,0.1949543182480774],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.4865732386991771,0.30935363903067636,0.2453995325295555,0.5304798433963821],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9056143883551074,0.5301349720404799,0.07308363225701764,0.37108341947533885],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8423238633886448,0.19188393097177325,0.11121043208086456,0.9901024789918798],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.1913540264284409,0.9942730446315622,0.06234004239446056,0.5397166126991605],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.195082010708175,0.6548161142352943,0.11041028803857944,0.7516058063670282],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9914096998076212,0.23776391929871088,0.21344358159382215,0.26982878353850126],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.47661488061439694,0.9233664603483619,0.3147068400593347,0.28718465984225827],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8641529206434768,0.5706142657304446,0.8480088165476329,0.6921125329180894],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.17958075530562145,0.3690868809329443,0.557737259675276,0.6924690844077797],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8589528360480404,0.20824804742003944,0.14054901879408066,0.7111615649418469],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.15302064887013844,0.5577910267456313,0.21799934621031036,0.7808200590175681],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.60629482830175,0.1368486483780853,0.9799215831018983,0.9889152928857359],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.3163535610490755,0.4120078792572368,0.8552112433498779,0.2743401933387335],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5684121237675275,0.860412227542184,0.16880956995576468,0.8636030334041114],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5127096310250071,0.6111415369833283,0.5399259607482558,0.7451415418283945],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.002176273967259279,0.9859294448867988,0.8898046270330662,0.580489513257745],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.1978418480209727,0.16974322051429824,0.9552345152231073,0.8898734319089532],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.19491129806592078,0.09488380284933773,0.8169134089011394,0.33902966085211517],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7620187418821339,0.33594317271131247,0.4065464515383411,0.7275050779823027],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9844702700710407,0.8593645579503563,0.46567565247637555,0.8038241429287354],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7332186253947849,0.6088604795524274,0.753642858054935,0.6914675250813718],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9875413734475786,0.3179427368327965,0.6974325416087662,0.32627803311569736],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5218468692779838,0.012415944299129067,0.6595695730142147,0.20291061040927294],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6653519433144672,0.24700736402350276,0.1677239670169981,0.5396892551315197],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.34984029448285003,0.7980968482504941,0.6963956851732028,0.8419829658039979],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7466016089942169,0.9902251338558882,0.0003629147493414697,0.3692780874248248],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6204467026123095,0.4280457774776102,0.6152525546614867,0.28117314100974455],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.4536015778198952,0.8141542353175207,0.7491197390800156,0.06479646267139294],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6454487776796001,0.2635086694302794,0.9794105412023086,0.043154176569525804],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.28112587914058174,0.36799976925032807,0.12823754042870716,0.5219592657177913],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.49997709293474324,0.9228333687658721,0.014165170343684697,0.4669833706876908],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.3665960413781233,0.3962815343621079,0.8844701086727828,0.3909465697992224],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5022310019653048,0.6250349801301431,0.6955696868025862,0.12748648715599942],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6847201116649027,0.43175685276921394,0.05925657346636892,0.1725294779258344],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7038178765098485,0.7879461076226656,0.337591071095799,0.4938925411646157],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.4417716777934191,0.38053445461385205,0.48669971311675075,0.10235881786049605],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.07035661931375015,0.8570792126647778,0.26863065016206644,0.7072615234194105],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.3694616419303608,0.9363401279936825,0.009118619760966284,0.5667140131096986],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.3832130836404646,0.6185692535170136,0.7022112392602075,0.8653710315666456],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8997941659309715,0.468459863273222,0.799182053897781,0.028860632195247593],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.756791960655705,0.8502276696964837,0.17591664131254248,0.8448641096896787],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.20353272718032922,0.050863503819587375,0.42670331335784994,0.7832970095500149],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5992742280607493,0.6258175052031845,0.9383093352100033,0.3310843672840219],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.46758666675076577,0.46581678147875727,0.725871094807289,0.28959325070001796],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6956042028358944,0.15432011948118152,0.24027193936935143,0.6790948768240024],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.41505181796602963,0.563971135039167,0.08938012251394922,0.5388462606021547],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9199247266331126,0.9536904456605775,0.6599175580343315,0.21177380643569965],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.28093080398418724,0.12324201832315684,0.17205326103692742,0.4611284632486674],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8300825403480663,0.04546942431874701,0.5001034139917033,0.6212061378135922],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.20355151801253557,0.5334886358588145,0.6037664082321031,0.9990294282339264],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6663423850382337,0.03781840572477435,0.45169193782649886,0.7033460191469194],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7532027834558215,0.8362472195938023,0.7396424525782871,0.21584130201339335],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.3129184837449204,0.8756526240491823,0.40040276516741613,0.2517536842270268],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.8336081031389777,0.07063888881460989,0.884303888760167,0.47530844684242135],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5375775460041416,0.25789357728604945,0.006868317183007511,0.30940198318853573],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.3253734285010659,0.4375733020182655,0.9983981907307194,0.26855367213740156],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.4595148793282956,0.5195304541637115,0.6929121397200447,0.14636060937781192],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5639811820210703,0.16524081923517375,0.7346681009293223,0.9560985461548064],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.713355840536083,0.7275364485396192,0.2480512997859734,0.5694749656693717],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5825440545727529,0.3152684199195409,0.5972017434584799,0.5668134735411832],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.17363569447423677,0.8787633582048012,0.871415703973967,0.1287333895351811],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.650430733863415,0.7548865685135818,0.8581350515733714,0.368960896848857],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.38837378768744624,0.4973249822855098,0.1654395198466202,0.45525611757936557],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.23588413280827292,0.41568041947166967,0.12155832290975965,0.18697964355550922],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.897299171783402,0.3736936886773248,0.23282938199211567,0.7302731651333509],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.23256735988754174,0.6940780627029551,0.33349256085471424,0.942892053404403],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6472318847834215,0.28849237211455436,0.7122636694929954,0.9838174542244505],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.05283804322755181,0.49641085087108394,0.9588481077337991,0.07514206147490332],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.31502572193084766,0.5893495362257134,0.47495961578864787,0.5099062169961317],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9392154791689156,0.5738187651494886,0.3920669437903239,0.1927744608639934],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.05705548175297326,0.04798849963313434,0.4025132147369185,0.9398452106311611],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6796980962010597,0.7398819328502394,0.19032016363038884,0.12874401279481718],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.4819290702442449,0.2261019247423084,0.701010410278158,0.7910491068960464],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9541711413815673,0.8601271257906501,0.17029059379951772,0.8826919198224622],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5225393387829881,0.9309464329216594,0.9209409109685891,0.07783663184578893],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.18766999512413973,0.07611438342127774,0.3629445642098531,0.45073734284126776],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7200249903416844,0.4703210334226733,0.9389598991479212,0.8534035541271356],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.3720666199676561,0.3423821387562581,0.10839908488356265,0.8523070312451344],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.6371058126741196,0.4191548728146638,0.2727578341435908,0.057484824535847356],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.09189989058205039,0.7363281204589441,0.10081799800905011,0.7352201152189689],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7225774671932079,0.5481723495814025,0.020207696676895925,0.9058787884629211],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.9937771803194357,0.38278702314745683,0.4492951453301063,0.5302987275029805],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.948298031734466,0.2852498978762198,0.703961822613879,0.5528951948664508],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.2229550308814664,0.5318161216993699,0.7761426019106473,0.316226374912127],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.33423668386584304,0.4878615753665906,0.30795251326572437,0.9134164932764093],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7321891225153738,0.4420208326103072,0.4525077171208032,0.7970362961463309],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.06026485984470065,0.2882809415064216,0.1359731641778732,0.1433459470447327],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.7516282608215267,0.3414638017260677,0.5608650901547423,0.562139966911385],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.058739630838841816,0.9724834893957689,0.016029803610963955,0.8964247339363989],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.07870622344915146,0.7677184054567261,0.14480821196659954,0.737129195795995],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.5850264206970843,0.4447833095936493,0.3735214673682151,0.5669535985852721],"datatype":"FP64","shape":[1,4]}]} +{"inputs":[{"name":"predict","data":[0.4641920065899603,0.4508934349125199,0.556170795521556,0.6653501298129617],"datatype":"FP64","shape":[1,4]}]} diff --git a/samples/batch-inputs/tfsimple-input.txt b/samples/batch-inputs/tfsimple-input.txt new file mode 100644 index 0000000000..686bda34e5 --- /dev/null +++ b/samples/batch-inputs/tfsimple-input.txt @@ -0,0 +1,100 @@ +{"inputs":[{"name":"INPUT0","data":[75,39,9,44,32,97,99,40,13,27,25,36,18,77,62,60],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[39,7,14,58,13,88,98,66,97,57,49,3,49,63,37,12],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[19,51,61,30,12,9,5,91,32,46,10,78,49,21,8,15],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[96,18,36,82,61,97,53,91,82,20,54,32,51,3,14,62],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[48,11,17,62,62,99,23,19,92,49,77,64,10,95,7,8],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[65,10,42,65,41,19,41,20,18,92,82,97,69,13,34,16],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[65,82,37,10,77,20,9,24,41,99,96,88,44,97,82,96],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[75,82,48,48,75,56,61,32,59,42,2,93,71,80,24,97],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[38,27,83,58,96,81,25,8,30,59,35,52,31,92,98,71],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[25,48,69,19,63,43,67,34,63,96,66,99,72,17,41,34],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[70,39,61,98,30,8,36,46,32,80,77,3,86,6,1,40],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[80,96,98,23,55,55,67,21,32,3,53,49,53,99,52,20],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[41,33,60,13,52,96,23,62,29,74,14,60,88,41,38,29],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[35,66,14,33,24,7,59,9,37,25,53,44,79,59,52,47],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[59,91,54,3,2,44,29,84,80,12,68,42,85,8,99,29],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[17,29,2,20,92,42,58,75,43,44,31,20,36,39,33,11],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[45,17,55,53,52,42,77,35,65,46,62,26,21,63,10,15],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[64,80,13,22,56,56,56,12,80,49,29,52,53,31,3,43],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[80,12,23,86,47,74,26,19,65,4,42,5,29,3,40,72],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[89,38,66,73,24,14,49,51,69,1,80,63,81,76,23,27],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[44,52,74,54,92,6,6,61,20,96,3,24,76,47,67,2],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[43,9,4,37,88,76,41,86,65,97,90,92,78,12,30,42],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[93,19,5,96,23,85,50,35,7,22,8,69,82,68,47,61],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[77,89,24,19,84,98,67,79,14,23,47,51,63,5,13,72],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[40,53,58,61,46,15,78,22,25,36,92,41,17,98,59,38],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[24,8,33,79,46,70,93,25,85,96,14,16,28,63,16,85],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[62,35,83,39,54,72,20,98,42,5,93,18,8,44,48,13],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[81,41,44,17,71,51,95,56,13,68,73,32,43,26,80,73],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[8,82,55,96,37,68,91,36,2,33,65,19,14,72,23,53],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[70,95,50,22,58,77,29,99,70,12,66,79,3,80,90,55],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[39,61,19,90,35,48,1,33,16,1,53,78,11,96,44,33],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[42,57,32,14,41,64,2,84,5,73,63,26,16,14,83,15],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[18,86,97,69,93,67,63,27,14,26,64,90,76,55,56,9],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[60,83,9,72,33,1,68,30,68,12,98,9,67,31,55,13],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[17,27,52,34,93,23,4,77,97,85,72,24,49,40,38,22],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[91,20,81,94,5,70,31,45,63,12,2,6,26,6,19,80],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[80,7,3,12,83,44,38,57,95,58,45,84,26,18,41,69],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[85,70,14,21,92,95,20,83,15,9,59,15,84,66,89,35],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[44,7,69,72,18,75,88,3,32,68,77,34,91,68,9,31],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[91,17,23,64,24,60,35,80,56,57,90,9,17,4,64,91],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[43,97,7,73,13,17,90,75,61,65,52,3,23,16,62,95],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[22,91,49,56,39,92,13,22,42,83,22,86,1,81,48,92],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[49,75,54,38,11,65,6,43,57,56,32,93,25,84,84,10],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[21,87,22,62,32,63,70,94,30,8,78,14,67,67,17,61],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[96,30,23,36,63,69,41,67,68,30,8,50,2,47,79,37],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[75,25,58,79,86,26,60,25,62,1,86,50,85,67,51,69],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[19,34,84,79,57,71,28,90,28,89,86,23,6,50,4,67],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[25,9,54,16,98,89,42,87,3,61,25,36,90,20,86,82],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[74,21,44,15,12,49,37,15,29,49,85,63,81,70,18,47],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[50,58,51,64,5,15,37,28,91,87,59,13,8,99,52,84],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[73,11,80,47,72,29,30,53,38,22,22,4,87,88,15,63],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[44,66,36,73,28,52,73,61,36,31,81,57,71,98,44,3],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[44,81,11,39,65,74,26,64,93,21,86,23,82,44,54,88],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[26,88,65,45,96,23,40,80,49,47,66,96,56,72,43,97],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[22,40,21,85,52,34,12,44,35,54,22,73,19,27,63,12],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[98,92,13,11,87,29,4,33,53,15,67,27,22,54,9,14],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[60,21,70,18,91,96,76,39,27,90,46,64,22,5,38,67],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[59,80,60,2,67,61,91,37,95,53,2,19,93,73,12,49],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[16,19,90,65,10,33,93,92,57,53,92,26,49,22,94,49],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[35,59,77,19,47,73,84,78,54,82,97,75,39,90,36,26],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[66,95,13,45,88,14,49,12,98,65,15,95,58,24,97,35],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[40,55,94,77,2,3,53,26,62,3,1,36,64,21,62,65],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[12,85,53,46,66,88,39,99,26,16,13,99,12,98,43,82],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[10,53,94,83,56,91,9,35,65,53,40,54,75,60,35,96],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[6,3,37,38,80,48,51,88,99,9,27,22,29,44,64,1],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[7,62,18,96,20,3,97,7,98,65,27,49,2,54,94,10],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[98,52,88,41,53,30,22,93,3,96,77,79,59,34,79,83],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[5,94,22,20,75,94,84,61,81,25,56,75,66,24,40,82],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[62,1,69,43,73,37,29,19,22,61,10,18,48,69,26,24],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[49,13,34,54,54,14,13,30,49,78,21,46,21,10,75,44],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[61,73,60,66,94,85,30,39,57,17,51,45,18,75,42,50],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[71,91,97,11,76,60,44,2,21,62,28,58,49,39,27,21],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[71,75,91,56,20,88,20,59,57,72,27,40,53,78,50,99],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[66,34,51,79,1,52,95,74,30,67,58,19,63,91,30,4],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[6,48,22,13,27,75,53,24,99,45,81,51,2,95,14,76],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[11,52,90,71,90,60,22,28,6,20,87,49,86,52,87,77],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[91,32,71,32,43,6,51,20,8,71,2,88,42,33,46,98],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[40,36,96,53,71,14,22,40,44,9,89,79,82,33,49,67],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[89,78,55,60,57,15,98,53,61,59,68,93,14,10,43,78],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[52,37,57,48,55,11,22,54,33,60,45,84,13,47,64,98],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[66,64,57,6,47,14,65,29,58,80,2,89,60,69,20,32],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[67,73,68,76,13,81,36,26,30,68,11,5,38,95,19,40],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[92,75,33,32,61,66,52,34,30,9,73,86,82,87,10,85],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[45,7,82,52,22,17,3,63,59,65,91,97,34,80,43,90],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[78,68,67,97,70,10,2,32,22,22,39,51,8,14,13,67],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[5,91,81,55,96,17,45,83,13,1,66,99,58,74,42,63],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[43,21,63,71,98,6,25,13,45,74,11,52,80,47,41,94],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[86,2,83,94,84,20,30,60,61,26,46,10,96,17,60,1],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[79,45,62,32,71,54,18,84,28,64,78,10,32,46,11,39],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[70,90,37,28,1,97,20,53,60,23,97,91,9,2,49,85],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[37,24,99,52,74,91,58,65,78,90,54,31,32,77,93,76],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[62,22,66,44,2,3,25,6,72,55,57,83,48,99,12,47],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[48,86,10,14,66,57,52,57,51,82,90,96,64,18,89,7],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[66,85,1,55,28,74,16,43,51,87,74,32,31,52,26,73],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[50,71,52,14,15,54,89,12,94,84,69,74,71,60,34,2],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[10,45,84,88,4,29,66,18,61,18,24,76,74,10,4,32],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[64,65,72,16,1,94,26,69,29,32,13,13,3,16,15,78],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[49,92,47,56,17,16,84,98,8,17,29,81,42,50,55,55],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[55,47,13,80,76,33,84,44,22,56,21,93,13,69,14,72],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[78,82,98,55,82,26,64,14,5,48,60,26,26,99,87,7],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[19,67,24,61,3,63,73,85,67,98,73,71,22,88,10,95],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[42,57,76,58,2,40,80,86,94,72,38,71,83,8,78,63],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[30,38,44,56,4,37,94,86,86,62,42,61,66,93,73,16],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[57,31,63,40,76,9,78,44,40,11,59,99,18,72,13,65],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[8,93,83,84,54,15,17,38,33,87,7,16,28,85,65,21],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[43,18,25,48,27,29,17,42,44,35,17,8,51,23,80,93],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[55,69,77,1,14,98,86,8,65,47,67,71,77,92,23,51],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[71,38,74,82,16,63,69,33,76,89,74,47,43,87,24,26],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[79,81,84,67,18,49,4,51,24,59,97,20,77,93,41,88],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[83,25,56,84,74,31,73,91,42,87,58,44,87,62,53,86],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[8,78,98,5,9,75,70,15,47,12,72,94,76,84,4,94],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[37,65,42,9,24,96,81,40,78,17,36,81,82,12,49,23],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[73,1,62,51,8,82,86,90,74,56,3,20,46,62,6,60],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[11,70,77,55,53,17,88,5,25,18,94,63,68,32,8,43],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[53,36,49,60,71,79,84,25,52,70,6,43,84,74,9,43],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[75,60,39,3,83,58,53,22,81,33,6,97,37,34,99,74],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[71,4,28,25,88,84,61,71,84,7,36,42,85,35,20,51],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[58,83,46,83,78,59,58,29,3,23,16,80,96,61,58,87],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[74,3,84,2,93,46,53,42,11,38,34,60,18,45,89,47],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[91,6,20,6,2,8,70,46,25,63,96,36,81,54,17,15],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[7,6,19,15,78,94,27,52,21,73,47,76,73,72,23,93],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[21,66,37,2,75,77,82,35,59,73,34,62,52,80,33,12],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[40,66,92,55,36,95,67,8,20,11,11,47,95,97,74,32],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[87,12,6,6,85,73,72,4,22,34,72,62,3,15,38,54],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[19,16,97,55,83,89,30,50,60,58,23,70,99,25,72,89],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[81,81,75,92,63,44,56,97,48,1,22,79,4,16,4,33],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[72,60,59,22,25,68,9,55,28,13,31,53,78,41,30,88],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[65,6,71,23,10,74,78,77,81,50,4,73,16,71,19,54],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[67,98,66,58,44,42,2,85,65,24,38,63,37,52,62,18],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[39,2,20,17,88,39,29,67,58,96,47,58,5,79,20,45],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[46,23,80,12,32,5,63,52,32,93,86,33,75,82,4,33],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[64,20,63,79,71,6,1,44,51,50,57,94,5,66,17,99],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[3,26,15,90,97,24,87,67,99,94,26,85,68,47,40,23],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[97,34,38,21,91,78,43,41,97,92,63,90,29,94,77,23],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[70,6,45,58,93,23,95,42,15,59,77,15,80,6,92,62],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[36,44,79,55,77,32,60,13,96,52,2,55,63,91,5,15],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[38,18,16,96,14,75,56,93,98,46,16,67,66,46,9,80],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[46,23,10,32,30,64,7,29,57,49,34,78,28,86,72,97],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[15,80,55,37,95,67,98,58,60,15,59,16,93,82,1,7],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[39,38,6,38,84,42,50,2,39,71,29,62,54,87,2,5],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[93,10,78,11,80,23,35,4,92,68,65,88,85,20,61,64],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[11,65,26,74,90,61,76,95,32,31,52,69,53,32,85,62],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[77,72,47,41,97,75,72,32,60,92,88,29,60,81,85,65],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[61,3,60,23,2,18,3,59,85,89,12,66,44,47,85,9],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[27,20,46,74,80,36,77,33,39,4,58,85,59,8,13,45],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[15,15,63,80,63,53,78,6,49,54,99,53,93,59,30,93],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[60,53,46,23,72,67,32,40,14,3,1,45,67,56,69,68],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[69,36,78,50,13,73,76,12,67,54,57,2,61,44,53,62],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[61,1,26,92,48,61,40,2,40,87,75,63,26,6,60,28],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[55,98,99,91,77,46,10,82,65,15,29,94,82,16,94,1],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[83,56,45,45,69,97,65,93,21,47,45,74,46,64,55,56],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[92,98,37,25,5,31,3,10,92,14,9,72,5,47,95,97],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[54,62,80,56,14,48,62,89,1,44,93,12,53,76,60,27],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[85,86,57,17,98,56,99,73,63,17,84,65,4,98,65,37],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[3,96,96,50,80,44,38,64,43,7,4,48,14,50,59,3],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[82,82,54,78,7,2,42,40,72,69,98,6,90,58,94,12],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[39,3,96,26,80,93,36,65,79,37,80,41,53,71,7,77],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[14,12,76,29,30,91,54,56,17,62,29,56,64,50,33,64],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[4,89,34,64,46,54,46,6,28,82,87,26,64,26,50,85],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[24,24,57,60,82,84,94,81,98,84,10,2,26,94,28,34],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[6,4,1,63,18,54,6,96,85,63,91,47,4,47,50,49],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[12,82,7,45,2,21,23,34,55,49,33,32,70,87,56,79],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[63,25,38,37,87,56,16,10,11,96,20,18,33,18,56,42],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[53,14,63,78,42,15,74,89,44,4,81,4,70,97,53,33],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[22,33,84,30,97,58,62,35,45,15,68,82,94,70,54,50],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[16,31,44,35,51,3,96,58,85,10,31,82,8,42,41,88],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[80,87,69,40,3,1,53,70,53,27,89,5,63,6,12,83],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[25,24,82,30,40,51,42,8,51,36,13,89,40,12,73,1],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[86,32,54,27,24,20,26,15,38,48,24,24,57,83,10,30],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[27,93,50,47,32,92,63,35,97,39,73,7,83,13,91,88],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[29,30,56,90,15,84,36,25,66,62,28,69,40,27,36,5],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[78,69,45,38,98,32,27,19,10,57,75,34,32,90,63,56],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[52,6,54,55,32,97,51,62,88,2,88,94,40,32,46,63],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[80,30,25,5,91,60,84,38,7,19,88,83,30,9,70,11],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[92,80,34,37,84,65,71,44,94,40,39,5,38,26,63,57],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[39,79,7,45,6,43,4,17,75,57,1,84,71,38,42,81],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[12,43,77,81,6,22,76,3,15,9,79,58,36,93,8,91],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[97,33,31,43,6,8,85,77,21,56,41,51,12,68,35,80],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[28,90,84,12,59,12,5,36,45,17,36,64,37,81,98,59],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[67,9,67,63,38,98,27,99,89,80,88,12,7,79,75,66],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[44,47,84,38,29,77,57,42,38,74,88,58,51,84,49,10],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[20,12,75,45,5,18,95,23,62,51,22,77,1,87,58,58],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[54,96,69,33,50,68,49,60,84,15,60,46,70,13,84,83],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[18,73,32,31,80,3,66,89,75,33,12,46,23,15,48,6],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[72,45,25,83,32,88,90,27,37,33,23,37,14,70,13,68],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[28,84,2,72,79,99,71,50,75,41,5,66,43,16,88,59],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[97,48,68,93,28,41,53,48,98,51,74,59,2,65,38,94],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[98,97,77,70,58,43,19,23,96,62,69,48,24,76,18,63],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[39,49,85,56,15,58,61,39,71,55,95,34,80,75,76,90],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[44,80,13,29,32,42,36,20,62,27,50,65,75,69,49,40],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[81,59,33,88,85,1,20,39,62,53,77,14,92,80,45,40],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[23,60,33,1,10,56,78,2,58,7,82,83,52,22,34,17],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[31,85,61,96,47,78,48,31,93,75,91,4,23,29,98,49],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[67,43,90,54,38,1,90,73,71,52,51,39,93,84,83,51],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[91,39,39,67,88,65,97,52,57,7,30,80,79,20,68,97],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[16,39,27,44,93,10,58,65,19,5,74,57,88,90,12,23],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[76,36,67,4,75,3,55,94,34,10,90,34,92,75,99,54],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[6,77,8,9,26,70,12,5,50,88,85,19,2,15,51,82],"datatype":"INT32","shape":[1,16]}]} +{"inputs":[{"name":"INPUT0","data":[29,3,92,17,63,73,63,15,8,34,71,69,42,29,54,89],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[7,49,9,81,41,30,91,95,60,51,6,89,15,87,68,91],"datatype":"INT32","shape":[1,16]}]} diff --git a/samples/k8s-batch-examples.ipynb b/samples/k8s-batch-examples.ipynb new file mode 100644 index 0000000000..462fc72095 --- /dev/null +++ b/samples/k8s-batch-examples.ipynb @@ -0,0 +1,940 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3b70cb38-a0a0-4f0a-9d77-eafa92301338", + "metadata": {}, + "source": [ + "### Seldon V2 Batch Examples" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "3d069ec2-50a0-4fd3-b65e-d178cb7b142d", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"NAMESPACE\"] = \"seldon\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "93d79639-2651-4b83-bbcf-404e772f1f3e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'172.19.255.12'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "MESH_IP=!kubectl get svc seldon-mesh -n ${NAMESPACE} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'\n", + "MESH_IP=MESH_IP[0]\n", + "import os\n", + "os.environ['MESH_IP'] = MESH_IP\n", + "MESH_IP" + ] + }, + { + "cell_type": "markdown", + "id": "c1098488-4929-4df4-b3f9-8e448e87bcec", + "metadata": {}, + "source": [ + "## Deploy Models and Pipelines" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "77169ee9-896f-4ee3-b154-b79997d71925", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "apiVersion: mlops.seldon.io/v1alpha1\n", + "kind: Model\n", + "metadata:\n", + " name: iris\n", + "spec:\n", + " storageUri: \"gs://seldon-models/mlserver/iris\"\n", + " requirements:\n", + " - sklearn\n", + " memory: 100Ki\n" + ] + } + ], + "source": [ + "!cat models/sklearn-iris-gs.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e5653b7f-b8f5-4f08-ac2c-a26775ddf027", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "apiVersion: mlops.seldon.io/v1alpha1\n", + "kind: Pipeline\n", + "metadata:\n", + " name: iris-pipeline\n", + "spec:\n", + " steps:\n", + " - name: iris\n", + " output:\n", + " steps:\n", + " - iris\n" + ] + } + ], + "source": [ + "!cat pipelines/iris.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "3c51fc8b-3164-4e1a-8616-452f62d61d22", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "apiVersion: mlops.seldon.io/v1alpha1\n", + "kind: Model\n", + "metadata:\n", + " name: tfsimple1\n", + "spec:\n", + " storageUri: \"gs://seldon-models/triton/simple\"\n", + " requirements:\n", + " - tensorflow\n", + " memory: 100Ki\n" + ] + } + ], + "source": [ + "!cat models/tfsimple1.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "992a0404-e1d6-4d55-a963-2cdebd42231c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "apiVersion: mlops.seldon.io/v1alpha1\n", + "kind: Pipeline\n", + "metadata:\n", + " name: tfsimple\n", + "spec:\n", + " steps:\n", + " - name: tfsimple1\n", + " output:\n", + " steps:\n", + " - tfsimple1\n" + ] + } + ], + "source": [ + "!cat pipelines/tfsimple.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "b4f7affb-5418-4209-b665-4478482ea3dd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model.mlops.seldon.io/iris unchanged\n", + "pipeline.mlops.seldon.io/iris-pipeline unchanged\n", + "model.mlops.seldon.io/tfsimple1 unchanged\n", + "pipeline.mlops.seldon.io/tfsimple unchanged\n" + ] + } + ], + "source": [ + "!kubectl apply -f models/sklearn-iris-gs.yaml -n ${NAMESPACE}\n", + "!kubectl apply -f pipelines/iris.yaml -n ${NAMESPACE}\n", + "\n", + "!kubectl apply -f models/tfsimple1.yaml -n ${NAMESPACE}\n", + "!kubectl apply -f pipelines/tfsimple.yaml -n ${NAMESPACE}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "66b6b1f3-08bc-4212-be47-9a1df0b18348", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model.mlops.seldon.io/iris condition met\n", + "model.mlops.seldon.io/tfsimple condition met\n", + "model.mlops.seldon.io/tfsimple1 condition met\n", + "pipeline.mlops.seldon.io/iris-pipeline condition met\n", + "pipeline.mlops.seldon.io/tfsimple condition met\n", + "pipeline.mlops.seldon.io/tfsimple-pipeline condition met\n", + "model.mlops.seldon.io/tfsimple1 condition met\n", + "pipeline.mlops.seldon.io/tfsimple condition met\n" + ] + } + ], + "source": [ + "!kubectl wait --for condition=ready --timeout=300s model --all -n ${NAMESPACE}\n", + "!kubectl wait --for condition=ready --timeout=300s pipelines --all -n ${NAMESPACE}\n", + "\n", + "!kubectl wait --for condition=ready --timeout=300s model tfsimple1 -n ${NAMESPACE}\n", + "!kubectl wait --for condition=ready --timeout=300s pipelines tfsimple -n ${NAMESPACE}" + ] + }, + { + "cell_type": "markdown", + "id": "2c8e501b-fd01-47f6-a211-7de81942d754", + "metadata": {}, + "source": [ + "## Test Predictions" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "5b2678d3-a5cb-4b41-ad5b-3f29a7a1202d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"model_name\": \"iris_1\",\n", + " \"model_version\": \"1\",\n", + " \"id\": \"5b88fb3f-b467-4c36-b172-0ca81ecfab2e\",\n", + " \"parameters\": {\n", + " \"content_type\": null,\n", + " \"headers\": null\n", + " },\n", + " \"outputs\": [\n", + " {\n", + " \"name\": \"predict\",\n", + " \"shape\": [\n", + " 1\n", + " ],\n", + " \"datatype\": \"INT64\",\n", + " \"parameters\": null,\n", + " \"data\": [\n", + " 2\n", + " ]\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "!seldon model infer iris --inference-host ${MESH_IP}:80 \\\n", + " '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' | jq -M ." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "31701fb2-1185-415c-a0f8-b05ab89ae2d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"model_name\": \"\",\n", + " \"outputs\": [\n", + " {\n", + " \"data\": [\n", + " 2\n", + " ],\n", + " \"name\": \"predict\",\n", + " \"shape\": [\n", + " 1\n", + " ],\n", + " \"datatype\": \"INT64\"\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], + "source": [ + "!seldon pipeline infer iris-pipeline --inference-host ${MESH_IP}:80 \\\n", + " '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' | jq -M ." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e7fd9ec3-fd25-48ad-bf5d-90231805cd01", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"model_name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"tfsimple_1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"model_version\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"outputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"OUTPUT0\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m2\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m4\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m6\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m8\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m12\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m14\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m18\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m20\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m22\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m24\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m26\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m28\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m30\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m32\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!seldon model infer tfsimple --inference-host ${MESH_IP}:80 \\\n", + " '{\"outputs\":[{\"name\":\"OUTPUT0\"}], \"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9473915b-0ec4-457c-b304-1ba50e9cd78d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"model_name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"outputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m2\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m4\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m6\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m8\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m12\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m14\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m18\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m20\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m22\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m24\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m26\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m28\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m30\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m32\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"OUTPUT0\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"OUTPUT1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!seldon pipeline infer tfsimple-pipeline --inference-host ${MESH_IP}:80 \\\n", + " '{\"outputs\":[{\"name\":\"OUTPUT0\"}], \"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}' | jq ." + ] + }, + { + "cell_type": "markdown", + "id": "aecd8d62-2802-4e1d-bd1e-78978f3d4972", + "metadata": {}, + "source": [ + "## MLServer Iris Batch Job" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "9e9bee26-3c1c-44b0-9889-587767cf933e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"inputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"predict\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m0.38606369295833043\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.006894049558299753\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.6104082981607108\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.3958954239450676\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"FP64\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m4\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!cat batch-inputs/iris-input.txt | head -n 1 | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "a3b60a8f-80b5-4803-82b9-9d68605480d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-10-10 08:40:18,854 [mlserver] INFO - Using asyncio event-loop policy: uvloop\n", + "2022-10-10 08:40:18,857 [mlserver] INFO - Server url: 172.19.255.12\n", + "2022-10-10 08:40:18,857 [mlserver] INFO - input file path: batch-inputs/iris-input.txt\n", + "2022-10-10 08:40:18,857 [mlserver] INFO - output file path: /tmp/iris-output.txt\n", + "2022-10-10 08:40:19,066 [mlserver] INFO - Time taken: 0.21 seconds\n" + ] + } + ], + "source": [ + "%%bash\n", + "mlserver infer -u ${MESH_IP} -m iris -i batch-inputs/iris-input.txt -o /tmp/iris-output.txt --workers 5" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "d3b7ef85-6435-4041-9857-2f843091118a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-10-10 08:40:20,849 [mlserver] INFO - Using asyncio event-loop policy: uvloop\n", + "2022-10-10 08:40:20,852 [mlserver] INFO - Server url: 172.19.255.12\n", + "2022-10-10 08:40:20,852 [mlserver] INFO - input file path: batch-inputs/iris-input.txt\n", + "2022-10-10 08:40:20,852 [mlserver] INFO - output file path: /tmp/iris-pipeline-output.txt\n", + "2022-10-10 08:40:21,307 [mlserver] INFO - Time taken: 0.45 seconds\n" + ] + } + ], + "source": [ + "%%bash\n", + "mlserver infer -u ${MESH_IP} -m iris-pipeline.pipeline -i batch-inputs/iris-input.txt -o /tmp/iris-pipeline-output.txt --workers 5" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "2fd223f2-9b91-47c2-99a6-e0f16879b44e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"model_name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"iris_1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"model_version\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"08359080-6fe2-4429-8c37-cc6276f4340b\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"content_type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;30mnull\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"headers\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;30mnull\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"batch_index\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m0\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"outputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"predict\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT64\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;30mnull\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!cat /tmp/iris-output.txt | head -n 1 | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "c14358ae-6206-4f67-84e6-2baa6a54c7db", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"model_name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"outputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m0\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"predict\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT64\"\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"batch_index\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m0\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!cat /tmp/iris-pipeline-output.txt | head -n 1 | jq ." + ] + }, + { + "cell_type": "markdown", + "id": "b881fc4f-6e57-4164-9d8d-d174662a43ec", + "metadata": {}, + "source": [ + "## Triton TFSimple Batch Job" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "5ae725ef-bf07-42f7-8c57-d4e99ae99a87", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"inputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INPUT0\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m75\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m39\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m9\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m44\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m32\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m97\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m99\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m40\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m13\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m27\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m25\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m36\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m18\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m77\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m62\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m60\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INPUT1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m39\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m7\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m14\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m58\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m13\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m88\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m98\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m66\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m97\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m57\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m49\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m3\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m49\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m63\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m37\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m12\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!cat batch-inputs/tfsimple-input.txt | head -n 1 | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "5c5f360a-0c2c-4759-b30d-6e907aa5c75e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-10-10 08:40:23,543 [mlserver] INFO - Using asyncio event-loop policy: uvloop\n", + "2022-10-10 08:40:23,546 [mlserver] INFO - Server url: 172.19.255.12\n", + "2022-10-10 08:40:23,546 [mlserver] INFO - input file path: batch-inputs/tfsimple-input.txt\n", + "2022-10-10 08:40:23,546 [mlserver] INFO - output file path: /tmp/tfsimple-output.txt\n", + "2022-10-10 08:40:23,733 [mlserver] INFO - Time taken: 0.19 seconds\n" + ] + } + ], + "source": [ + "%%bash\n", + "mlserver infer -u ${MESH_IP} -m tfsimple -i batch-inputs/tfsimple-input.txt -o /tmp/tfsimple-output.txt --workers 5" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "753bc0ad-2fa0-47d8-a5cc-428cba326806", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-10-10 08:40:25,397 [mlserver] INFO - Using asyncio event-loop policy: uvloop\n", + "2022-10-10 08:40:25,400 [mlserver] INFO - Server url: 172.19.255.12\n", + "2022-10-10 08:40:25,400 [mlserver] INFO - input file path: batch-inputs/tfsimple-input.txt\n", + "2022-10-10 08:40:25,400 [mlserver] INFO - output file path: /tmp/tfsimple-pipeline-output.txt\n", + "2022-10-10 08:40:25,752 [mlserver] INFO - Time taken: 0.35 seconds\n" + ] + } + ], + "source": [ + "%%bash\n", + "mlserver infer -u ${MESH_IP} -m tfsimple-pipeline.pipeline -i batch-inputs/tfsimple-input.txt -o /tmp/tfsimple-pipeline-output.txt --workers 5" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "0f8a3641-c8a3-41c5-af4b-2647655fb10d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"id\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"168eef39-0510-4729-b8df-3d0ec063df55\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"model_name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"tfsimple_1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"model_version\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"outputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"OUTPUT0\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m114\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m46\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m23\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m102\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m45\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m185\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m197\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m106\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m110\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m84\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m74\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m39\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m67\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m140\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m99\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m72\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"OUTPUT1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m36\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m32\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-5\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-14\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m19\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m9\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-26\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-84\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-30\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-24\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m33\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-31\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m14\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m25\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m48\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"batch_index\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m0\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!cat /tmp/tfsimple-output.txt | head -n 1 | jq ." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "d17f03fa-08bc-42aa-95e8-8427e609e045", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"model_name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"outputs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m114\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m46\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m23\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m102\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m45\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m185\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m197\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m106\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m110\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m84\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m74\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m39\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m67\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m140\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m99\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m72\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"OUTPUT0\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m36\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m32\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-5\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-14\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m19\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m9\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-26\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-84\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-30\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-24\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m33\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-31\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m14\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m25\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m48\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"OUTPUT1\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"shape\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[0;39m1\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m16\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"datatype\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"INT32\"\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"parameters\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"batch_index\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m0\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m\n" + ] + } + ], + "source": [ + "!cat /tmp/tfsimple-pipeline-output.txt | head -n 1 | jq ." + ] + }, + { + "cell_type": "markdown", + "id": "bfd8d138-4825-4a06-a395-bfe332dbfb8e", + "metadata": {}, + "source": [ + "## Cleanup" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "b8d20578-5e4a-445a-a947-f09c8327033c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model.mlops.seldon.io \"iris\" deleted\n", + "pipeline.mlops.seldon.io \"iris-pipeline\" deleted\n" + ] + } + ], + "source": [ + "!kubectl delete -f models/sklearn-iris-gs.yaml -n ${NAMESPACE}\n", + "!kubectl delete -f pipelines/iris.yaml -n ${NAMESPACE}" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "5de7287b-577f-409f-af56-03aac9ec1f2d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model.mlops.seldon.io \"tfsimple1\" deleted\n", + "pipeline.mlops.seldon.io \"tfsimple\" deleted\n" + ] + } + ], + "source": [ + "!kubectl delete -f models/tfsimple1.yaml -n ${NAMESPACE}\n", + "!kubectl delete -f pipelines/tfsimple.yaml -n ${NAMESPACE}" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "mlserver-dev", + "language": "python", + "name": "mlserver-dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/samples/k8s-batch-examples.md b/samples/k8s-batch-examples.md new file mode 100644 index 0000000000..7a76d86e3e --- /dev/null +++ b/samples/k8s-batch-examples.md @@ -0,0 +1,630 @@ +### Seldon V2 Batch Examples + + +```python +import os +os.environ["NAMESPACE"] = "seldon" +``` + + +```python +MESH_IP=kubectl get svc seldon-mesh -n ${NAMESPACE} -o jsonpath='{.status.loadBalancer.ingress[0].ip}' +MESH_IP=MESH_IP[0] +import os +os.environ['MESH_IP'] = MESH_IP +MESH_IP +``` +```bash + + + + '172.19.255.12' +``` + +## Deploy Models and Pipelines + + +```bash +cat models/sklearn-iris-gs.yaml +``` +```yaml + apiVersion: mlops.seldon.io/v1alpha1 + kind: Model + metadata: + name: iris + spec: + storageUri: "gs://seldon-models/mlserver/iris" + requirements: + - sklearn + memory: 100Ki +``` + +```bash +cat pipelines/iris.yaml +``` +```yaml + apiVersion: mlops.seldon.io/v1alpha1 + kind: Pipeline + metadata: + name: iris-pipeline + spec: + steps: + - name: iris + output: + steps: + - iris +``` + +```bash +cat models/tfsimple1.yaml +``` +```yaml + apiVersion: mlops.seldon.io/v1alpha1 + kind: Model + metadata: + name: tfsimple1 + spec: + storageUri: "gs://seldon-models/triton/simple" + requirements: + - tensorflow + memory: 100Ki +``` + +```bash +cat pipelines/tfsimple.yaml +``` +```yaml + apiVersion: mlops.seldon.io/v1alpha1 + kind: Pipeline + metadata: + name: tfsimple + spec: + steps: + - name: tfsimple1 + output: + steps: + - tfsimple1 +``` + +```bash +kubectl apply -f models/sklearn-iris-gs.yaml -n ${NAMESPACE} +kubectl apply -f pipelines/iris.yaml -n ${NAMESPACE} + +kubectl apply -f models/tfsimple1.yaml -n ${NAMESPACE} +kubectl apply -f pipelines/tfsimple.yaml -n ${NAMESPACE} +``` +```json + + model.mlops.seldon.io/iris unchanged + pipeline.mlops.seldon.io/iris-pipeline unchanged + model.mlops.seldon.io/tfsimple1 unchanged + pipeline.mlops.seldon.io/tfsimple unchanged +``` + +```bash +kubectl wait --for condition=ready --timeout=300s model --all -n ${NAMESPACE} +kubectl wait --for condition=ready --timeout=300s pipelines --all -n ${NAMESPACE} + +kubectl wait --for condition=ready --timeout=300s model tfsimple1 -n ${NAMESPACE} +kubectl wait --for condition=ready --timeout=300s pipelines tfsimple -n ${NAMESPACE} +``` +```json + + model.mlops.seldon.io/iris condition met + model.mlops.seldon.io/tfsimple condition met + model.mlops.seldon.io/tfsimple1 condition met + pipeline.mlops.seldon.io/iris-pipeline condition met + pipeline.mlops.seldon.io/tfsimple condition met + pipeline.mlops.seldon.io/tfsimple-pipeline condition met + model.mlops.seldon.io/tfsimple1 condition met + pipeline.mlops.seldon.io/tfsimple condition met +``` +## Test Predictions + + +```bash +seldon model infer iris --inference-host ${MESH_IP}:80 \ + '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' | jq -M . +``` +```json + + { + "model_name": "iris_1", + "model_version": "1", + "id": "5b88fb3f-b467-4c36-b172-0ca81ecfab2e", + "parameters": { + "content_type": null, + "headers": null + }, + "outputs": [ + { + "name": "predict", + "shape": [ + 1 + ], + "datatype": "INT64", + "parameters": null, + "data": [ + 2 + ] + } + ] + } +``` + +```bash +seldon pipeline infer iris-pipeline --inference-host ${MESH_IP}:80 \ + '{"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]}' | jq -M . +``` +```json + + { + "model_name": "", + "outputs": [ + { + "data": [ + 2 + ], + "name": "predict", + "shape": [ + 1 + ], + "datatype": "INT64" + } + ] + } +``` + +```bash +seldon model infer tfsimple --inference-host ${MESH_IP}:80 \ + '{"outputs":[{"name":"OUTPUT0"}], "inputs":[{"name":"INPUT0","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]}]}' | jq . +``` +```json + + { + "model_name": "tfsimple_1", + "model_version": "1", + "outputs": [ + { + "name": "OUTPUT0", + "datatype": "INT32", + "shape": [ + 1, + 16 + ], + "data": [ + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32 + ] + } + ] + } +``` + +```bash +seldon pipeline infer tfsimple-pipeline --inference-host ${MESH_IP}:80 \ + '{"outputs":[{"name":"OUTPUT0"}], "inputs":[{"name":"INPUT0","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]}]}' | jq . +``` +```json + + { + "model_name": "", + "outputs": [ + { + "data": [ + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32 + ], + "name": "OUTPUT0", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + }, + { + "data": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "name": "OUTPUT1", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + } + ] + } +``` +## MLServer Iris Batch Job + + +```bash +cat batch-inputs/iris-input.txt | head -n 1 | jq . +``` +```yaml + { + "inputs": [ + { + "name": "predict", + "data": [ + 0.38606369295833043, + 0.006894049558299753, + 0.6104082981607108, + 0.3958954239450676 + ], + "datatype": "FP64", + "shape": [ + 1, + 4 + ] + } + ] + } +``` + +```bash +%%bash +mlserver infer -u ${MESH_IP} -m iris -i batch-inputs/iris-input.txt -o /tmp/iris-output.txt --workers 5 +``` +```bash + 2022-10-10 08:40:18,854 [mlserver] INFO - Using asyncio event-loop policy: uvloop + 2022-10-10 08:40:18,857 [mlserver] INFO - Server url: 172.19.255.12 + 2022-10-10 08:40:18,857 [mlserver] INFO - input file path: batch-inputs/iris-input.txt + 2022-10-10 08:40:18,857 [mlserver] INFO - output file path: /tmp/iris-output.txt + 2022-10-10 08:40:19,066 [mlserver] INFO - Time taken: 0.21 seconds +``` + +```bash +%%bash +mlserver infer -u ${MESH_IP} -m iris-pipeline.pipeline -i batch-inputs/iris-input.txt -o /tmp/iris-pipeline-output.txt --workers 5 +``` +```bash + 2022-10-10 08:40:20,849 [mlserver] INFO - Using asyncio event-loop policy: uvloop + 2022-10-10 08:40:20,852 [mlserver] INFO - Server url: 172.19.255.12 + 2022-10-10 08:40:20,852 [mlserver] INFO - input file path: batch-inputs/iris-input.txt + 2022-10-10 08:40:20,852 [mlserver] INFO - output file path: /tmp/iris-pipeline-output.txt + 2022-10-10 08:40:21,307 [mlserver] INFO - Time taken: 0.45 seconds +``` + +```bash +cat /tmp/iris-output.txt | head -n 1 | jq . +``` +```yaml + { + "model_name": "iris_1", + "model_version": "1", + "id": "08359080-6fe2-4429-8c37-cc6276f4340b", + "parameters": { + "content_type": null, + "headers": null, + "batch_index": 0 + }, + "outputs": [ + { + "name": "predict", + "shape": [ + 1 + ], + "datatype": "INT64", + "parameters": null, + "data": [ + 0 + ] + } + ] + } +``` + +```bash +cat /tmp/iris-pipeline-output.txt | head -n 1 | jq . +``` +```yaml + { + "model_name": "", + "outputs": [ + { + "data": [ + 0 + ], + "name": "predict", + "shape": [ + 1 + ], + "datatype": "INT64" + } + ], + "parameters": { + "batch_index": 0 + } + } +``` +## Triton TFSimple Batch Job + + +```bash +cat batch-inputs/tfsimple-input.txt | head -n 1 | jq . +``` +```yaml + { + "inputs": [ + { + "name": "INPUT0", + "data": [ + 75, + 39, + 9, + 44, + 32, + 97, + 99, + 40, + 13, + 27, + 25, + 36, + 18, + 77, + 62, + 60 + ], + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "INPUT1", + "data": [ + 39, + 7, + 14, + 58, + 13, + 88, + 98, + 66, + 97, + 57, + 49, + 3, + 49, + 63, + 37, + 12 + ], + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ] + } +``` + +```bash +%%bash +mlserver infer -u ${MESH_IP} -m tfsimple -i batch-inputs/tfsimple-input.txt -o /tmp/tfsimple-output.txt --workers 5 +``` +```bash + 2022-10-10 08:40:23,543 [mlserver] INFO - Using asyncio event-loop policy: uvloop + 2022-10-10 08:40:23,546 [mlserver] INFO - Server url: 172.19.255.12 + 2022-10-10 08:40:23,546 [mlserver] INFO - input file path: batch-inputs/tfsimple-input.txt + 2022-10-10 08:40:23,546 [mlserver] INFO - output file path: /tmp/tfsimple-output.txt + 2022-10-10 08:40:23,733 [mlserver] INFO - Time taken: 0.19 seconds +``` + +```bash +%%bash +mlserver infer -u ${MESH_IP} -m tfsimple-pipeline.pipeline -i batch-inputs/tfsimple-input.txt -o /tmp/tfsimple-pipeline-output.txt --workers 5 +``` +```bash + 2022-10-10 08:40:25,397 [mlserver] INFO - Using asyncio event-loop policy: uvloop + 2022-10-10 08:40:25,400 [mlserver] INFO - Server url: 172.19.255.12 + 2022-10-10 08:40:25,400 [mlserver] INFO - input file path: batch-inputs/tfsimple-input.txt + 2022-10-10 08:40:25,400 [mlserver] INFO - output file path: /tmp/tfsimple-pipeline-output.txt + 2022-10-10 08:40:25,752 [mlserver] INFO - Time taken: 0.35 seconds +``` + +```bash +cat /tmp/tfsimple-output.txt | head -n 1 | jq . +``` +```yaml + { + "id": "168eef39-0510-4729-b8df-3d0ec063df55", + "model_name": "tfsimple_1", + "model_version": "1", + "outputs": [ + { + "name": "OUTPUT0", + "datatype": "INT32", + "shape": [ + 1, + 16 + ], + "parameters": {}, + "data": [ + 114, + 46, + 23, + 102, + 45, + 185, + 197, + 106, + 110, + 84, + 74, + 39, + 67, + 140, + 99, + 72 + ] + }, + { + "name": "OUTPUT1", + "datatype": "INT32", + "shape": [ + 1, + 16 + ], + "parameters": {}, + "data": [ + 36, + 32, + -5, + -14, + 19, + 9, + 1, + -26, + -84, + -30, + -24, + 33, + -31, + 14, + 25, + 48 + ] + } + ], + "parameters": { + "batch_index": 0 + } + } +``` + +```bash +cat /tmp/tfsimple-pipeline-output.txt | head -n 1 | jq . +``` +```yaml + { + "model_name": "", + "outputs": [ + { + "data": [ + 114, + 46, + 23, + 102, + 45, + 185, + 197, + 106, + 110, + 84, + 74, + 39, + 67, + 140, + 99, + 72 + ], + "name": "OUTPUT0", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + }, + { + "data": [ + 36, + 32, + -5, + -14, + 19, + 9, + 1, + -26, + -84, + -30, + -24, + 33, + -31, + 14, + 25, + 48 + ], + "name": "OUTPUT1", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + } + ], + "parameters": { + "batch_index": 0 + } + } +``` +## Cleanup + + +```bash +kubectl delete -f models/sklearn-iris-gs.yaml -n ${NAMESPACE} +kubectl delete -f pipelines/iris.yaml -n ${NAMESPACE} +``` +```json + + model.mlops.seldon.io "iris" deleted + pipeline.mlops.seldon.io "iris-pipeline" deleted +``` + +```bash +kubectl delete -f models/tfsimple1.yaml -n ${NAMESPACE} +kubectl delete -f pipelines/tfsimple.yaml -n ${NAMESPACE} +``` +```json + + model.mlops.seldon.io "tfsimple1" deleted + pipeline.mlops.seldon.io "tfsimple" deleted +``` \ No newline at end of file diff --git a/samples/k8s-tritonclient.ipynb b/samples/k8s-tritonclient.ipynb index b1daed1bd2..98e3398087 100644 --- a/samples/k8s-tritonclient.ipynb +++ b/samples/k8s-tritonclient.ipynb @@ -183,13 +183,6 @@ "print(\"model metadata:\", http_triton_client.get_model_metadata(\"iris\"))" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Against Model" - ] - }, { "cell_type": "code", "execution_count": 8, @@ -207,6 +200,8 @@ } ], "source": [ + "# Against model\n", + "\n", "binary_data = False\n", "\n", "inputs = [httpclient.InferInput(\"predict\", (1, 4), \"FP64\")]\n", @@ -218,13 +213,6 @@ "result.as_numpy(\"predict\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Against Pipeline" - ] - }, { "cell_type": "code", "execution_count": 9, @@ -242,6 +230,8 @@ } ], "source": [ + "# Agaist pipeline\n", + "\n", "binary_data = False\n", "\n", "inputs = [httpclient.InferInput(\"predict\", (1, 4), \"FP64\")]\n", @@ -271,7 +261,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## With Tritonserver" + "## With Tritonserver\n", + "\n", + "- Note: binary data support in HTTP is blocked by https://github.com/SeldonIO/seldon-core-v2/issues/475" ] }, { @@ -404,13 +396,6 @@ "print(\"model metadata:\", http_triton_client.get_model_metadata(\"iris\"))" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Against Model" - ] - }, { "cell_type": "code", "execution_count": 15, @@ -429,6 +414,8 @@ } ], "source": [ + "# Against model (no binary data)\n", + "\n", "binary_data = False\n", "\n", "inputs = [\n", @@ -463,6 +450,8 @@ } ], "source": [ + "# Against model (with binary data)\n", + "\n", "binary_data = True\n", "\n", "inputs = [\n", @@ -479,13 +468,6 @@ "result.as_numpy(\"OUTPUT0\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Against Pipeline" - ] - }, { "cell_type": "code", "execution_count": 17, @@ -504,6 +486,8 @@ } ], "source": [ + "# Against Pipeline (no binary data)\n", + "\n", "binary_data = False\n", "\n", "inputs = [\n", @@ -526,7 +510,7 @@ "metadata": {}, "outputs": [], "source": [ - "# binary data does not work with http behind pipeline: no opened issue yet\n", + "## binary data does not work with http behind pipeline\n", "\n", "# import numpy as np\n", "\n", @@ -618,13 +602,6 @@ "print(grpc_triton_client.get_model_metadata(model_name, headers=headers))" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Against Model" - ] - }, { "cell_type": "code", "execution_count": 21, @@ -643,6 +620,8 @@ } ], "source": [ + "# Against Model\n", + "\n", "model_name = \"tfsimple1\"\n", "headers = {\"seldon-model\": model_name}\n", "\n", @@ -660,13 +639,6 @@ "result.as_numpy(\"OUTPUT0\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Against Pipeline" - ] - }, { "cell_type": "code", "execution_count": 22, @@ -685,6 +657,8 @@ } ], "source": [ + "# Against Pipeline\n", + "\n", "model_name = \"tfsimple.pipeline\"\n", "headers = {\"seldon-model\": model_name}\n", "\n", diff --git a/samples/k8s-tritonclient.md b/samples/k8s-tritonclient.md index 27837c274a..179be41e9a 100644 --- a/samples/k8s-tritonclient.md +++ b/samples/k8s-tritonclient.md @@ -100,10 +100,10 @@ print("model metadata:", http_triton_client.get_model_metadata("iris")) model ready: True model metadata: {'name': 'iris_1', 'versions': [], 'platform': '', 'inputs': [], 'outputs': [], 'parameters': {'content_type': None, 'headers': None}} ``` -#### Against Model - ```python +# Against model + binary_data = False inputs = [httpclient.InferInput("predict", (1, 4), "FP64")] @@ -121,10 +121,10 @@ result.as_numpy("predict") array([2]) ``` -#### Against Pipeline - ```python +# Agaist pipeline + binary_data = False inputs = [httpclient.InferInput("predict", (1, 4), "FP64")] @@ -148,6 +148,8 @@ result.as_numpy("predict") ## With Tritonserver +- Note: binary data support in HTTP is blocked by https://github.com/SeldonIO/seldon-core-v2/issues/475 + ### Deploy Model and Pipeline @@ -220,10 +222,10 @@ print("model metadata:", http_triton_client.get_model_metadata("iris")) model ready: True model metadata: {'name': 'iris_1', 'versions': [], 'platform': '', 'inputs': [], 'outputs': [], 'parameters': {'content_type': None, 'headers': None}} ``` -#### Against Model - ```python +# Against model (no binary data) + binary_data = False inputs = [ @@ -249,6 +251,8 @@ result.as_numpy("OUTPUT0") ```python +# Against model (with binary data) + binary_data = True inputs = [ @@ -272,10 +276,10 @@ result.as_numpy("OUTPUT0") dtype=int32) ``` -#### Against Pipeline - ```python +# Against Pipeline (no binary data) + binary_data = False inputs = [ @@ -301,7 +305,7 @@ result.as_numpy("OUTPUT0") ```python -# binary data does not work with http behind pipeline: no opened issue yet +## binary data does not work with http behind pipeline # import numpy as np @@ -374,10 +378,10 @@ print(grpc_triton_client.get_model_metadata(model_name, headers=headers)) } ``` -#### Against Model - ```python +# Against Model + model_name = "tfsimple1" headers = {"seldon-model": model_name} @@ -402,10 +406,10 @@ result.as_numpy("OUTPUT0") dtype=int32) ``` -#### Against Pipeline - ```python +# Against Pipeline + model_name = "tfsimple.pipeline" headers = {"seldon-model": model_name}