Added Tensorflow model REST protocol test on triton for Modelmesh

ods_ci/tests/Resources/Files/triton/modelmesh-triton-tensorflow-output.json

@@ -0,0 +1,7 @@
+{"model_name":"inceptiongraphdef__isvc-3d47a57850","model_version":"1","outputs":[{"name":"InceptionV3/Predictions/Softmax","datatype":"FP32","shape":[1,1001],"data":[0.00017835239,0.000102207334,0.00040144665,0.00053033314,0.00027971406,0.0013423931,0.00035822403,0.0005146323,0.00068280095,0.0005593356,0.0006294884,0.00083396374,0.00078253815,0.0013488036,0.001376984,0.0012715475,0.0023802875,0.0008721544,0.00068400276,0.0067927954,0.0005489907,0.0009313915,0.0026896393,0.0019404711,0.0008952451,0.0005062338,0.0003050709,0.00013300705,0.0005231623,0.00022160984,0.00030028666,0.00029445975,0.0003046454,0.000092344344,0.00020437122,0.00075666956,0.00010319233,0.0001324062,0.00018883395,0.00024938607,0.00019521586,0.00031935133,0.00017681718,0.0004550522,0.00018139159,0.0002587802,0.00018249966,0.0005547497,0.0002967668,0.00028402125,0.00007612907,0.00033937016,0.00014225116,0.00078431075,0.00083683984,0.000370249,0.0008498151,0.00013691763,0.00022890454,0.0002641032,0.0015006971,0.00053243374,0.0000907384,0.00018857156,0.0009986189,0.0005585813,0.00068764354,0.0002741844,0.0005079193,0.00036508933,0.00020865294,0.0003651198,0.002425264,0.00015305843,0.0006082568,0.00008815172,0.00018912724,0.00025719462,0.0011470666,0.005355532,0.0015475369,0.00028846355,0.0007858467,0.0005904269,0.00014465109,0.00047894707,0.001544611,0.00036520264,0.00035897215,0.0005380435,0.0008711086,0.00041960916,0.0013282995,0.005573367,0.0005016874,0.0022453011,0.0005089283,0.0012538801,0.00048011617,0.00036048194,0.00086849695,0.00023785683,0.000085305524,0.000147455,0.00026433982,0.00048668866,0.000051320465,0.000071503346,0.0015093589,0.00025220847,0.00007458099,0.0003256829,0.01768877,0.0013401258,0.0008828955,0.0009820414,0.00009175115,0.00023021524,0.0010776042,0.000074073185,0.00022550095,0.00043988987,0.0001422142,0.00011354866,0.00011207844,0.00028180832,0.00069873285,0.0014339413,0.00061056926,0.0010022998,0.00080171385,0.0005640231,0.0011954595,0.0008176829,0.00093923154,0.0006816383,0.000403282,0.0008311831,0.0007086835,0.00023295381,0.0020728235,0.0006494798,0.0013178162,0.0004405929,0.0014927381,0.00037015334,0.00040886833,0.0007335298,0.00019913877,0.00011503936,0.0002871042,0.0001830359,0.00030646005,0.00015972432,0.00023312216,0.00013499522,0.00021743796,0.00017426591,0.00014629841,0.00019824128,0.00009409763,0.00006008714,0.00025801244,0.00035659526,0.00023508102,0.00036738874,0.00029072392,0.00019926035,0.00007201355,0.00016318183,0.00024176168,0.00013716123,0.00034282508,0.000244012,0.00041990817,0.00008925026,0.000075463686,0.00008040648,0.00026947507,0.0004879551,0.00005132869,0.00020642264,0.00018678093,0.000062778905,0.00008810902,0.00020598323,0.00027764216,0.00020435112,0.0003086493,0.00020994239,0.00018400357,0.00024319401,0.00023872619,0.000041000807,0.00035233732,0.000077463956,0.000738902,0.0001775999,0.00005067854,0.00007615074,0.00037568333,0.00007790807,0.000522395,0.000053426083,0.00027413393,0.00017606771,0.00024204752,0.00014987995,0.00032702586,0.0006044253,0.00010395334,0.00009053333,0.00049105485,0.00005918142,0.0001769739,0.00010154179,0.00026478033,0.00013854016,0.00024305627,0.00011937214,0.000060964383,0.0002696135,0.0002063026,0.00010198593,0.0006219903,0.0008454653,0.00008211383,0.000028383754,0.0003089983,0.00020034878,0.000093728544,0.000051650073,0.0001640213,0.00009212404,0.000055484947,0.00021427503,0.00029559954,0.00022805137,0.00028024128,0.00027840285,0.0000566063,0.00007180165,0.00011418179,0.000051459356,0.00006817839,0.000051230123,0.00013504595,0.00020312706,0.00017861207,0.00009802688,0.00028527458,0.00013910547,0.00033508116,0.000042119696,0.00013549863,0.00023382978,0.000024864481,0.00013506644,0.00012762065,0.000117501055,0.0001501146,0.00016458068,0.00015489469,0.000057573627,0.0002557536,0.00008909825,0.00046674616,0.0002495526,0.00013226138,0.00013817348,0.00029219376,0.00039512216,0.00013842329,0.0001844859,0.00014447675,0.00044776473,0.00027444927,0.00053243095,0.0002416364,0.0001307533,0.00017635315,0.00025219683,0.00025972057,0.00028857114,0.00032476266,0.00018019999,0.0003804509,0.0000777012,0.00005080189,0.00018974548,0.00030944892,0.00008445143,0.0003789483,0.000078942736,0.00021748069,0.00007060281,0.00021786887,0.00030860235,0.0003202483,0.000106123,0.0006324393,0.00046078366,0.0006578997,0.00037099535,0.00033067126,0.00022917136,0.0012496089,0.0011680174,0.0006907965,0.00085849164,0.00027547035,0.0025329872,0.0008017742,0.0021440159,0.0003289842,0.0014165195,0.001093865,0.00035744972,0.0006003339,0.0006349742,0.0018614361,0.00035786262,0.00028580957,0.00016161149,0.00015772911,0.0002939121,0.00020385168,0.0009963117,0.00082697434,0.0005679312,0.0000899606,0.00019148916,0.0006266442,0.00019463057,0.00013829496,0.00029031083,0.00011121606,0.00013282616,0.00018648601,0.0001475045,0.00015517115,0.00021347476,0.000027480966,0.000054556585,0.00021445155,0.000080955695,0.00017839433,0.00008893356,0.00010984412,0.000080668,0.000316906,0.00032236427,0.0007483713,0.000076792734,0.00014704984,0.00017764038,0.00008033428,0.00032332688,0.0002447767,0.00013141322,0.000103172104,0.00009258914,0.00012251739,0.0001292211,0.00021459917,0.00008703628,0.00025172715,0.00009795908,0.00014371832,0.00005924153,0.00019493791,0.000057968322,0.000033556695,0.00012716457,0.000067935136,0.000038219292,0.00016343943,0.00015513571,0.000056901492,0.00007170315,0.00007143774,0.00008145567,0.00010362868,0.00009668509,0.000099624616,0.00007192738,0.00010685247,0.000043279433,0.00009158636,0.00007130372,0.000094594434,0.00015705163,0.00022545407,0.00012575476,0.00014849206,0.000223194,0.00015139495,0.00030354728,0.00028608923,0.0013192215,0.001403842,0.00020806883,0.0003691022,0.0040364834,0.0013921959,0.0006633193,0.0031842294,0.000045517027,0.0002168302,0.00018141218,0.0030451696,0.00009154793,0.00031590316,0.00024416772,0.0048474,0.0002763712,0.000029041588,0.0033493885,0.0043727234,0.0030661768,0.002369332,0.0011442924,0.0028893303,0.00032274573,0.00014774835,0.00011483302,0.00038201534,0.0010366994,0.00092214404,0.00038107543,0.0008838411,0.00035160253,0.00009349593,0.0008556403,0.00019950203,0.0001864901,0.00028345303,0.0003067845,0.0013818095,0.00030355828,0.00028524632,0.0007447381,0.0002495489,0.0009296579,0.00140108,0.00016300015,0.00016811701,0.00036982857,0.0008958007,0.0005342791,0.000106154635,0.000063036576,0.001741675,0.00039008766,0.000078749785,0.00012124701,0.0012847964,0.00065030163,0.00022374769,0.0002483791,0.00025212177,0.00065681076,0.00024165599,0.0018948171,0.00025228088,0.0004775899,0.00069639744,0.00024676663,0.000038350732,0.001200436,0.00080211036,0.00075345434,0.0015152233,0.00035428008,0.004112556,0.0001951495,0.00061645353,0.000098992445,0.00017631438,0.00078945793,0.00060797605,0.00024320038,0.00046861276,0.0002277647,0.00050608197,0.00017617916,0.0005553847,0.0011908142,0.00051734917,0.0028355892,0.00069059565,0.0003349687,0.0010733765,0.00029804767,0.000050256953,0.0009810476,0.000029564575,0.0002786734,0.000933873,0.00038728403,0.0022665972,0.00011026162,0.0002744439,0.0017284655,0.00023476018,0.0008836608,0.00084331725,0.0006392365,0.0010762424,0.001603201,0.0003066153,0.000711892,0.000108833665,0.0058265585,0.0014563833,0.0017395327,0.00059482025,0.00076831126,0.010042664,0.00032482494,0.000107811065,0.00013973106,0.00006957281,0.0010428235,0.00067793654,0.00019815093,0.00021008508,0.00014761792,0.00071482203,0.0004916591,0.00025937488,0.0034239723,0.0002571528,0.00011311816,0.00028388776,0.0010397222,0.00018629963,0.00022421911,0.00008409767,0.0006989828,0.0033818313,0.0020780212,0.0007760042,0.004544874,0.0025679986,0.0002976773,0.0053171357,0.0015232351,0.00022509959,0.00015130446,0.012206049,0.0004276749,0.00063501147,0.00023649135,0.00025754547,0.00042028737,0.00015487059,0.0009435575,0.0011027067,0.001248472,0.00045270938,0.00032269617,0.00024420524,0.0022358012,0.0041488735,0.000046255183,0.00041421753,0.00077277294,0.0011817658,0.000050995975,0.0003016126,0.00050453894,0.00016893902,0.0008354603,0.0004457178,0.011148509,0.00013389454,0.00040217626,0.0010041584,0.00031786386,0.0004971236,0.00013705167,0.00021471013,0.000016915965,0.0010742393,0.0008020687,0.0004456241,0.00015775084,0.003618529,0.0002232187,0.00085707597,0.0002508674,0.00043864024,0.00056102884,0.001899861,0.00090584805,0.000266974,0.0023265022,0.0018649801,0.00023662444,0.0007380255,0.037261456,0.0003623819,0.0013669459,0.000067756875,0.0053892056,0.0006254379,0.0012615741,0.00095487334,0.0012872609,0.00011640068,0.000729834,0.0013817401,0.0003444225,0.002406554,0.0006917861,0.00015708405,0.0020407005,0.000105629,0.0035581756,0.0013645202,0.0020981357,0.00046341922,0.0051837927,0.002179016,0.00004320092,0.00023630266,0.002490797,0.00039311152,0.0006941348,0.0036199505,0.0026996436,0.0007925497,0.00095659913,0.00428512,0.00007312392,0.0023936334,0.00014196947,0.00032312793,0.00010818772,0.00014008758,0.0004528977,0.0020267086,0.00080763653,0.000982342,0.013259826,0.00021091447,0.0002048899,0.00027469697,0.000017142662,0.00007675652,0.007975964,0.00018975479,0.00025617323,0.00033124097,0.0001821004,0.000093883355,0.0003571206,0.0058551086,0.0002476898,0.000296414,0.00006894638,0.00006747801,0.0008592365,0.0000695606,0.0045379023,0.00009836251,0.0017430192,0.00028716825,0.00054618163,0.00007301163,0.00013585511,0.000094328,0.00016924411,0.00127435,0.0023144803,0.00012559193,0.00031413935,0.0069125583,0.00028249391,0.0010946925,0.00054600096,0.0013013397,0.0010795998,0.0011773141,0.0014056392,0.00082018244,0.0016551036,0.00010772124,0.0006380802,0.0002971639,0.000082640116,0.00054090447,0.0005578009,0.0009261426,0.00017377482,0.0016182435,0.0036994962,0.0001870288,0.00049401197,0.0013906906,0.00048411812,0.001834066,0.00079706236,0.00035098742,0.0021795172,0.00021023741,0.00013284414,0.00009983805,0.00038627133,0.00028792143,0.0025772077,0.0018581677,0.0005829069,0.00014343714,0.0020163471,0.00033858992,0.00018256005,0.00011751299,0.0007108326,0.0015287012,0.0013338513,0.00022580814,0.0033763312,0.0023593293,0.00043880322,0.0006182527,0.00064747385,0.00037535938,0.00076103705,0.00024393381,0.00046148224,0.0045932718,0.0005719968,0.0006449121,0.00008820327,0.00018018228,0.0021844713,0.00072943943,0.00022069873,0.0002282004,0.0013440789,0.00014249611,0.00032225007,0.00021842144,0.0039116833,0.0006248599,0.0012359661,0.0011899553,0.000182711,0.008071831,0.0001825666,0.00013105109,0.00049069227,0.0002553863,0.00046770708,0.00041329203,0.00017046703,0.000098912475,0.0022716685,0.0008398628,0.0002625317,0.0019544952,0.000031104704,0.004525521,0.003579421,0.00078281213,0.00013750634,0.0015492897,0.00056206615,0.0058603394,0.001893709,0.00051119557,0.0017577657,0.0006468123,0.0010205436,0.00011399188,0.0023611772,0.0013202629,0.0011324267,0.000727222,0.00051634834,0.00020021353,0.0018607181,0.008475949,0.0052904403,0.00045475873,0.00031731548,0.000058050064,0.0002852032,0.0009644329,0.0001863916,0.00029415256,0.0015417462,0.00012234908,0.00015761399,0.0011037083,0.00065405393,0.00029858723,0.0008058643,0.00014661749,0.00023351179,0.0003220547,0.00006103279,0.00008615536,0.0009677816,0.0013030757,0.00073273596,0.00036188748,0.0003554497,0.00022180415,0.0010117634,0.0010877674,0.001380062,0.0034309812,0.00015278636,0.00027024394,0.0026573264,0.00009061625,0.024113253,0.00047462338,0.00012469708,0.00031514082,0.00022834355,0.00073696946,0.0001212344,0.000106530635,0.0013601119,0.00027087505,0.0020511665,0.00021103851,0.000116014126,0.00044286708,0.00043897043,0.0016050341,0.00025953216,0.00033500654,0.0017078656,0.0019913097,0.00009611203,0.001064674,0.004349143,0.00029633933,0.00012283585,0.0013484384,0.0019367342,0.07469517,0.0011353533,0.0002673028,0.00022293766,0.00042465903,0.00042034852,0.008018898,0.013551103,0.0002485279,0.001021193,0.0012690768,0.000256252,0.00008555234,0.0008433578,0.00039355847,0.000102088685,0.0006141957,0.0015982271,0.00033384372,0.000101914615,0.000047681177,0.00041923174,0.00016688087,0.0004173857,0.00020143064,0.0014055638,0.00037421455,0.05869831,0.00012585091,0.00022066399,0.001657945,0.00023162599,0.00022608439,0.0002850951,0.005612193,0.00033379867,0.00024859284,0.00180066,0.0016111085,0.000038140388,0.0004536094,0.00031694575,0.00003914241,0.0001770351,0.0005497496,0.00006558722,0.0003470032,0.0024498603,0.0046578874,0.00004529914,0.001654259,0.0003155682,0.00018544408,0.0014470295,0.00021291168,0.00095484965,0.00035205568,0.0023617088,0.00017884192,0.0010543455,0.0005402438,0.000514546,0.000282502,0.0018147838,0.0011563046,0.0016247227,0.000081843566,0.00003625867,0.0007202791,0.00008258335,0.00007576049,0.015008674,0.00041184507,0.00044448892,0.0009496339,0.0013799306,0.00065369473,0.00043590466,0.000042055537,0.000114831426,0.00008075575,0.00021121852,0.00019990226,0.00040108233,0.0017243506,0.000110695866,0.00010813377,0.00034703314,0.00011404842,0.00030244116,0.00013547098,0.00010620623,0.0009395016,0.0001395816,0.00004431353,0.00016717332,0.00008510453,0.000121146804,0.0002601061,0.00010962545,0.00034622176,0.000113574104,0.00034237388,0.0026554833,0.000637925,0.0015632289,0.0032087804,0.00031608652,0.0003256953,0.0016997077,0.00007564314,0.00033386808,0.0014199506,0.00045486196,0.00006317439,0.00013640166,0.00012421366,0.00011879574,0.00035571354,0.0001387539,0.00007732129,0.0013838689,0.0010745389,0.0010412018,0.00021517093,0.000053256023,0.0042061987,0.00010023789,0.0000716717,0.00048092156,0.0001340758,0.00018658697,0.00025106687,0.0014207345,0.00012070068,0.00024411941,0.00023241615,0.0003205586,0.00024526508,0.0002495307,0.0002598417,0.00015471457,0.0013903328,0.0018311141,0.0002922665,0.0018191877,0.000058023634,0.00017409449,0.00004336262,0.00006328939,0.000036708705,0.0000074575432,0.00010303052,0.000077619145,0.0010946816]}]}
+
+
+
+
+
+

ods_ci/tests/Resources/Files/triton/triton_tensorflow_modelmesh_runtime.yaml

@@ -0,0 +1,73 @@
+apiVersion: serving.kserve.io/v1alpha1
+kind: ServingRuntime
+metadata:
+  annotations:
+    enable-route: "true"
+  name: modelmesh-triton
+  namespace: triton-modelmesh
+spec:
+  annotations:
+    opendatahub.io/modelServingSupport: '["multi"x`x`]'
+    prometheus.kserve.io/path: /metrics
+    prometheus.kserve.io/port: "8002"
+  builtInAdapter:
+    env:
+      - name: CONTAINER_MEM_REQ_BYTES
+        value: "268435456"
+      - name: USE_EMBEDDED_PULLER
+        value: "true"
+    memBufferBytes: 134217728
+    modelLoadingTimeoutMillis: 90000
+    runtimeManagementPort: 8001
+    serverType: triton
+  containers:
+    - args:
+        - -c
+        - 'mkdir -p /models/_triton_models;  chmod 777
+          /models/_triton_models;  exec
+          tritonserver  "--model-repository=/models/_triton_models"  "--model-control-mode=explicit"  "--strict-model-config=false"  "--strict-readiness=false"  "--allow-http=true"  "--allow-grpc=true"  '
+      command:
+        - /bin/sh
+      image: nvcr.io/nvidia/tritonserver:23.05-py3
+      name: triton
+      resources:
+        limits:
+          cpu: "1"
+          memory: 2Gi
+        requests:
+          cpu: "1"
+          memory: 2Gi
+  grpcDataEndpoint: port:8001
+  grpcEndpoint: port:8085
+  multiModel: true
+  protocolVersions:
+    - grpc-v2
+    - v2
+  supportedModelFormats:
+    - autoSelect: true
+      name: keras
+      version: "2"
+    - autoSelect: true
+      name: onnx
+      version: "1"
+    - autoSelect: true
+      name: pytorch
+      version: "1"
+    - autoSelect: true
+      name: tensorflow
+      version: "1"
+    - autoSelect: true
+      name: tensorflow
+      version: "2"
+    - autoSelect: true
+      name: tensorrt
+      version: "7"
+    - autoSelect: false
+      name: sklearn
+      version: "0"
+    - autoSelect: false
+      name: xgboost
+      version: "1"
+    - autoSelect: false
+      name: lightgbm
+      version: "3"

ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource

@@ -220,15 +220,15 @@
         SeleniumLibrary.Wait Until Page Contains Element    xpath://td[@data-label="Tokens"]/button
         SeleniumLibrary.Click Element    xpath://td[@data-label="Tokens"]/button
         ${token}=    SeleniumLibrary.Get Element Attribute
-        ...    xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input    value
+        ...    xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//span/input    value
     END
     RETURN    ${token}
 
 Get Single Model Token
     [Documentation]    Gets the (Single Model) Token of an specific Service Account
     [Arguments]    ${service_account_name}
     ${token}=    Get Element Attribute
-            ...    xpath://div[text()='${service_account_name}']/parent::td[@data-label='Token Name']/following-sibling::td[@data-label='Token Secret']//input   value
+    ...    xpath://div[text()='${service_account_name}']/parent::td[@data-label='Token Name']/following-sibling::td[@data-label='Token Secret']//input   value
     RETURN    ${token}
 
 Click Plus Button

ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource

@@ -121,7 +121,7 @@
     ...    Selects the given one
     [Arguments]    ${model_server}
     ${selectable}=    Run Keyword And Return Status
-    ...    Wait Until Element Is Visible    xpath://span[.="Model servers"]/../../..//button[@aria-label="Options menu"]    timeout=1
+    ...    Wait Until Element Is Visible    xpath://span[.="Model servers"]/../../..//button[@aria-label="Options menu"]    timeout=3
     IF    ${selectable}==True
         Open Model servers Options Menu  # robocop:disable
         Click Element    xpath://li/button[.="${model_server}"]
@@ -246,6 +246,7 @@
     ...    by using the UI of DSP
     [Arguments]    ${project_name}    ${service_account_name}=default-name    ${single_model}=${FALSE}
     ...            ${model_name}=${NONE}
+    Log     ${service_account_name}
     Open Data Science Project Details Page    ${project_name}    tab_id=model-server
     ${token}=    Get Model Serving Access Token via UI    ${service_account_name}    ${single_model}    ${model_name}
     RETURN    ${token}
@@ -265,7 +266,7 @@
     ...    set to ${TRUE}.
     [Arguments]    ${model_name}    ${inference_input}    ${token_auth}=${FALSE}   ${project_title}=${NONE}
     ...    ${kserve_mode}=Serverless   ${deployment_mode}=UI   ${service_port}=8888   ${end_point}=${NONE}
-    ...    ${service_account_name}=${NONE}    ${token}=${NONE}
+    ...    ${service_account_name}=default-name   ${token}=${NONE}
     ${curl_cmd}=     Set Variable    ${NONE}
     ${self_managed}=    Is RHODS Self-Managed
     IF    $deployment_mode == 'UI'
@@ -336,7 +337,7 @@
     ${inference_output}=    Get Model Inference    model_name=${model_name}   inference_input=${inference_input}
     ...    token_auth=${token_auth}    kserve_mode=${kserve_mode}    project_title=${project_title}
     ...    deployment_mode=${deployment_mode}    service_port=${service_port}    end_point=${end_point}
-    ...    token=${token}    # robocop: disable
+    ...    token=${token}      # robocop: disable
     Log    ${inference_output}
     ${result}    ${list}=    Inference Comparison    ${expected_inference_output}    ${inference_output}
     Log    ${result}