diff --git a/engine/src/main/java/io/seldon/engine/predictors/PredictiveUnitBean.java b/engine/src/main/java/io/seldon/engine/predictors/PredictiveUnitBean.java index 37bd186c9a..c6167f2809 100644 --- a/engine/src/main/java/io/seldon/engine/predictors/PredictiveUnitBean.java +++ b/engine/src/main/java/io/seldon/engine/predictors/PredictiveUnitBean.java @@ -87,7 +87,11 @@ private void addMetrics(SeldonMessage msg,PredictiveUnitState state,Map()); + List current = metrics.get(state.name); + current.addAll(msg.getMeta().getMetricsList()); + metrics.put(state.name,current); } } diff --git a/engine/src/test/java/io/seldon/engine/api/rest/TestRestClientControllerExternalGraphs.java b/engine/src/test/java/io/seldon/engine/api/rest/TestRestClientControllerExternalGraphs.java index 6ad0d81944..bb1a8b8339 100644 --- a/engine/src/test/java/io/seldon/engine/api/rest/TestRestClientControllerExternalGraphs.java +++ b/engine/src/test/java/io/seldon/engine/api/rest/TestRestClientControllerExternalGraphs.java @@ -326,7 +326,8 @@ public void testTransformOutputMetrics() throws Exception public void testRouterMetrics() throws Exception { String jsonStr = readFile("src/test/resources/router_simple.json",StandardCharsets.UTF_8); - String responseStr = readFile("src/test/resources/router_response.json",StandardCharsets.UTF_8); + String responseStrRouter = readFile("src/test/resources/router_response.json",StandardCharsets.UTF_8); + String responseStrModel = readFile("src/test/resources/router_model_response.json",StandardCharsets.UTF_8); PredictorSpec.Builder PredictorSpecBuilder = PredictorSpec.newBuilder(); updateMessageBuilderFromJson(PredictorSpecBuilder, jsonStr); PredictorSpec predictorSpec = PredictorSpecBuilder.build(); @@ -337,9 +338,19 @@ public void testRouterMetrics() throws Exception enginePredictor.setPredictorSpec(predictorSpec); - ResponseEntity httpResponse = new ResponseEntity(responseStr, null, HttpStatus.OK); + ResponseEntity httpResponse1 = new ResponseEntity(responseStrRouter, null, HttpStatus.OK); + ResponseEntity httpResponse2 = new ResponseEntity(responseStrModel, null, HttpStatus.OK); Mockito.when(restTemplate.postForEntity(Matchers.any(), Matchers.>>any(), Matchers.>any())) - .thenReturn(httpResponse); + .thenAnswer(new Answer>() { + private int count = 0; + + public ResponseEntity answer(InvocationOnMock invocation) { + count++; + if (count == 1) + return httpResponse1; + + return httpResponse2; + }}); internalPredictionService.setRestTemplate(restTemplate); MvcResult res = mvc.perform(MockMvcRequestBuilders.post("/api/v0.1/predictions") @@ -357,23 +368,37 @@ public void testRouterMetrics() throws Exception // Check for returned metrics Assert.assertEquals("COUNTER",seldonMessage.getMeta().getMetrics(0).getType().toString()); Assert.assertEquals(1.0F,seldonMessage.getMeta().getMetrics(0).getValue(),0.0); - Assert.assertEquals("mycounter",seldonMessage.getMeta().getMetrics(0).getKey()); + Assert.assertEquals("myroutercounter",seldonMessage.getMeta().getMetrics(0).getKey()); Assert.assertEquals("GAUGE",seldonMessage.getMeta().getMetrics(1).getType().toString()); Assert.assertEquals(22.0F,seldonMessage.getMeta().getMetrics(1).getValue(),0.0); - Assert.assertEquals("mygauge",seldonMessage.getMeta().getMetrics(1).getKey()); + Assert.assertEquals("myroutergauge",seldonMessage.getMeta().getMetrics(1).getKey()); Assert.assertEquals("TIMER",seldonMessage.getMeta().getMetrics(2).getType().toString()); Assert.assertEquals(1.0F,seldonMessage.getMeta().getMetrics(2).getValue(),0.0); - Assert.assertEquals("mytimer",seldonMessage.getMeta().getMetrics(2).getKey()); + Assert.assertEquals("myroutertimer",seldonMessage.getMeta().getMetrics(2).getKey()); + + Assert.assertEquals("COUNTER",seldonMessage.getMeta().getMetrics(3).getType().toString()); + Assert.assertEquals(1.0F,seldonMessage.getMeta().getMetrics(3).getValue(),0.0); + Assert.assertEquals("myroutermodelcounter",seldonMessage.getMeta().getMetrics(3).getKey()); + + Assert.assertEquals("GAUGE",seldonMessage.getMeta().getMetrics(4).getType().toString()); + Assert.assertEquals(22.0F,seldonMessage.getMeta().getMetrics(4).getValue(),0.0); + Assert.assertEquals("myroutermodelgauge",seldonMessage.getMeta().getMetrics(4).getKey()); + + Assert.assertEquals("TIMER",seldonMessage.getMeta().getMetrics(5).getType().toString()); + Assert.assertEquals(1.0F,seldonMessage.getMeta().getMetrics(5).getValue(),0.0); + Assert.assertEquals("myroutermodeltimer",seldonMessage.getMeta().getMetrics(5).getKey()); // Check prometheus endpoint for metric MvcResult res2 = mvc.perform(MockMvcRequestBuilders.get("/prometheus")).andReturn(); Assert.assertEquals(200, res2.getResponse().getStatus()); response = res2.getResponse().getContentAsString(); System.out.println(response); - Assert.assertTrue(response.indexOf("mycounter_total{deployment_name=\"None\",model_image=\"seldonio/router\",model_name=\"router\",model_version=\"0.6\",mytag1=\"mytagval1\",predictor_name=\"fx-market-predictor\",predictor_version=\"unknown\",} 1.0")>-1); - Assert.assertTrue(response.indexOf("mytimer_seconds_count{deployment_name=\"None\",model_image=\"seldonio/router\",model_name=\"router\",model_version=\"0.6\",predictor_name=\"fx-market-predictor\",predictor_version=\"unknown\",} 1.0")>-1); + Assert.assertTrue(response.indexOf("myroutercounter_total{deployment_name=\"None\",model_image=\"seldonio/router\",model_name=\"router\",model_version=\"0.6\",mytag1=\"mytagval1\",predictor_name=\"fx-market-predictor\",predictor_version=\"unknown\",} 1.0")>-1); + Assert.assertTrue(response.indexOf("myroutertimer_seconds_count{deployment_name=\"None\",model_image=\"seldonio/router\",model_name=\"router\",model_version=\"0.6\",predictor_name=\"fx-market-predictor\",predictor_version=\"unknown\",} 1.0")>-1); + Assert.assertTrue(response.indexOf("myroutermodelcounter_total{deployment_name=\"None\",model_image=\"seldonio/model\",model_name=\"model\",model_version=\"0.6\",mytag1=\"mytagval1\",predictor_name=\"fx-market-predictor\",predictor_version=\"unknown\",} 1.0")>-1); + Assert.assertTrue(response.indexOf("myroutermodeltimer_seconds_count{deployment_name=\"None\",model_image=\"seldonio/model\",model_name=\"model\",model_version=\"0.6\",predictor_name=\"fx-market-predictor\",predictor_version=\"unknown\",} 1.0")>-1); } diff --git a/engine/src/test/resources/router_model_response.json b/engine/src/test/resources/router_model_response.json new file mode 100644 index 0000000000..dbdf9d148e --- /dev/null +++ b/engine/src/test/resources/router_model_response.json @@ -0,0 +1,30 @@ +{ + "meta": { + "metrics": [ + { + "type": "COUNTER", + "key": "myroutermodelcounter", + "value": 1.0, + "tags": {"mytag1":"mytagval1"} + }, + { + "type": "GAUGE", + "key": "myroutermodelgauge", + "value": 22.0 + }, + { + "type": "TIMER", + "key": "myroutermodeltimer", + "value": 1.0 + } + ] + }, + "data": { + "ndarray": [ + [ + 1, + 2 + ] + ] + } +} diff --git a/engine/src/test/resources/router_response.json b/engine/src/test/resources/router_response.json index cd6c08ffb5..2d066dcaaf 100644 --- a/engine/src/test/resources/router_response.json +++ b/engine/src/test/resources/router_response.json @@ -3,18 +3,18 @@ "metrics": [ { "type": "COUNTER", - "key": "mycounter", + "key": "myroutercounter", "value": 1.0, "tags": {"mytag1":"mytagval1"} }, { "type": "GAUGE", - "key": "mygauge", + "key": "myroutergauge", "value": 22.0 }, { "type": "TIMER", - "key": "mytimer", + "key": "myroutertimer", "value": 1.0 } ]