Changes for duration from milli to micro

hypertrace-view-generator/hypertrace-view-creator/src/main/resources/configs/backend-entity-view/application.conf

@@ -17,7 +17,7 @@ pinot.controllerHost = pinot-controller
 pinot.controllerPort = 9000
 pinot.timeColumn = start_time_millis
 pinot.timeUnit = MILLISECONDS
-pinot.dimensionColumns = [tenant_id, trace_id, span_id, backend_id, backend_host, backend_port, backend_protocol, backend_path, span_kind, exception_count, error_count, duration_millis, end_time_millis, num_calls, backend_name, backend_trace_id, display_name, tags__KEYS, tags__VALUES, status_code, status_message, status, caller_service_id, caller_api_id, space_ids, backend_operation, backend_destination]
+pinot.dimensionColumns = [tenant_id, trace_id, span_id, backend_id, backend_host, backend_port, backend_protocol, backend_path, span_kind, exception_count, error_count, duration_millis, duration_micros, end_time_millis, num_calls, backend_name, backend_trace_id, display_name, tags__KEYS, tags__VALUES, status_code, status_message, status, caller_service_id, caller_api_id, space_ids, backend_operation, backend_destination]
 pinot.columnsMaxLength={}
 pinot.metricColumns = []
 pinot.invertedIndexColumns= []

hypertrace-view-generator/hypertrace-view-creator/src/main/resources/configs/raw-service-view/application.conf

@@ -18,7 +18,7 @@ pinot.controllerPort = 9000
 pinot.timeColumn = start_time_millis
 pinot.timeUnit = MILLISECONDS
 # todo: Add Attributes and Metrics after adding map type value
-pinot.dimensionColumns = [tenant_id, span_kind, error_count, exception_count, duration_millis, end_time_millis, api_name, service_name, span_id, trace_id, protocol_name, status_code, service_id, api_id, num_calls, api_discovery_state, space_ids]
+pinot.dimensionColumns = [tenant_id, span_kind, error_count, exception_count, duration_millis, duration_micros, end_time_millis, api_name, service_name, span_id, trace_id, protocol_name, status_code, service_id, api_id, num_calls, api_discovery_state, space_ids]
 pinot.columnsMaxLength={}
 pinot.metricColumns = []
 pinot.invertedIndexColumns= []

hypertrace-view-generator/hypertrace-view-creator/src/main/resources/configs/raw-trace-view/application.conf

@@ -17,7 +17,7 @@ pinot.controllerHost = pinot-controller
 pinot.controllerPort = 9000
 pinot.timeColumn = start_time_millis
 pinot.timeUnit = MILLISECONDS
-pinot.dimensionColumns = [tenant_id, duration_millis, start_time_millis, end_time_millis, services, transaction_name, trace_id, num_spans, num_services, space_ids]
+pinot.dimensionColumns = [tenant_id, duration_millis, duration_micros, start_time_millis, end_time_millis, services, transaction_name, trace_id, num_spans, num_services, space_ids]
 pinot.rangeIndexColumns = [start_time_millis]
 pinot.bloomFilterColumns = []
 pinot.noDictionaryColumns = []

hypertrace-view-generator/hypertrace-view-creator/src/main/resources/configs/service-call-view/application.conf

@@ -20,7 +20,7 @@ pinot.timeUnit = MILLISECONDS
 # todo: Add Attributes and Metrics after adding map type value
 pinot.dimensionColumns = [tenant_id, end_time_millis, client_event_id, server_event_id, caller_service_id_str, caller_service, caller_api_id_str, caller_api, callee_service_id_str, callee_service, callee_api_id_str, callee_api, trace_id, transaction_name, request_url, request_method, protocol_name, response_status_code, callee_backend_id, callee_backend_name, callee_space_ids, caller_space_ids]
 pinot.columnsMaxLength={}
-pinot.metricColumns = [duration_millis, error_count, exception_count, num_calls]
+pinot.metricColumns = [duration_millis, duration_micros, error_count, exception_count, num_calls]
 pinot.invertedIndexColumns= []
 pinot.rangeIndexColumns = [start_time_millis]
 pinot.bloomFilterColumns = []

hypertrace-view-generator/hypertrace-view-creator/src/main/resources/configs/span-event-view/application.conf

@@ -17,7 +17,7 @@ pinot.controllerHost = pinot-controller
 pinot.controllerPort = 9000
 pinot.timeColumn = start_time_millis
 pinot.timeUnit = MILLISECONDS
-pinot.dimensionColumns = [tenant_id, span_id, span_kind, parent_span_id, trace_id, service_id, api_id, api_name, entry_api_id, protocol_name, tags__KEYS, tags__VALUES, status_code, start_time_millis, end_time_millis, duration_millis, api_trace_id, service_name, api_boundary_type, event_name, status_message, status, api_trace_count, display_entity_name, display_span_name, request_url, error_count, api_discovery_state, exception_count, space_ids, api_exit_calls, api_callee_name_count__KEYS, api_callee_name_count__VALUES, api_trace_error_span_count]
+pinot.dimensionColumns = [tenant_id, span_id, span_kind, parent_span_id, trace_id, service_id, api_id, api_name, entry_api_id, protocol_name, tags__KEYS, tags__VALUES, status_code, start_time_millis, end_time_millis, duration_millis, duration_micros, api_trace_id, service_name, api_boundary_type, event_name, status_message, status, api_trace_count, display_entity_name, display_span_name, request_url, error_count, api_discovery_state, exception_count, space_ids, api_exit_calls, api_callee_name_count__KEYS, api_callee_name_count__VALUES, api_trace_error_span_count]
 pinot.columnsMaxLength={}
 pinot.metricColumns = []
 pinot.invertedIndexColumns= [tags__KEYS, tags__VALUES]

hypertrace-view-generator/hypertrace-view-generator-api/src/main/avro/BackendEntityView.avdl

@@ -24,6 +24,9 @@ protocol BackendEntityViewProtocol {
 
     long duration_millis = 0;
 
+    // duration micro
+    double duration_micros = 0;
+
     union { null, string } span_kind = null;
 
     int error_count = 0;

hypertrace-view-generator/hypertrace-view-generator-api/src/main/avro/RawServiceView.avdl

@@ -18,6 +18,9 @@ protocol RawServiceViewProtocol {
 
     long duration_millis;
 
+    // duration micro
+    double duration_micros = 0;
+
     union { null, string } span_kind = null;
 
     int error_count = 0;

hypertrace-view-generator/hypertrace-view-generator-api/src/main/avro/RawTraceView.avdl

@@ -20,6 +20,9 @@ protocol TraceViewProtocol {
     //total time for the trace
     long duration_millis = 0;
 
+    // duration micro
+    double duration_micros = 0;
+
     //How deep was the call.
     int num_services = 0;
 

hypertrace-view-generator/hypertrace-view-generator-api/src/main/avro/ServiceCallView.avdl

@@ -41,6 +41,9 @@ protocol ServiceCallViewProtocol {
     long end_time_millis = 0;
     long duration_millis = 0;
 
+    // duration micro
+    double duration_micros = 0;
+
     // Error count, which will be either a 0 or 1 where one indicates that this
     // api/service call has erred out.
     int error_count = 0;

hypertrace-view-generator/hypertrace-view-generator-api/src/main/avro/SpanEventView.avdl

@@ -47,6 +47,9 @@ protocol SpanEventViewProtocol {
     // duration
     long duration_millis = 0;
 
+    // duration micro
+    double duration_micros = 0;
+
     // span id of an entry api span.
     union { null, bytes } api_trace_id = null;
 

hypertrace-view-generator/hypertrace-view-generator/src/main/java/org/hypertrace/viewgenerator/generators/BackendEntityViewGenerator.java

@@ -88,6 +88,7 @@ private BackendEntityView.Builder generateViewBuilder(
       builder.setStartTimeMillis(event.getStartTimeMillis());
       builder.setEndTimeMillis(event.getEndTimeMillis());
       builder.setDurationMillis(event.getEndTimeMillis() - event.getStartTimeMillis());
+      builder.setDurationMicros(getDurationMetricValue(event));
       Protocol protocol = EnrichedSpanUtils.getProtocol(event);
 
       double exceptionCount = getMetricValue(event, EXCEPTION_COUNT_ATTR, 0.0d);

hypertrace-view-generator/hypertrace-view-generator/src/main/java/org/hypertrace/viewgenerator/generators/BaseViewGenerator.java

@@ -20,13 +20,16 @@
 import org.hypertrace.traceenricher.enrichedspan.constants.EnrichedSpanConstants;
 import org.hypertrace.traceenricher.enrichedspan.constants.v1.CommonAttribute;
 import org.hypertrace.viewgenerator.generators.ViewGeneratorState.TraceState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Pre-processing for View Generators, for data that are mostly needed by the the view generators
  */
 public abstract class BaseViewGenerator<OUT extends GenericRecord>
     implements JavaCodeBasedViewGenerator<StructuredTrace, OUT> {
 
+  private static final Logger LOG = LoggerFactory.getLogger(BaseViewGenerator.class);
   private static final String VIEW_GENERATION_ARRIVAL_TIME = "view.generation.arrival.time";
   private static final Timer viewGeneratorArrivalTimer =
       PlatformMetricsRegistry.registerTimer(DataflowMetricUtils.ARRIVAL_LAG, new HashMap<>());
@@ -45,6 +48,19 @@ static double getMetricValue(Event event, String metricName, double defaultValue
     return value.getValue();
   }
 
+  static double getDurationMetricValue(Event event) {
+    MetricValue value = event.getMetrics().getMetricMap().get("Duration-micro");
+    LOG.info("TraceDuration-micro value is {}", value.getValue());
+    return value.getValue();
+  }
+
+  static double getDurationMetricValueTrace(StructuredTrace structuredTrace) {
+    MetricValue value =
+        structuredTrace.getEventList().get(0).getMetrics().getMetricMap().get("Duration-micro");
+    LOG.info("Duration-micro value is {}", value.getValue());
+    return value.getValue();
+  }
+
   @Nullable
   static String getTransactionName(StructuredTrace trace) {
     Attributes attributes = trace.getAttributes();

hypertrace-view-generator/hypertrace-view-generator/src/main/java/org/hypertrace/viewgenerator/generators/RawServiceViewGenerator.java

@@ -50,6 +50,7 @@ List<RawServiceView> generateView(
         builder.setStartTimeMillis(event.getStartTimeMillis());
         builder.setEndTimeMillis(event.getEndTimeMillis());
         builder.setDurationMillis(event.getEndTimeMillis() - event.getStartTimeMillis());
+        builder.setDurationMicros(getDurationMetricValue(event));
         builder.setTransactionName(getTransactionName(structuredTrace));
 
         String spanType = EnrichedSpanUtils.getSpanType(event);

hypertrace-view-generator/hypertrace-view-generator/src/main/java/org/hypertrace/viewgenerator/generators/RawTraceViewGenerator.java

@@ -43,6 +43,8 @@ List<RawTraceView> generateView(
     builder.setEndTimeMillis(structuredTrace.getEndTimeMillis());
     builder.setDurationMillis(
         structuredTrace.getEndTimeMillis() - structuredTrace.getStartTimeMillis());
+    builder.setDurationMicros(getDurationMetricValueTrace(structuredTrace));
+
     Set<String> services = new HashSet<>();
     for (Event event : structuredTrace.getEventList()) {
       String serviceName = EnrichedSpanUtils.getServiceName(event);

hypertrace-view-generator/hypertrace-view-generator/src/main/java/org/hypertrace/viewgenerator/generators/ServiceCallViewGenerator.java

@@ -207,6 +207,7 @@ private void buildCommonServiceCallView(
     builder.setStartTimeMillis(event.getStartTimeMillis());
     builder.setEndTimeMillis(event.getEndTimeMillis());
     builder.setDurationMillis(event.getEndTimeMillis() - event.getStartTimeMillis());
+    builder.setDurationMicros(getDurationMetricValue(event));
   }
 
   private void buildExitSpanView(Event event, ServiceCallView.Builder builder) {

hypertrace-view-generator/hypertrace-view-generator/src/main/java/org/hypertrace/viewgenerator/generators/SpanEventViewGenerator.java

@@ -219,6 +219,7 @@ private SpanEventView.Builder generateViewBuilder(
     builder.setStartTimeMillis(event.getStartTimeMillis());
     builder.setEndTimeMillis(event.getEndTimeMillis());
     builder.setDurationMillis(event.getEndTimeMillis() - event.getStartTimeMillis());
+    builder.setDurationMicros(getDurationMetricValue(event));
 
     // error count
     MetricValue errorMetric = event.getMetrics().getMetricMap().get(ERROR_COUNT_CONSTANT);

hypertrace-view-generator/hypertrace-view-generator/src/test/java/org/hypertrace/viewgenerator/generators/SpanEventViewGeneratorTest.java

@@ -1,29 +1,22 @@
 package org.hypertrace.viewgenerator.generators;
 
+import static org.hypertrace.core.datamodel.shared.AvroBuilderCache.fastNewBuilder;
 import static org.hypertrace.core.span.constants.v1.Http.HTTP_PATH;
 import static org.hypertrace.core.span.constants.v1.Http.HTTP_URL;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.Maps;
-import java.io.File;
 import java.io.IOException;
-import java.net.URL;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.hypertrace.core.datamodel.AttributeValue;
-import org.hypertrace.core.datamodel.Attributes;
-import org.hypertrace.core.datamodel.Entity;
-import org.hypertrace.core.datamodel.Event;
-import org.hypertrace.core.datamodel.Metrics;
-import org.hypertrace.core.datamodel.StructuredTrace;
+import org.hypertrace.core.datamodel.*;
 import org.hypertrace.core.datamodel.shared.trace.AttributeValueCreator;
 import org.hypertrace.core.span.constants.RawSpanConstants;
 import org.hypertrace.traceenricher.enrichedspan.constants.EnrichedSpanConstants;
@@ -33,6 +26,7 @@
 import org.hypertrace.traceenricher.enrichedspan.constants.v1.Protocol;
 import org.hypertrace.viewgenerator.api.SpanEventView;
 import org.hypertrace.viewgenerator.generators.ViewGeneratorState.TraceState;
+import org.hypertrace.viewgenerator.generators.utils.TestUtilities;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -143,15 +137,7 @@ public void testGetRequestUrl() {
 
   @Test
   public void testSpanEventViewGen_HotrodTrace() throws IOException {
-    URL resource =
-        Thread.currentThread().getContextClassLoader().getResource("StructuredTrace-Hotrod.avro");
-
-    SpecificDatumReader<StructuredTrace> datumReader =
-        new SpecificDatumReader<>(StructuredTrace.getClassSchema());
-    DataFileReader<StructuredTrace> dfrStructuredTrace =
-        new DataFileReader<>(new File(resource.getPath()), datumReader);
-    StructuredTrace trace = dfrStructuredTrace.next();
-    dfrStructuredTrace.close();
+    StructuredTrace trace = TestUtilities.getSampleHotRodTrace();
 
     TraceState traceState = new TraceState(trace);
     verifyGetExitSpanToApiEntrySpan_HotrodTrace(trace, traceState);
@@ -177,6 +163,9 @@ private void verifyGetExitSpanToApiEntrySpan_HotrodTrace(
 
   @Test
   public void testExitCallsInfo() {
+    MetricValue metricValue = fastNewBuilder(MetricValue.Builder.class).setValue(645.0).build();
+    Map<String, MetricValue> metricMap = new HashMap<>();
+    metricMap.put("Duration-micro", metricValue);
     StructuredTrace.Builder traceBuilder = StructuredTrace.newBuilder();
     traceBuilder
         .setCustomerId("customer1")
@@ -198,12 +187,12 @@ public void testExitCallsInfo() {
                     .setEntityIdList(Collections.singletonList("sample-entity-id"))
                     .setStartTimeMillis(System.currentTimeMillis())
                     .setEndTimeMillis(System.currentTimeMillis())
-                    .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+                    .setMetrics(Metrics.newBuilder().setMetricMap(metricMap).build())
                     .setAttributesBuilder(Attributes.newBuilder().setAttributeMap(new HashMap<>()))
                     .setEnrichedAttributesBuilder(
                         Attributes.newBuilder().setAttributeMap(Maps.newHashMap()))
                     .build()))
-        .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+        .setMetrics(Metrics.newBuilder().setMetricMap(metricMap).build())
         .setEntityEdgeList(new ArrayList<>())
         .setEventEdgeList(new ArrayList<>())
         .setEntityEventEdgeList(new ArrayList<>())
@@ -215,6 +204,7 @@ public void testExitCallsInfo() {
     List<SpanEventView> list = spanEventViewGenerator.process(trace);
     assertEquals(Maps.newHashMap(), list.get(0).getApiCalleeNameCount());
     assertEquals(0, list.get(0).getApiExitCalls());
+    assertTrue(trace.getMetrics().getMetricMap().get("Duration-micro").getValue().equals(645.0));
 
     Map<String, AttributeValue> spanAttributes = new HashMap<>();
     spanAttributes.put(
@@ -239,7 +229,7 @@ public void testExitCallsInfo() {
                     .setEntityIdList(Collections.singletonList("sample-entity-id"))
                     .setStartTimeMillis(System.currentTimeMillis())
                     .setEndTimeMillis(System.currentTimeMillis())
-                    .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+                    .setMetrics(Metrics.newBuilder().setMetricMap(metricMap).build())
                     .setAttributesBuilder(Attributes.newBuilder().setAttributeMap(new HashMap<>()))
                     .setEnrichedAttributesBuilder(
                         Attributes.newBuilder().setAttributeMap(spanAttributes))
@@ -251,10 +241,14 @@ public void testExitCallsInfo() {
     list = spanEventViewGenerator.process(trace);
     assertEquals(calleeNameCount, list.get(0).getApiCalleeNameCount());
     assertEquals(5, list.get(0).getApiExitCalls());
+    assertTrue(trace.getMetrics().getMetricMap().get("Duration-micro").getValue().equals(645.0));
   }
 
   @Test
   public void testApiTraceErrorSpanCount() {
+    MetricValue metricValue = fastNewBuilder(MetricValue.Builder.class).setValue(645.0).build();
+    Map<String, MetricValue> metricMap = new HashMap<>();
+    metricMap.put("Duration-micro", metricValue);
     StructuredTrace.Builder traceBuilder = StructuredTrace.newBuilder();
     traceBuilder
         .setCustomerId("customer1")
@@ -276,12 +270,12 @@ public void testApiTraceErrorSpanCount() {
                     .setEntityIdList(Collections.singletonList("sample-entity-id"))
                     .setStartTimeMillis(System.currentTimeMillis())
                     .setEndTimeMillis(System.currentTimeMillis())
-                    .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+                    .setMetrics(Metrics.newBuilder().setMetricMap(metricMap).build())
                     .setAttributesBuilder(Attributes.newBuilder().setAttributeMap(new HashMap<>()))
                     .setEnrichedAttributesBuilder(
                         Attributes.newBuilder().setAttributeMap(Maps.newHashMap()))
                     .build()))
-        .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+        .setMetrics(Metrics.newBuilder().setMetricMap(metricMap).build())
         .setEntityEdgeList(new ArrayList<>())
         .setEventEdgeList(new ArrayList<>())
         .setEntityEventEdgeList(new ArrayList<>())
@@ -308,7 +302,7 @@ public void testApiTraceErrorSpanCount() {
                     .setEntityIdList(Collections.singletonList("sample-entity-id"))
                     .setStartTimeMillis(System.currentTimeMillis())
                     .setEndTimeMillis(System.currentTimeMillis())
-                    .setMetrics(Metrics.newBuilder().setMetricMap(new HashMap<>()).build())
+                    .setMetrics(Metrics.newBuilder().setMetricMap(metricMap).build())
                     .setAttributesBuilder(Attributes.newBuilder().setAttributeMap(new HashMap<>()))
                     .setEnrichedAttributesBuilder(
                         Attributes.newBuilder().setAttributeMap(spanAttributes))
@@ -319,6 +313,7 @@ public void testApiTraceErrorSpanCount() {
     spanEventViewGenerator = new SpanEventViewGenerator();
     list = spanEventViewGenerator.process(trace);
     assertEquals(5, list.get(0).getApiTraceErrorSpanCount());
+    assertTrue(trace.getMetrics().getMetricMap().get("Duration-micro").getValue().equals(645.0));
   }
 
   private Event createMockEventWithAttribute(String key, String value) {