Support tracing Catchpoint requests (#406)

* Support tracing Catchpoint requests

* Fix Hapi instrumentation issue caused by non-waited trace hook

src/Constants.ts

@@ -77,6 +77,7 @@ export function getDefaultCollectorEndpoint() {
 export const MONITORING_DATA_PATH = '/monitoring-data';
 export const COMPOSITE_MONITORING_DATA_PATH = '/composite-monitoring-data';
 export const LOCAL_COLLECTOR_ENDPOINT = 'localhost:3333';
+export const MAX_MONITOR_DATA_BATCH_SIZE = 100;
 
 export const PROC_STAT_PATH: string = '/proc/self/stat';
 export const PROC_IO_PATH: string = '/proc/self/io';
@@ -572,6 +573,13 @@ export const GoogleBigQueryTags = {
     RESPONSE: 'google.bigquery.response',
 };
 
+export const CatchpointTags = {
+    REGION_NAME: 'catchpoint.region.name',
+    COUNTRY_NAME: 'catchpoint.country.name',
+    CITY_NAME: 'catchpoint.city.name',
+    TEST_ID: 'catchpoint.test.id',
+};
+
 export const ErrorTags = {
     ERROR: 'error',
     ERROR_KIND: 'error.kind',
@@ -590,6 +598,33 @@ export const TriggerHeaderTags = {
     RESOURCE_NAME: 'x-catchpoint-resource-name',
 };
 
+export const HTTPHeaders = {
+    USER_AGENT: 'User-Agent',
+    CONTENT_TYPE: 'Content-Type',
+    AUTHORIZATION: 'Authorization',
+};
+
+export const CatchpointHeaders = {
+    REGION_NAME: 'x-catchpoint-region-name',
+    COUNTRY_NAME: 'x-catchpoint-country-name',
+    CITY_NAME: 'x-catchpoint-city-name',
+    TEST_ID: 'x-catchpoint-test-id',
+    TIME: 'x-catchpoint-time',
+};
+
+export const CatchpointProperties = {
+    DEFAULT_APP_NAME: 'Catchpoint',
+    APP_DOMAIN_NAME: 'Monitoring',
+    APP_CLASS_NAME: 'Catchpoint',
+    APP_REGION_PLACEHOLDER: '${cp-app-region}',
+    APP_NAME_PLACEHOLDER: '${cp-app-name}',
+    APP_ID_TEMPLATE: ':catchpoint:${cp-app-region}:${cp-app-name}',
+    HTTP_REQUEST_DOMAIN_NAME: 'API',
+    HTTP_REQUEST_CLASS_NAME: 'HTTP',
+    TIME_MAX_DIFF: 30000,
+    AGENT_VERSION: 'Catchpoint',
+};
+
 export const AlreadyTracedHeader = 'x-catchpoint-already-traced';
 
 export const SpanTypes = {

src/Reporter.ts

@@ -3,23 +3,21 @@ import * as http from 'http';
 import * as https from 'https';
 import * as url from 'url';
 import {
-    COMPOSITE_MONITORING_DATA_PATH,
     LOCAL_COLLECTOR_ENDPOINT,
     SPAN_TAGS_TO_TRIM_1,
     SPAN_TAGS_TO_TRIM_2,
     SPAN_TAGS_TO_TRIM_3,
     REPORTER_HTTP_TIMEOUT,
     REPORTER_DATA_SIZE_LIMIT,
+    MONITORING_DATA_PATH,
+    MAX_MONITOR_DATA_BATCH_SIZE,
     getDefaultCollectorEndpoint,
 } from './Constants';
 import Utils from './utils/Utils';
 import ThundraLogger from './ThundraLogger';
-import BaseMonitoringData from './plugins/data/base/BaseMonitoringData';
 import MonitoringDataType from './plugins/data/base/MonitoringDataType';
 import ConfigNames from './config/ConfigNames';
 import ConfigProvider from './config/ConfigProvider';
-import CompositeMonitoringData from './plugins/data/composite/CompositeMonitoringData';
-import MonitorDataType from './plugins/data/base/MonitoringDataType';
 
 const httpAgent = new http.Agent({
     keepAlive: true,
@@ -117,16 +115,60 @@ class Reporter {
      */
     sendReports(reports: any[], disableTrim: boolean = false): Promise<void> {
         ThundraLogger.debug('<Reporter> Sending reports ...');
-        let compositeReport: any;
-        try {
-            compositeReport = this.getCompositeReport(reports);
-        } catch (err) {
-            ThundraLogger.error('<Reporter> Cannot create batch request will send no report:', err);
+
+        // Split reports into batches
+        const batchedReports: any[] = this.getBatchedReports(reports);
+
+        const reportPromises: Array<Promise<void>> = [];
+        // Report each batch asynchronously in parallel
+        batchedReports.forEach((batch: any) => {
+            reportPromises.push(this.doSendReports(batch, disableTrim));
+        });
+
+        return new Promise<void>((resolve, reject) => {
+            try {
+                const allPromises: Promise<void[]> = Promise.all(reportPromises);
+                allPromises
+                    .then(() => {
+                        ThundraLogger.debug('<Reporter> Sent all reports successfully');
+                        resolve();
+                    })
+                    .catch((err) => {
+                        ThundraLogger.debug('<Reporter> Failed to send all reports:', err);
+                        reject(err);
+                    });
+            } catch (err) {
+                ThundraLogger.debug('<Reporter> Unable to send all reports:', err);
+                reject(err);
+            }
+        });
+    }
+
+    private getBatchedReports(reports: any[]): any[] {
+        const batchedReports: any[] = [];
+        const batchCount: number = Math.ceil(reports.length / MAX_MONITOR_DATA_BATCH_SIZE);
+
+        for (let i = 0; i < batchCount; i++) {
+            const batch: any[] = [];
+            for (let j = 0; j < MAX_MONITOR_DATA_BATCH_SIZE; j++) {
+                const report = reports[i * MAX_MONITOR_DATA_BATCH_SIZE + j];
+                if (!report) {
+                    continue;
+                }
+                batch.push(report);
+            }
+            batchedReports.push(batch);
         }
 
+        return batchedReports;
+    }
+
+    private doSendReports(monitoringDataList: any[], disableTrim: boolean = false): Promise<void> {
+        ThundraLogger.debug('<Reporter> Sending batch ...');
+
         return new Promise<void>((resolve, reject) => {
             try {
-                const reportJson: string = this.serializeReport(compositeReport, disableTrim);
+                const reportJson: string = this.serializeReport(monitoringDataList, disableTrim);
                 this.doReport(reportJson)
                     .then((res: any) => {
                         ThundraLogger.debug('<Reporter> Sent reports successfully');
@@ -157,7 +199,7 @@ class Reporter {
     }
 
     private createRequestOptions(u?: url.URL): http.RequestOptions {
-        const path = COMPOSITE_MONITORING_DATA_PATH;
+        const path = MONITORING_DATA_PATH;
 
         return {
             timeout: REPORTER_HTTP_TIMEOUT,
@@ -191,6 +233,7 @@ class Reporter {
         };
     }
 
+    /*
     private getCompositeReport(reports: any[]): any {
         ThundraLogger.debug('<Reporter> Generating composite report ...');
 
@@ -230,6 +273,7 @@ class Reporter {
 
         return monitoringData;
     }
+    */
 
     private doReport(reportJson: string) {
         ThundraLogger.debug('<Reporter> Reporting ...');
@@ -332,44 +376,44 @@ class Reporter {
         });
     }
 
-    private serializeReport(batch: any, disableTrim: boolean): string {
+    private serializeReport(monitoringDataList: any[], disableTrim: boolean): string {
         // If trimming is disabled, trim if and only if data size is bigger than maximum allowed limit
         const maxReportDataSize: number = disableTrim ? REPORTER_DATA_SIZE_LIMIT : this.maxReportSize;
 
-        let json: string = this.serializeMasked(batch, this.maskedKeys, this.hide);
+        let json: string = this.serializeMasked(monitoringDataList, this.maskedKeys, this.hide);
 
         if (json.length < maxReportDataSize) {
             return json;
         }
         for (const trimmer of this.trimmers) {
-            const trimResult: TrimResult = trimmer.trim(batch.data.allMonitoringData);
-            batch.data.allMonitoringData = trimResult.monitoringDataList;
+            const trimResult: TrimResult = trimmer.trim(monitoringDataList);
+            monitoringDataList = trimResult.monitoringDataList;
             if (!trimResult.mutated) {
                 continue;
             }
-            json = this.serializeMasked(batch, this.maskedKeys, this.hide);
+            json = this.serializeMasked(monitoringDataList, this.maskedKeys, this.hide);
             if (json.length < maxReportDataSize) {
                 return json;
             }
         }
-        return this.serializeMasked(batch, this.maskedKeys, this.hide);
+        return this.serializeMasked(monitoringDataList, this.maskedKeys, this.hide);
     }
 
-    private serializeMasked(batch: any, maskedKeys: MaskedKey[], hide?: boolean): string {
+    private serializeMasked(monitoringDataList: any[], maskedKeys: MaskedKey[], hide?: boolean): string {
         if (maskedKeys && maskedKeys.length) {
             try {
                 ThundraLogger.debug(`<Reporter> Serializing masked ...`);
 
                 const maskCheckSet: WeakSet<any> = new WeakSet<any>();
 
-                for (const monitoringData of batch.data.allMonitoringData) {
+                for (const monitoringData of monitoringDataList) {
                     if (monitoringData.tags) {
                         maskCheckSet.add(monitoringData.tags);
                     }
                 }
 
                 const result: string =
-                    JSON.stringify(batch, this.createMaskingReplacer(maskCheckSet, maskedKeys, hide));
+                    JSON.stringify(monitoringDataList, this.createMaskingReplacer(maskCheckSet, maskedKeys, hide));
 
                 ThundraLogger.debug(`<Reporter> Serialized masked`);
 
@@ -378,7 +422,7 @@ class Reporter {
                 ThundraLogger.debug(`<Reporter> Error occurred while serializing masked`, err);
             }
         }
-        return Utils.serializeJSON(batch);
+        return Utils.serializeJSON(monitoringDataList);
     }
 
     private isMasked(key: string, maskedKeys: MaskedKey[]): boolean {

src/application/ApplicationInfo.ts

@@ -9,6 +9,8 @@ export interface ApplicationInfo {
     applicationDomainName: string;
     applicationRegion: string;
     applicationVersion: string;
+    applicationRuntime: string;
+    applicationRuntimeVersion: string;
     applicationStage: string;
     applicationTags: any;
 }

src/application/GlobalApplicationInfoProvider.ts

@@ -38,6 +38,8 @@ export class GlobalApplicationInfoProvider implements ApplicationInfoProvider {
             applicationDomainName: ConfigProvider.get<string>(ConfigNames.THUNDRA_APPLICATION_DOMAIN_NAME),
             applicationRegion: ConfigProvider.get<string>(ConfigNames.THUNDRA_APPLICATION_REGION),
             applicationVersion: ConfigProvider.get<string>(ConfigNames.THUNDRA_APPLICATION_VERSION),
+            applicationRuntime: 'node',
+            applicationRuntimeVersion: process.version,
             applicationStage: ConfigProvider.get<string>(ConfigNames.THUNDRA_APPLICATION_STAGE),
             applicationTags: Utils.getApplicationTags(),
         };

src/plugins/data/invocation/Resource.ts

@@ -21,7 +21,7 @@ class Resource {
 
     constructor(opt: any = {}) {
         this.resourceType = opt.resourceType;
-        this.resourceName = opt.resourceOperation;
+        this.resourceName = opt.resourceName;
         this.resourceOperation = opt.resourceOperation;
         this.resourceCount = opt.resourceCount;
         this.resourceErrorCount = opt.resourceErrorCount;

src/utils/TestRunnerUtils.ts

@@ -26,7 +26,7 @@ class TestRunnerUtils {
 
         const testRunIdSeed = environment + '_' + repoURL + '_' + commitHash + '_' + testRunKey;
 
-        return Utils.generareIdFrom(testRunIdSeed);
+        return Utils.generateIdFrom(testRunIdSeed);
     }
 
     /**

src/utils/Utils.ts

@@ -55,7 +55,7 @@ class Utils {
      * Generates id in UUID format with given value
      * @param value value
      */
-    static generareIdFrom(value: string): string {
+    static generateIdFrom(value: string): string {
         return uuidv5(value, AGENT_UUID_CONST);
     }
 
@@ -384,24 +384,33 @@ class Utils {
      * Injects common application properties into given {@link BaseMonitoringData} monitoring data
      * @param {BaseMonitoringData} monitoringData the @link BaseMonitoringData} monitoring data
      *        in which common application properties will be injected to
+     * @param {ApplicationInfo} applicationInfo the the {@link ApplicationInfo} to be injected into
      */
-    static injectCommonApplicationProperties(monitoringData: BaseMonitoringData) {
-        const applicationInfo = ApplicationManager.getApplicationInfo();
+    static injectCommonApplicationProperties(monitoringData: BaseMonitoringData,
+                                             applicationInfo?: ApplicationInfo) {
+        const appInfo = applicationInfo || ApplicationManager.getApplicationInfo();
 
         monitoringData.agentVersion = AGENT_VERSION;
         monitoringData.dataModelVersion = DATA_MODEL_VERSION;
+        monitoringData.applicationRuntime = 'node';
         monitoringData.applicationRuntimeVersion = process.version;
-        if (applicationInfo) {
-            monitoringData.applicationInstanceId = applicationInfo.applicationInstanceId;
-            monitoringData.applicationId = applicationInfo.applicationId;
-            monitoringData.applicationName = applicationInfo.applicationName;
-            monitoringData.applicationClassName = applicationInfo.applicationClassName;
-            monitoringData.applicationDomainName = applicationInfo.applicationDomainName;
-            monitoringData.applicationStage = applicationInfo.applicationStage;
-            monitoringData.applicationVersion = applicationInfo.applicationVersion;
+        if (appInfo) {
+            if (appInfo.applicationRuntime) {
+                monitoringData.applicationRuntime = appInfo.applicationRuntime;
+            }
+            if (appInfo.applicationRuntimeVersion) {
+                monitoringData.applicationRuntimeVersion = appInfo.applicationRuntimeVersion;
+            }
+            monitoringData.applicationInstanceId = appInfo.applicationInstanceId;
+            monitoringData.applicationId = appInfo.applicationId;
+            monitoringData.applicationName = appInfo.applicationName;
+            monitoringData.applicationClassName = appInfo.applicationClassName;
+            monitoringData.applicationDomainName = appInfo.applicationDomainName;
+            monitoringData.applicationStage = appInfo.applicationStage;
+            monitoringData.applicationVersion = appInfo.applicationVersion;
             monitoringData.applicationTags = {
                 ...monitoringData.applicationTags,
-                ...applicationInfo.applicationTags,
+                ...appInfo.applicationTags,
             };
         }
     }
@@ -570,6 +579,8 @@ class Utils {
             ...(applicationRegion ? { applicationRegion } : undefined),
             ...(applicationVersion ? { applicationVersion } : undefined),
             ...(applicationStage ? { applicationStage } : undefined),
+            applicationRuntime: 'node',
+            applicationRuntimeVersion: process.version,
             applicationTags: Utils.getApplicationTags(),
         };
     }