Skip to content
This repository has been archived by the owner on Jul 10, 2023. It is now read-only.

Commit

Permalink
Add masked key support while reporting collected traces, spans, metri…
Browse files Browse the repository at this point in the history 
…cs and logs (#396)

* Add masked key support while reporting collected traces, spans, metrics and logs

* Catch and log unexpected errors occurred while serializing masked

* Update doc
  • Loading branch information
Serkan ÖZAL authored Feb 24, 2023
1 parent 992b245 commit 0fa0031
Show file tree
Hide file tree
Showing 7 changed files with 294 additions and 148 deletions.
247 changes: 131 additions & 116 deletions README.md
View file

Large diffs are not rendered by default.

129 changes: 126 additions & 3 deletions src/Reporter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ import * as https from 'https';
import * as url from 'url';
import {
COMPOSITE_MONITORING_DATA_PATH,
getDefaultCollectorEndpoint,
LOCAL_COLLECTOR_ENDPOINT,
SPAN_TAGS_TO_TRIM_1,
SPAN_TAGS_TO_TRIM_2,
SPAN_TAGS_TO_TRIM_3,
REPORTER_HTTP_TIMEOUT,
REPORTER_DATA_SIZE_LIMIT,
getDefaultCollectorEndpoint,
} from './Constants';
import Utils from './utils/Utils';
import ThundraLogger from './ThundraLogger';
Expand All @@ -30,6 +30,11 @@ const httpsAgent = new https.Agent({
keepAlive: true,
});

const REGEXP_PATTERN = /^\/(.*?)\/([gimyu]*)$/;
const MASKED_VALUE = '*****';

type MaskedKey = string | RegExp;

/**
* Reports given telemetry data to given/configured Thundra collector endpoint
*/
Expand All @@ -43,6 +48,8 @@ class Reporter {
private async: boolean;
private trimmers: Trimmer[];
private maxReportSize: number;
private maskedKeys: MaskedKey[];
private hide: boolean;

constructor(apiKey: string, opt: any = {}) {
this.url = url.parse(opt.url || Reporter.getCollectorURL());
Expand All @@ -67,6 +74,29 @@ class Reporter {
`<Reporter> Max report size cannot be bigger than ${REPORTER_DATA_SIZE_LIMIT} ` +
`but it is set to ${this.maxReportSize}. So limiting to ${REPORTER_DATA_SIZE_LIMIT}.`);
}
this.maskedKeys = opt.maskedKeys || Reporter.getMaskedKeys();
this.hide = opt.hide || ConfigProvider.get<boolean>(ConfigNames.THUNDRA_REPORT_HIDE);
}

private static getMaskedKeys(): MaskedKey[] | undefined {
const maskedKeysConfig: string | undefined =
ConfigProvider.get<string>(ConfigNames.THUNDRA_REPORT_MASKED_KEYS);
const maskedKeys: MaskedKey[] = [];
if (maskedKeysConfig) {
for (const maskedKey of maskedKeysConfig.split(',')) {
const regexpParts: string[] = maskedKey.match(REGEXP_PATTERN);
if (regexpParts) {
maskedKeys.push(new RegExp(regexpParts[1], regexpParts[2]));
} else {
maskedKeys.push(maskedKey);
}
}
}
if (maskedKeys && maskedKeys.length) {
return maskedKeys;
} else {
return undefined;
}
}

private static getCollectorURL(): string {
Expand Down Expand Up @@ -306,7 +336,7 @@ class Reporter {
// If trimming is disabled, trim if and only if data size is bigger than maximum allowed limit
const maxReportDataSize: number = disableTrim ? REPORTER_DATA_SIZE_LIMIT : this.maxReportSize;

let json: string = Utils.serializeJSON(batch);
let json: string = this.serializeMasked(batch, this.maskedKeys, this.hide);

if (json.length < maxReportDataSize) {
return json;
Expand All @@ -317,14 +347,107 @@ class Reporter {
if (!trimResult.mutated) {
continue;
}
json = Utils.serializeJSON(batch);
json = this.serializeMasked(batch, this.maskedKeys, this.hide);
if (json.length < maxReportDataSize) {
return json;
}
}
return this.serializeMasked(batch, this.maskedKeys, this.hide);
}

private serializeMasked(batch: any, maskedKeys: MaskedKey[], hide?: boolean): string {
if (maskedKeys && maskedKeys.length) {
try {
ThundraLogger.debug(`<Reporter> Serializing masked ...`);

const maskCheckSet: WeakSet<any> = new WeakSet<any>();

for (const monitoringData of batch.data.allMonitoringData) {
if (monitoringData.tags) {
maskCheckSet.add(monitoringData.tags);
}
}

const result: string =
JSON.stringify(batch, this.createMaskingReplacer(maskCheckSet, maskedKeys, hide));

ThundraLogger.debug(`<Reporter> Serialized masked`);

return result;
} catch (err) {
ThundraLogger.debug(`<Reporter> Error occurred while serializing masked`, err);
}
}
return Utils.serializeJSON(batch);
}

private isMasked(key: string, maskedKeys: MaskedKey[]): boolean {
for (const maskedKey of maskedKeys) {
if (typeof maskedKey === 'string' && maskedKey === key) {
return true;
}
if (maskedKey instanceof RegExp && maskedKey.test(key)) {
return true;
}
}
return false;
}

private createMaskingReplacer(maskCheckSet: WeakSet<any>, maskedKeys: MaskedKey[], hide?: boolean)
: (this: any, key: string, value: any) => any {
const isObject: Function = (o: any) => o != null && typeof o === 'object';
const isArray: Function = (o: any) => o != null && Array.isArray(o);
const isObjectOrArray: Function = (o: any) => isObject(o) || isArray(o);
const isJson = (str: any) =>
typeof str === 'string' &&
(
(str.charAt(0) === '{' && str.charAt(str.length - 1) === '}') ||
(str.charAt(0) === '[' && str.charAt(str.length - 1) === ']')
);

const seen: WeakSet<any> = new WeakSet<any>();
const me = this;

return function (key: string, value: any) {
if (isObject(value)) {
if (seen.has(value)) {
return;
}
seen.add(value);
}

// The parent needs to be checked to check the current property
const checkForMask: boolean = maskCheckSet.has(this);
if (checkForMask) {
if (me.isMasked(key, maskedKeys)) {
if (ThundraLogger.isDebugEnabled()) {
ThundraLogger.debug(`<Reporter> Masking (hide=${hide}) key ${key} ...`);
}
return hide ? undefined : MASKED_VALUE;
} else {
if (isObjectOrArray(value)) {
maskCheckSet.add(value);
} else if (isJson(value)) {
try {
const jsonObj: any = JSON.parse(value);
const jsonMaskCheckSet: WeakSet<any> = new WeakSet<any>();
jsonMaskCheckSet.add(jsonObj);
const maskedJson =
JSON.stringify(jsonObj, me.createMaskingReplacer(jsonMaskCheckSet, maskedKeys, hide));
if (maskedJson) {
value = maskedJson;
}
} catch (e) {
ThundraLogger.debug(
`<Reporter> Unable to mask (hide=${hide}) json with key ${key}: ${value}`, e);
}
}
}
}
return value;
};
}

}

export class TrimResult {
Expand Down
7 changes: 7 additions & 0 deletions src/config/ConfigMetadata.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,13 @@ export const ConfigMetadata: {[key: string]: { type: string, defaultValue?: any
type: 'number',
defaultValue: 32 * 1024, // 32 KB
},
[ConfigNames.THUNDRA_REPORT_MASKED_KEYS]: {
type: 'string',
},
[ConfigNames.THUNDRA_REPORT_HIDE]: {
type: 'boolean',
defaultValue: false,
},
[ConfigNames.THUNDRA_LAMBDA_HANDLER]: {
type: 'string',
},
Expand Down
4 changes: 4 additions & 0 deletions src/config/ConfigNames.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ class ConfigNames {
'thundra.agent.report.cloudwatch.enable';
public static readonly THUNDRA_REPORT_SIZE_MAX: string =
'thundra.agent.report.size.max';
public static readonly THUNDRA_REPORT_MASKED_KEYS: string =
'thundra.agent.report.masked.keys';
public static readonly THUNDRA_REPORT_HIDE: string =
'thundra.agent.report.hide';

/////////////////////////////////////////////////////////////////////////////

Expand Down
17 changes: 8 additions & 9 deletions src/integrations/MySQL2Integration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,20 +142,19 @@ class MySQL2Integration implements Integration {
span.setErrorTag(err);
} else {
try {
let {rowCount, rows} = res;
let { rowCount, rows } = res;
if (!rowCount && res instanceof Array) {
rowCount = res.length;
rows = res;
}
span.addTags({
[DBTags.DB_RESULT_COUNT]: rowCount,
[DBTags.DB_RESULTS]:
config.maskRdbResult
? undefined
: (rows.length > MAX_DB_RESULT_COUNT
span.setTag(DBTags.DB_RESULT_COUNT, rowCount);
if (!config.maskRdbResult && Array.isArray(rows) && rows.length) {
span.setTag(
DBTags.DB_RESULTS,
rows.length > MAX_DB_RESULT_COUNT
? rows.slice(0, MAX_DB_RESULT_COUNT)
: rows),
});
: rows);
}
} catch (e) {
ThundraLogger.debug(`<MySQL2Integration> Unable to capture DB results`, e);
}
Expand Down
19 changes: 9 additions & 10 deletions src/integrations/MySQLIntegration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,20 +148,19 @@ class MySQLIntegration implements Integration {
span.setErrorTag(err);
} else {
try {
let {rowCount, rows} = res;
let { rowCount, rows } = res;
if (!rowCount && res instanceof Array) {
rowCount = res.length;
rows = res;
}
span.addTags({
[DBTags.DB_RESULT_COUNT]: rowCount,
[DBTags.DB_RESULTS]:
config.maskRdbResult
? undefined
: (rows.length > MAX_DB_RESULT_COUNT
? rows.slice(0, MAX_DB_RESULT_COUNT)
: rows),
});
span.setTag(DBTags.DB_RESULT_COUNT, rowCount);
if (!config.maskRdbResult && Array.isArray(rows) && rows.length) {
span.setTag(
DBTags.DB_RESULTS,
rows.length > MAX_DB_RESULT_COUNT
? rows.slice(0, MAX_DB_RESULT_COUNT)
: rows);
}
} catch (e) {
ThundraLogger.debug(`<MySQLIntegration> Unable to capture DB results`, e);
}
Expand Down
19 changes: 9 additions & 10 deletions src/integrations/PostgreIntegration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,20 +120,19 @@ class PostgreIntegration implements Integration {
span.setErrorTag(err);
} else {
try {
let {rowCount, rows} = res;
let { rowCount, rows } = res;
if (!rowCount && res instanceof Array) {
rowCount = res.length;
rows = res;
}
span.addTags({
[DBTags.DB_RESULT_COUNT]: rowCount,
[DBTags.DB_RESULTS]:
config.maskRdbResult
? undefined
: (rows.length > MAX_DB_RESULT_COUNT
? rows.slice(0, MAX_DB_RESULT_COUNT)
: rows),
});
span.setTag(DBTags.DB_RESULT_COUNT, rowCount);
if (!config.maskRdbResult && Array.isArray(rows) && rows.length) {
span.setTag(
DBTags.DB_RESULTS,
rows.length > MAX_DB_RESULT_COUNT
? rows.slice(0, MAX_DB_RESULT_COUNT)
: rows);
}
} catch (e) {
ThundraLogger.debug(`<PostgreIntegration> Unable to capture DB results`, e);
}
Expand Down

0 comments on commit 0fa0031

Please sign in to comment.