Skip to content

Commit

Permalink
feat(observability): spin up micro-benchmarking
Browse files Browse the repository at this point in the history
This requires you to add to your environment

	SPANNER_RUN_BENCHMARKS=true

and then it can be run by npm run observability-test whose results look like:

```shell
  Benchmarking
Total Runs: 10000
databaseRunSelect1AsyncAwait
	RAM Untraced(206.453kB) vs Traced     (240.953kB): increase by (34.500kB) or 16.71%
	RAM Untraced(206.453kB) vs Traced+OTEL(243.391kB): increase by (36.938kB) or 17.89%
	Time Untraced(911.057µs) vs Traced     (1.014ms):  increase by (102.749µs) or 11.28%
	Time Untraced(911.057µs) vs Traced+OTEL(1.017ms):  increase by (106.116µs) or 11.65%

databaseRunSelect1Callback
	RAM Untraced(210.219kB) vs Traced     (238.797kB): increase by (28.578kB) or 13.59%
	RAM Untraced(210.219kB) vs Traced+OTEL(242.914kB): increase by (32.695kB) or 15.55%
	Time Untraced(890.877µs) vs Traced     (997.541µs):  increase by (106.664µs) or 11.97%
	Time Untraced(890.877µs) vs Traced+OTEL(1.019ms):  increase by (127.997µs) or 14.37%

databaseRunTransactionAsyncTxRunUpdate
	RAM Untraced(308.898kB) vs Traced     (325.313kB): increase by (16.414kB) or 5.31%
	RAM Untraced(308.898kB) vs Traced+OTEL(339.836kB): increase by (30.938kB) or 10.02%
	Time Untraced(1.510ms) vs Traced     (1.652ms):  increase by (141.330µs) or 9.36%
	Time Untraced(1.510ms) vs Traced+OTEL(1.668ms):  increase by (157.466µs) or 10.43%

databaseRunTransactionAsyncTxRun
	RAM Untraced(298.977kB) vs Traced     (326.227kB): increase by (27.250kB) or 9.11%
	RAM Untraced(298.977kB) vs Traced+OTEL(315.164kB): increase by (16.188kB) or 5.41%
	Time Untraced(1.450ms) vs Traced     (1.581ms):  increase by (130.870µs) or 9.03%
	Time Untraced(1.450ms) vs Traced+OTEL(1.615ms):  increase by (165.426µs) or 11.41%
```
  • Loading branch information
odeke-em committed Oct 24, 2024
1 parent cbc86fa commit 90fb9d7
Show file tree
Hide file tree
Showing 4 changed files with 474 additions and 1 deletion.
143 changes: 143 additions & 0 deletions observability-test/benchmark.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/*!
* Copyright 2024 Google LLC. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

const lessComparator = (a, b) => {
if (a < b) return -1;
if (a > b) return 1;
return 0;
};

/*
* runBenchmarks runs each of the functions in runners ${nRuns} times
* each time collecting RAM usage and time spent and then produces
* a map of functionNames to the percentiles of RAM usage and time spent.
*/
export async function runBenchmarks(runners: Function[], done: Function) {
const nRuns = 10000;
const benchmarkValues = {_totalRuns: nRuns};

let k = 0;
for (k = 0; k < runners.length; k++) {
const fn = runners[k];
const functionName = fn.name;
const timeSpentL: bigint[] = [];
const ramL: number[] = [];
let i = 0;

// Warm up runs to ensure stable behavior.
for (i = 0; i < 8; i++) {
const value = await fn();
}

for (i = 0; i < nRuns; i++) {
const startTime: bigint = process.hrtime.bigint();
const startHeapUsedBytes: number = process.memoryUsage().heapUsed;
const value = await fn();
timeSpentL.push(process.hrtime.bigint() - startTime);
ramL.push(process.memoryUsage().heapUsed - startHeapUsedBytes);
}

timeSpentL.sort(lessComparator);
ramL.sort(lessComparator);

benchmarkValues[functionName] = {
ram: percentiles(functionName, ramL, 'bytes'),
timeSpent: percentiles(functionName, timeSpentL, 'time'),
};
}

done(benchmarkValues);
}

function percentiles(method, sortedValues, kind) {
const n = sortedValues.length;
const p50 = sortedValues[Math.floor(n * 0.5)];
const p75 = sortedValues[Math.floor(n * 0.75)];
const p90 = sortedValues[Math.floor(n * 0.9)];
const p95 = sortedValues[Math.floor(n * 0.95)];
const p99 = sortedValues[Math.floor(n * 0.99)];

return {
p50: p50,
p75: p75,
p90: p90,
p95: p95,
p99: p99,
p50_s: humanize(p50, kind),
p75_s: humanize(p75, kind),
p90_s: humanize(p90, kind),
p95_s: humanize(p95, kind),
p99_s: humanize(p99, kind),
};
}

function humanize(values, kind) {
let converterFn = humanizeTime;
if (kind === 'bytes') {
converterFn = humanizeBytes;
}
return converterFn(values);
}

const secondUnits = ['ns', 'µs', 'ms', 's'];
interface unitDivisor {
unit: string;
divisor: number;
}
const pastSecondUnits: unitDivisor[] = [
{unit: 'min', divisor: 60},
{unit: 'hr', divisor: 60},
{unit: 'day', divisor: 24},
{unit: 'week', divisor: 7},
{unit: 'month', divisor: 30},
];
function humanizeTime(ns) {
const sign: number = ns < 0 ? -1 : +1;
let value = Math.abs(Number(ns));
for (const unit of secondUnits) {
if (value < 1000) {
return `${(sign * value).toFixed(3)}${unit}`;
}
value /= 1000;
}

let i = 0;
for (i = 0; i < pastSecondUnits.length; i++) {
const unitPlusValue = pastSecondUnits[i];
const unitName = unitPlusValue.unit;
const divisor = unitPlusValue.divisor;
if (value < divisor) {
return `${sign * value}${unitName}`;
}
value = value / divisor;
}
return `${(sign * value).toFixed(3)}${pastSecondUnits[pastSecondUnits.length - 1][0]}`;
}

const bytesUnits = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'ExB'];
function humanizeBytes(b) {
const sign: number = b < 0 ? -1 : +1;
let value = Math.abs(b);
for (const unit of bytesUnits) {
if (value < 1024) {
return `${(sign * value).toFixed(3)}${unit}`;
}
value = value / 1024;
}

return `${(sign * value).toFixed(3)}${bytesUnits[bytesUnits.length - 1]}`;
}
export {humanizeTime, humanizeBytes};
Loading

0 comments on commit 90fb9d7

Please sign in to comment.