Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into nls/obs-bread
Browse files Browse the repository at this point in the history
  • Loading branch information
smith committed Jun 16, 2021
2 parents 59196b6 + 35e10ba commit eba7482
Show file tree
Hide file tree
Showing 65 changed files with 1,238 additions and 145 deletions.
4 changes: 2 additions & 2 deletions .ci/end2end.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ pipeline {
BASE_DIR = 'src/github.com/elastic/kibana'
HOME = "${env.WORKSPACE}"
E2E_DIR = 'x-pack/plugins/apm/e2e'
PIPELINE_LOG_LEVEL = 'DEBUG'
PIPELINE_LOG_LEVEL = 'INFO'
KBN_OPTIMIZER_THEMES = 'v7light'
}
options {
timeout(time: 1, unit: 'HOURS')
buildDiscarder(logRotator(numToKeepStr: '40', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
buildDiscarder(logRotator(numToKeepStr: '30', artifactNumToKeepStr: '10', daysToKeepStr: '30'))
timestamps()
ansiColor('xterm')
disableResume()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export interface ExpressionFunctionDefinitions
| [derivative](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.derivative.md) | <code>ExpressionFunctionDerivative</code> | |
| [font](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.font.md) | <code>ExpressionFunctionFont</code> | |
| [moving\_average](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.moving_average.md) | <code>ExpressionFunctionMovingAverage</code> | |
| [overall\_metric](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md) | <code>ExpressionFunctionOverallMetric</code> | |
| [theme](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.theme.md) | <code>ExpressionFunctionTheme</code> | |
| [var\_set](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.var_set.md) | <code>ExpressionFunctionVarSet</code> | |
| [var](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.var.md) | <code>ExpressionFunctionVar</code> | |
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->

[Home](./index.md) &gt; [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) &gt; [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.md) &gt; [overall\_metric](./kibana-plugin-plugins-expressions-public.expressionfunctiondefinitions.overall_metric.md)

## ExpressionFunctionDefinitions.overall\_metric property

<b>Signature:</b>

```typescript
overall_metric: ExpressionFunctionOverallMetric;
```
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export interface ExpressionFunctionDefinitions
| [derivative](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.derivative.md) | <code>ExpressionFunctionDerivative</code> | |
| [font](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.font.md) | <code>ExpressionFunctionFont</code> | |
| [moving\_average](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.moving_average.md) | <code>ExpressionFunctionMovingAverage</code> | |
| [overall\_metric](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md) | <code>ExpressionFunctionOverallMetric</code> | |
| [theme](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.theme.md) | <code>ExpressionFunctionTheme</code> | |
| [var\_set](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.var_set.md) | <code>ExpressionFunctionVarSet</code> | |
| [var](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.var.md) | <code>ExpressionFunctionVar</code> | |
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<!-- Do not edit this file. It is automatically generated by API Documenter. -->

[Home](./index.md) &gt; [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) &gt; [ExpressionFunctionDefinitions](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.md) &gt; [overall\_metric](./kibana-plugin-plugins-expressions-server.expressionfunctiondefinitions.overall_metric.md)

## ExpressionFunctionDefinitions.overall\_metric property

<b>Signature:</b>

```typescript
overall_metric: ExpressionFunctionOverallMetric;
```
16 changes: 12 additions & 4 deletions src/dev/typescript/build_ts_refs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,20 @@ import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';

export const REF_CONFIG_PATHS = [Path.resolve(REPO_ROOT, 'tsconfig.refs.json')];

export async function buildAllTsRefs(log: ToolingLog) {
export async function buildAllTsRefs(log: ToolingLog): Promise<{ failed: boolean }> {
for (const path of REF_CONFIG_PATHS) {
const relative = Path.relative(REPO_ROOT, path);
log.debug(`Building TypeScript projects refs for ${relative}...`);
await execa(require.resolve('typescript/bin/tsc'), ['-b', relative, '--pretty'], {
cwd: REPO_ROOT,
});
const { failed, stdout } = await execa(
require.resolve('typescript/bin/tsc'),
['-b', relative, '--pretty'],
{
cwd: REPO_ROOT,
reject: false,
}
);
log.info(stdout);
if (failed) return { failed };
}
return { failed: false };
}
6 changes: 5 additions & 1 deletion src/dev/typescript/run_type_check_cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ export async function runTypeCheckCli() {
process.exit();
}

await buildAllTsRefs(log);
const { failed } = await buildAllTsRefs(log);
if (failed) {
log.error('Unable to build TS project refs');
process.exit(1);
}

const tscArgs = [
// composite project cannot be used with --noEmit
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,57 @@ const nestedTermResponse = {
status: 200,
};

const exhaustiveNestedTermResponse = {
took: 10,
timed_out: false,
_shards: {
total: 1,
successful: 1,
skipped: 0,
failed: 0,
},
hits: {
total: 14005,
max_score: 0,
hits: [],
},
aggregations: {
'1': {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 8325,
buckets: [
{
'2': {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{ key: 'ios', doc_count: 2850 },
{ key: 'win xp', doc_count: 2830 },
{ key: '__missing__', doc_count: 1430 },
],
},
key: 'US-with-dash',
doc_count: 2850,
},
{
'2': {
doc_count_error_upper_bound: 0,
sum_other_doc_count: 0,
buckets: [
{ key: 'ios', doc_count: 1850 },
{ key: 'win xp', doc_count: 1830 },
{ key: '__missing__', doc_count: 130 },
],
},
key: 'IN-with-dash',
doc_count: 2830,
},
],
},
},
status: 200,
};

const nestedTermResponseNoResults = {
took: 10,
timed_out: false,
Expand Down Expand Up @@ -326,6 +377,17 @@ describe('Terms Agg Other bucket helper', () => {
}
});

test('does not build query if sum_other_doc_count is 0 (exhaustive terms)', () => {
const aggConfigs = getAggConfigs(nestedTerm.aggs);
expect(
buildOtherBucketAgg(
aggConfigs,
aggConfigs.aggs[1] as IBucketAggConfig,
exhaustiveNestedTermResponse
)
).toBeFalsy();
});

test('excludes exists filter for scripted fields', () => {
const aggConfigs = getAggConfigs(nestedTerm.aggs);
aggConfigs.aggs[1].params.field.scripted = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ export const buildOtherBucketAgg = (
};

let noAggBucketResults = false;
let exhaustiveBuckets = true;

// recursively create filters for all parent aggregation buckets
const walkBucketTree = (
Expand All @@ -175,6 +176,9 @@ export const buildOtherBucketAgg = (
const newAggIndex = aggIndex + 1;
const newAgg = bucketAggs[newAggIndex];
const currentAgg = bucketAggs[aggIndex];
if (aggIndex === index && agg && agg.sum_other_doc_count > 0) {
exhaustiveBuckets = false;
}
if (aggIndex < index) {
each(agg.buckets, (bucket: any, bucketObjKey) => {
const bucketKey = currentAgg.getKey(
Expand Down Expand Up @@ -223,7 +227,7 @@ export const buildOtherBucketAgg = (
walkBucketTree(0, response.aggregations, bucketAggs[0].id, [], '');

// bail if there were no bucket results
if (noAggBucketResults) {
if (noAggBucketResults || exhaustiveBuckets) {
return false;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export * from './var_set';
export * from './var';
export * from './theme';
export * from './cumulative_sum';
export * from './overall_metric';
export * from './derivative';
export * from './moving_average';
export * from './ui_setting';
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

import { i18n } from '@kbn/i18n';
import { ExpressionFunctionDefinition } from '../types';
import { Datatable } from '../../expression_types';
import { buildResultColumns, getBucketIdentifier } from '../series_calculation_helpers';

export interface OverallMetricArgs {
by?: string[];
inputColumnId: string;
outputColumnId: string;
outputColumnName?: string;
metric: 'sum' | 'min' | 'max' | 'average';
}

export type ExpressionFunctionOverallMetric = ExpressionFunctionDefinition<
'overall_metric',
Datatable,
OverallMetricArgs,
Datatable
>;

function getValueAsNumberArray(value: unknown) {
if (Array.isArray(value)) {
return value.map((innerVal) => Number(innerVal));
} else {
return [Number(value)];
}
}

/**
* Calculates the overall metric of a specified column in the data table.
*
* Also supports multiple series in a single data table - use the `by` argument
* to specify the columns to split the calculation by.
* For each unique combination of all `by` columns a separate overall metric will be calculated.
* The order of rows won't be changed - this function is not modifying any existing columns, it's only
* adding the specified `outputColumnId` column to every row of the table without adding or removing rows.
*
* Behavior:
* * Will write the overall metric of `inputColumnId` into `outputColumnId`
* * If provided will use `outputColumnName` as name for the newly created column. Otherwise falls back to `outputColumnId`
* * Each cell will contain the calculated metric based on the values of all cells belonging to the current series.
*
* Edge cases:
* * Will return the input table if `inputColumnId` does not exist
* * Will throw an error if `outputColumnId` exists already in provided data table
* * If the row value contains `null` or `undefined`, it will be ignored and overwritten with the overall metric of
* all cells of the same series.
* * For all values besides `null` and `undefined`, the value will be cast to a number before it's added to the
* overall metric of the current series - if this results in `NaN` (like in case of objects), all cells of the
* current series will be set to `NaN`.
* * To determine separate series defined by the `by` columns, the values of these columns will be cast to strings
* before comparison. If the values are objects, the return value of their `toString` method will be used for comparison.
* Missing values (`null` and `undefined`) will be treated as empty strings.
*/
export const overallMetric: ExpressionFunctionOverallMetric = {
name: 'overall_metric',
type: 'datatable',

inputTypes: ['datatable'],

help: i18n.translate('expressions.functions.overallMetric.help', {
defaultMessage: 'Calculates the overall sum, min, max or average of a column in a data table',
}),

args: {
by: {
help: i18n.translate('expressions.functions.overallMetric.args.byHelpText', {
defaultMessage: 'Column to split the overall calculation by',
}),
multi: true,
types: ['string'],
required: false,
},
metric: {
help: i18n.translate('expressions.functions.overallMetric.metricHelpText', {
defaultMessage: 'Metric to calculate',
}),
types: ['string'],
options: ['sum', 'min', 'max', 'average'],
},
inputColumnId: {
help: i18n.translate('expressions.functions.overallMetric.args.inputColumnIdHelpText', {
defaultMessage: 'Column to calculate the overall metric of',
}),
types: ['string'],
required: true,
},
outputColumnId: {
help: i18n.translate('expressions.functions.overallMetric.args.outputColumnIdHelpText', {
defaultMessage: 'Column to store the resulting overall metric in',
}),
types: ['string'],
required: true,
},
outputColumnName: {
help: i18n.translate('expressions.functions.overallMetric.args.outputColumnNameHelpText', {
defaultMessage: 'Name of the column to store the resulting overall metric in',
}),
types: ['string'],
required: false,
},
},

fn(input, { by, inputColumnId, outputColumnId, outputColumnName, metric }) {
const resultColumns = buildResultColumns(
input,
outputColumnId,
inputColumnId,
outputColumnName
);

if (!resultColumns) {
return input;
}

const accumulators: Partial<Record<string, number>> = {};
const valueCounter: Partial<Record<string, number>> = {};
input.rows.forEach((row) => {
const bucketIdentifier = getBucketIdentifier(row, by);
const accumulatorValue = accumulators[bucketIdentifier] ?? 0;

const currentValue = row[inputColumnId];
if (currentValue != null) {
const currentNumberValues = getValueAsNumberArray(currentValue);
switch (metric) {
case 'average':
valueCounter[bucketIdentifier] =
(valueCounter[bucketIdentifier] ?? 0) + currentNumberValues.length;
case 'sum':
accumulators[bucketIdentifier] =
accumulatorValue + currentNumberValues.reduce((a, b) => a + b, 0);
break;
case 'min':
accumulators[bucketIdentifier] = Math.min(accumulatorValue, ...currentNumberValues);
break;
case 'max':
accumulators[bucketIdentifier] = Math.max(accumulatorValue, ...currentNumberValues);
break;
}
}
});
if (metric === 'average') {
Object.keys(accumulators).forEach((bucketIdentifier) => {
accumulators[bucketIdentifier] =
accumulators[bucketIdentifier]! / valueCounter[bucketIdentifier]!;
});
}
return {
...input,
columns: resultColumns,
rows: input.rows.map((row) => {
const newRow = { ...row };
const bucketIdentifier = getBucketIdentifier(row, by);
newRow[outputColumnId] = accumulators[bucketIdentifier];

return newRow;
}),
};
},
};
Loading

0 comments on commit eba7482

Please sign in to comment.