Merge branch 'main' into unknown-so-types-docs

.gitignore

@@ -66,6 +66,7 @@ npm-debug.log*
 .ci/bash_standard_lib.sh
 .gradle
 .vagrant
+.envrc
 
 ## @cypress/snapshot from apm plugin
 /snapshots.js

docs/settings/apm-settings.asciidoc

@@ -74,7 +74,7 @@ Changing these settings may disable features of the APM App.
   | Index name where Observability annotations are stored. Defaults to `observability-annotations`.
 
 | `xpack.apm.searchAggregatedTransactions` {ess-icon}
-  | experimental[] Enables Transaction histogram metrics. Defaults to `never` and aggregated transactions are not used. When set to `auto`, the UI will use metric indices over transaction indices for transactions if aggregated transactions are found. When set to `always`, additional configuration in APM Server is required.
+  | Enables Transaction histogram metrics. Defaults to `auto` so the UI will use metric indices over transaction indices for transactions if aggregated transactions are found. When set to `always`, additional configuration in APM Server is required. When set to `never` and aggregated transactions are not used.
     See {apm-guide-ref}/transaction-metrics.html[Configure transaction metrics] for more information.
 
 | `xpack.apm.metricsInterval` {ess-icon}

docs/user/dashboard/tsvb.asciidoc

@@ -191,6 +191,30 @@ For example `dashboards#/view/f193ca90-c9f4-11eb-b038-dd3270053a27`.
 . In the toolbar, click *Save as*, then make sure *Store time with dashboard* is deselected. 
 ====
 
+[discrete]
+[[how-do-i-base-drilldowns-on-data]]
+.*How do I base drilldown URLs on my data?*
+[%collapsible]
+====
+
+You can build drilldown URLs dynamically with your visualization data.
+
+Do this by adding the `{{key}}` placeholder to your URL
+
+For example `https://example.org/{{key}}`
+
+This instructs TSVB to substitute the value from your visualization wherever it sees `{{key}}`.
+
+If your data contain reserved or invalid URL characters such as "#" or "&", you should apply a transform to URL-encode the key like this `{{encodeURIComponent key}}`. If you are dynamically constructing a drilldown to another location in Kibana (for example, clicking a table row takes to you a value-scoped saved search), you will likely want to Rison-encode your key as it may contain invalid Rison characters. (https://github.com/Nanonid/rison#rison---compact-data-in-uris[Rison] is the serialization format many parts of Kibana use to store information in their URL.)
+
+For example: `discover#/view/0ac50180-82d9-11ec-9f4a-55de56b00cc0?_a=(filters:!((query:(match_phrase:(foo.keyword:{{rison key}})))))`
+
+If both conditions apply, you can cover all cases by applying both transforms: `{{encodeURIComponent (rison key)}}`.
+
+Technical note: TSVB uses https://handlebarsjs.com/[Handlebars] to perform these interpolations. `rison` and `encodeURIComponent` are custom Handlebars helpers provided by Kibana.
+
+====
+
 [discrete]
 [[why-is-my-tsvb-visualiztion-missing-data]]
 .*Why is my TSVB visualization missing data?*

package.json

@@ -732,7 +732,7 @@
     "babel-plugin-require-context-hook": "^1.0.0",
     "babel-plugin-styled-components": "^2.0.2",
     "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
-    "backport": "6.1.5",
+    "backport": "7.0.0",
     "callsites": "^3.1.0",
     "chai": "3.5.0",
     "chance": "1.0.18",

src/plugins/vis_types/timelion/server/plugin.ts

@@ -17,7 +17,6 @@ import { CoreSetup, PluginInitializerContext, Plugin } from '../../../../../src/
 import { configSchema } from '../config';
 import loadFunctions from './lib/load_functions';
 import { functionsRoute } from './routes/functions';
-import { validateEsRoute } from './routes/validate_es';
 import { runRoute } from './routes/run';
 import { ConfigManager } from './lib/config_manager';
 import { getUiSettings } from './ui_settings';
@@ -66,7 +65,6 @@ export class TimelionPlugin implements Plugin<void, void, TimelionPluginStartDep
 
     functionsRoute(router, deps);
     runRoute(router, deps);
-    validateEsRoute(router);
 
     core.uiSettings.register(getUiSettings(config));
   }

src/plugins/vis_types/timeseries/public/application/components/lib/replace_vars.ts

@@ -6,11 +6,55 @@
  * Side Public License, v 1.
  */
 
-import handlebars from 'handlebars';
+import { encode, RisonValue } from 'rison-node';
+import {
+  create as createHandlebars,
+  compile as compileHandlebars,
+  HelperDelegate,
+  HelperOptions,
+} from 'handlebars';
 import { i18n } from '@kbn/i18n';
 import { emptyLabel } from '../../../../common/empty_label';
 
-type CompileOptions = Parameters<typeof handlebars.compile>[1];
+type CompileOptions = Parameters<typeof compileHandlebars>[1];
+
+const handlebars = createHandlebars();
+
+function createSerializationHelper(
+  fnName: string,
+  serializeFn: (value: unknown) => string
+): HelperDelegate {
+  return (...args) => {
+    const { hash } = args.slice(-1)[0] as HelperOptions;
+    const hasHash = Object.keys(hash).length > 0;
+    const hasValues = args.length > 1;
+    if (hasHash && hasValues) {
+      throw new Error(`[${fnName}]: both value list and hash are not supported`);
+    }
+    if (hasHash) {
+      if (Object.values(hash).some((v) => typeof v === 'undefined'))
+        throw new Error(`[${fnName}]: unknown variable`);
+      return serializeFn(hash);
+    } else {
+      const values = args.slice(0, -1) as unknown[];
+      if (values.some((value) => typeof value === 'undefined'))
+        throw new Error(`[${fnName}]: unknown variable`);
+      if (values.length === 0) throw new Error(`[${fnName}]: unknown variable`);
+      if (values.length === 1) return serializeFn(values[0]);
+      return serializeFn(values);
+    }
+  };
+}
+
+handlebars.registerHelper(
+  'rison',
+  createSerializationHelper('rison', (v) => encode(v as RisonValue))
+);
+
+handlebars.registerHelper('encodeURIComponent', (component: unknown) => {
+  const str = String(component);
+  return encodeURIComponent(str);
+});
 
 export function replaceVars(
   str: string,
@@ -24,6 +68,10 @@ export function replaceVars(
     const template = handlebars.compile(str.split(emptyLabel).join(`[${emptyLabel}]`), {
       strict: true,
       knownHelpersOnly: true,
+      knownHelpers: {
+        rison: true,
+        encodeURIComponent: true,
+      },
       ...compileOptions,
     });
     const string = template({

src/plugins/vis_types/timeseries/public/application/components/vis_types/table/vis.js

@@ -88,7 +88,7 @@ class TableVis extends Component {
     }
 
     if (model.drilldown_url) {
-      const url = replaceVars(model.drilldown_url, {}, { key: row.key });
+      const url = replaceVars(model.drilldown_url, {}, { key: row.key }, { noEscape: true });
       const handleDrilldownUrlClick = this.createDrilldownUrlClickHandler(url);
       rowDisplay = (
         <a

src/plugins/vis_types/timeseries/public/application/components/vis_types/top_n/vis.js

@@ -86,7 +86,7 @@ function TopNVisualization(props) {
 
   if (model.drilldown_url) {
     params.onClick = (item) => {
-      const url = replaceVars(model.drilldown_url, {}, { key: item.label });
+      const url = replaceVars(model.drilldown_url, {}, { key: item.label }, { noEscape: true });
       const validatedUrl = coreStart.http.externalUrl.validateUrl(url);
       if (validatedUrl) {
         setAccessDeniedDrilldownUrl(null);

src/plugins/vis_types/timeseries/tsconfig.json

@@ -10,6 +10,7 @@
     "common/**/*",
     "public/**/*",
     "server/**/*",
+    "../../../../typings/**/*",
     "*.ts"
   ],
   "references": [

src/plugins/vis_types/vega/public/vega_inspector/vega_adapter.ts

@@ -8,7 +8,7 @@
 
 import { i18n } from '@kbn/i18n';
 
-import { Observable, ReplaySubject, fromEventPattern, merge, timer } from 'rxjs';
+import { Observable, ReplaySubject, fromEventPattern, merge, timer, BehaviorSubject } from 'rxjs';
 import { map, switchMap, filter, debounce } from 'rxjs/operators';
 import type { View, Spec } from 'vega';
 import type { Assign } from '@kbn/utility-types';
@@ -81,6 +81,7 @@ const serializeColumns = (item: Record<string, unknown>, columns: string[]) => {
 
 export class VegaAdapter {
   private debugValuesSubject = new ReplaySubject<DebugValues>();
+  private error = new BehaviorSubject<string | undefined>(undefined);
 
   bindInspectValues(debugValues: DebugValues) {
     this.debugValuesSubject.next(debugValues);
@@ -159,4 +160,16 @@ export class VegaAdapter {
       map((debugValues) => JSON.stringify(debugValues.spec, null, 2))
     );
   }
+
+  getErrorObservable() {
+    return this.error.asObservable();
+  }
+
+  setError(error: string) {
+    this.error.next(error);
+  }
+
+  clearError() {
+    this.error.next(undefined);
+  }
 }

src/plugins/vis_types/vega/public/vega_inspector/vega_data_inspector.tsx

@@ -8,8 +8,8 @@
 
 import './vega_data_inspector.scss';
 
-import React from 'react';
-import { EuiTabbedContent } from '@elastic/eui';
+import React, { useState, useEffect } from 'react';
+import { EuiTabbedContent, EuiCallOut } from '@elastic/eui';
 
 import { i18n } from '@kbn/i18n';
 import { VegaInspectorAdapters } from './vega_inspector';
@@ -31,6 +31,31 @@ const specLabel = i18n.translate('visTypeVega.inspector.specLabel', {
 });
 
 const VegaDataInspector = ({ adapters }: VegaDataInspectorProps) => {
+  const [error, setError] = useState<string | undefined>();
+
+  useEffect(() => {
+    const subscription = adapters.vega.getErrorObservable().subscribe((data) => {
+      setError(data);
+    });
+    return () => {
+      subscription.unsubscribe();
+    };
+  }, [adapters.vega]);
+
+  if (error) {
+    return (
+      <EuiCallOut
+        title={i18n.translate('visTypeVega.inspector.errorHeading', {
+          defaultMessage: `Vega didn't render successfully`,
+        })}
+        color="danger"
+        iconType="alert"
+      >
+        <p>{error}</p>
+      </EuiCallOut>
+    );
+  }
+
   const tabs = [
     {
       id: 'data-viewer--id',

src/plugins/vis_types/vega/public/vega_view/vega_base_view.js

@@ -106,7 +106,7 @@ export class VegaBaseView {
       }
 
       if (this._parser.error) {
-        this._addMessage('err', this._parser.error);
+        this.onError(this._parser.error);
         return;
       }
 
@@ -235,7 +235,9 @@ export class VegaBaseView {
   }
 
   onError() {
-    this._addMessage('err', Utils.formatErrorToStr(...arguments));
+    const error = Utils.formatErrorToStr(...arguments);
+    this._addMessage('err', error);
+    this._parser.searchAPI.inspectorAdapters?.vega.setError(error);
   }
 
   onWarn() {

src/plugins/vis_types/vega/public/vega_visualization.ts

@@ -64,6 +64,7 @@ export const createVegaVisualization = (
 
     async _render(vegaParser: VegaParser) {
       if (vegaParser) {
+        vegaParser.searchAPI.inspectorAdapters?.vega.clearError();
         // New data received, rebuild the graph
         if (this.vegaView) {
           await this.vegaView.destroy();

x-pack/plugins/security_solution/server/lib/telemetry/sender.ts

@@ -318,6 +318,14 @@ export class TelemetryEventsSender implements ITelemetryEventsSender {
       this.logger.debug(`Events sent!. Response: ${resp.status} ${JSON.stringify(resp.data)}`);
     } catch (err) {
       this.logger.debug(`Error sending events: ${err}`);
+      const errorStatus = err?.response?.status;
+      if (errorStatus !== undefined && errorStatus !== null) {
+        this.telemetryUsageCounter?.incrementCounter({
+          counterName: createUsageCounterLabel(usageLabelPrefix.concat(['payloads', channel])),
+          counterType: errorStatus.toString(),
+          incrementBy: 1,
+        });
+      }
       this.telemetryUsageCounter?.incrementCounter({
         counterName: createUsageCounterLabel(usageLabelPrefix.concat(['payloads', channel])),
         counterType: 'docs_lost',