Merge remote-tracking branch 'upstream/master' into task/admin-breadc…

…rumbs

.gitignore

@@ -31,6 +31,7 @@ disabledPlugins
 webpackstats.json
 /config/*
 !/config/kibana.yml
+!/config/node.options
 coverage
 selenium
 .babel_register_cache.json

config/node.options

@@ -0,0 +1,6 @@
+## Node command line options
+## See `node --help` and `node --v8-options` for available options
+## Please note you should specify one option per line
+
+## max size of old space in megabytes
+#--max-old-space-size=4096

docs/development/core/server/kibana-plugin-core-server.httpservicesetup.md

@@ -88,8 +88,9 @@ async (context, request, response) => {
 |  [csp](./kibana-plugin-core-server.httpservicesetup.csp.md) | <code>ICspConfig</code> | The CSP config used for Kibana. |
 |  [getServerInfo](./kibana-plugin-core-server.httpservicesetup.getserverinfo.md) | <code>() =&gt; HttpServerInfo</code> | Provides common [information](./kibana-plugin-core-server.httpserverinfo.md) about the running http server. |
 |  [registerAuth](./kibana-plugin-core-server.httpservicesetup.registerauth.md) | <code>(handler: AuthenticationHandler) =&gt; void</code> | To define custom authentication and/or authorization mechanism for incoming requests. |
-|  [registerOnPostAuth](./kibana-plugin-core-server.httpservicesetup.registeronpostauth.md) | <code>(handler: OnPostAuthHandler) =&gt; void</code> | To define custom logic to perform for incoming requests. |
-|  [registerOnPreAuth](./kibana-plugin-core-server.httpservicesetup.registeronpreauth.md) | <code>(handler: OnPreAuthHandler) =&gt; void</code> | To define custom logic to perform for incoming requests. |
+|  [registerOnPostAuth](./kibana-plugin-core-server.httpservicesetup.registeronpostauth.md) | <code>(handler: OnPostAuthHandler) =&gt; void</code> | To define custom logic after Auth interceptor did make sure a user has access to the requested resource. |
+|  [registerOnPreAuth](./kibana-plugin-core-server.httpservicesetup.registeronpreauth.md) | <code>(handler: OnPreAuthHandler) =&gt; void</code> | To define custom logic to perform for incoming requests before the Auth interceptor performs a check that user has access to requested resources. |
 |  [registerOnPreResponse](./kibana-plugin-core-server.httpservicesetup.registeronpreresponse.md) | <code>(handler: OnPreResponseHandler) =&gt; void</code> | To define custom logic to perform for the server response. |
+|  [registerOnPreRouting](./kibana-plugin-core-server.httpservicesetup.registeronprerouting.md) | <code>(handler: OnPreRoutingHandler) =&gt; void</code> | To define custom logic to perform for incoming requests before server performs a route lookup. |
 |  [registerRouteHandlerContext](./kibana-plugin-core-server.httpservicesetup.registerroutehandlercontext.md) | <code>&lt;T extends keyof RequestHandlerContext&gt;(contextName: T, provider: RequestHandlerContextProvider&lt;T&gt;) =&gt; RequestHandlerContextContainer</code> | Register a context provider for a route handler. |
 

docs/development/core/server/kibana-plugin-core-server.httpservicesetup.registeronpostauth.md

@@ -4,7 +4,7 @@
 
 ## HttpServiceSetup.registerOnPostAuth property
 
-To define custom logic to perform for incoming requests.
+To define custom logic after Auth interceptor did make sure a user has access to the requested resource.
 
 <b>Signature:</b>
 
@@ -14,5 +14,5 @@ registerOnPostAuth: (handler: OnPostAuthHandler) => void;
 
 ## Remarks
 
-Runs the handler after Auth interceptor did make sure a user has access to the requested resource. The auth state is available at stage via http.auth.get(..) Can register any number of registerOnPreAuth, which are called in sequence (from the first registered to the last). See [OnPostAuthHandler](./kibana-plugin-core-server.onpostauthhandler.md)<!-- -->.
+The auth state is available at stage via http.auth.get(..) Can register any number of registerOnPreRouting, which are called in sequence (from the first registered to the last). See [OnPostAuthHandler](./kibana-plugin-core-server.onpostauthhandler.md)<!-- -->.
 

docs/development/core/server/kibana-plugin-core-server.httpservicesetup.registeronpreauth.md

@@ -4,7 +4,7 @@
 
 ## HttpServiceSetup.registerOnPreAuth property
 
-To define custom logic to perform for incoming requests.
+To define custom logic to perform for incoming requests before the Auth interceptor performs a check that user has access to requested resources.
 
 <b>Signature:</b>
 
@@ -14,5 +14,5 @@ registerOnPreAuth: (handler: OnPreAuthHandler) => void;
 
 ## Remarks
 
-Runs the handler before Auth interceptor performs a check that user has access to requested resources, so it's the only place when you can forward a request to another URL right on the server. Can register any number of registerOnPostAuth, which are called in sequence (from the first registered to the last). See [OnPreAuthHandler](./kibana-plugin-core-server.onpreauthhandler.md)<!-- -->.
+Can register any number of registerOnPostAuth, which are called in sequence (from the first registered to the last). See [OnPreRoutingHandler](./kibana-plugin-core-server.onpreroutinghandler.md)<!-- -->.
 

docs/development/core/server/kibana-plugin-core-server.httpservicesetup.registeronprerouting.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [HttpServiceSetup](./kibana-plugin-core-server.httpservicesetup.md) &gt; [registerOnPreRouting](./kibana-plugin-core-server.httpservicesetup.registeronprerouting.md)
+
+## HttpServiceSetup.registerOnPreRouting property
+
+To define custom logic to perform for incoming requests before server performs a route lookup.
+
+<b>Signature:</b>
+
+```typescript
+registerOnPreRouting: (handler: OnPreRoutingHandler) => void;
+```
+
+## Remarks
+
+It's the only place when you can forward a request to another URL right on the server. Can register any number of registerOnPreRouting, which are called in sequence (from the first registered to the last). See [OnPreRoutingHandler](./kibana-plugin-core-server.onpreroutinghandler.md)<!-- -->.
+

docs/development/core/server/kibana-plugin-core-server.md

@@ -122,7 +122,8 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [OnPreAuthToolkit](./kibana-plugin-core-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |
 |  [OnPreResponseExtensions](./kibana-plugin-core-server.onpreresponseextensions.md) | Additional data to extend a response. |
 |  [OnPreResponseInfo](./kibana-plugin-core-server.onpreresponseinfo.md) | Response status code. |
-|  [OnPreResponseToolkit](./kibana-plugin-core-server.onpreresponsetoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |
+|  [OnPreResponseToolkit](./kibana-plugin-core-server.onpreresponsetoolkit.md) | A tool set defining an outcome of OnPreRouting interceptor for incoming request. |
+|  [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md) | A tool set defining an outcome of OnPreRouting interceptor for incoming request. |
 |  [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) | Regroups metrics gathered by all the collectors. This contains metrics about the os/runtime, the kibana process and the http server. |
 |  [OpsOsMetrics](./kibana-plugin-core-server.opsosmetrics.md) | OS related metrics |
 |  [OpsProcessMetrics](./kibana-plugin-core-server.opsprocessmetrics.md) | Process related metrics |
@@ -256,7 +257,8 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [MutatingOperationRefreshSetting](./kibana-plugin-core-server.mutatingoperationrefreshsetting.md) | Elasticsearch Refresh setting for mutating operation |
 |  [OnPostAuthHandler](./kibana-plugin-core-server.onpostauthhandler.md) | See [OnPostAuthToolkit](./kibana-plugin-core-server.onpostauthtoolkit.md)<!-- -->. |
 |  [OnPreAuthHandler](./kibana-plugin-core-server.onpreauthhandler.md) | See [OnPreAuthToolkit](./kibana-plugin-core-server.onpreauthtoolkit.md)<!-- -->. |
-|  [OnPreResponseHandler](./kibana-plugin-core-server.onpreresponsehandler.md) | See [OnPreAuthToolkit](./kibana-plugin-core-server.onpreauthtoolkit.md)<!-- -->. |
+|  [OnPreResponseHandler](./kibana-plugin-core-server.onpreresponsehandler.md) | See [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md)<!-- -->. |
+|  [OnPreRoutingHandler](./kibana-plugin-core-server.onpreroutinghandler.md) | See [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md)<!-- -->. |
 |  [PluginConfigSchema](./kibana-plugin-core-server.pluginconfigschema.md) | Dedicated type for plugin configuration schema. |
 |  [PluginInitializer](./kibana-plugin-core-server.plugininitializer.md) | The <code>plugin</code> export at the root of a plugin's <code>server</code> directory should conform to this interface. |
 |  [PluginName](./kibana-plugin-core-server.pluginname.md) | Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays that use it as a key or value more obvious. |

docs/development/core/server/kibana-plugin-core-server.onpreauthtoolkit.md

@@ -17,5 +17,4 @@ export interface OnPreAuthToolkit
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [next](./kibana-plugin-core-server.onpreauthtoolkit.next.md) | <code>() =&gt; OnPreAuthResult</code> | To pass request to the next handler |
-|  [rewriteUrl](./kibana-plugin-core-server.onpreauthtoolkit.rewriteurl.md) | <code>(url: string) =&gt; OnPreAuthResult</code> | Rewrite requested resources url before is was authenticated and routed to a handler |
 

docs/development/core/server/kibana-plugin-core-server.onpreresponsehandler.md

@@ -4,7 +4,7 @@
 
 ## OnPreResponseHandler type
 
-See [OnPreAuthToolkit](./kibana-plugin-core-server.onpreauthtoolkit.md)<!-- -->.
+See [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md)<!-- -->.
 
 <b>Signature:</b>
 

docs/development/core/server/kibana-plugin-core-server.onpreresponsetoolkit.md

@@ -4,7 +4,7 @@
 
 ## OnPreResponseToolkit interface
 
-A tool set defining an outcome of OnPreAuth interceptor for incoming request.
+A tool set defining an outcome of OnPreRouting interceptor for incoming request.
 
 <b>Signature:</b>
 

docs/development/core/server/kibana-plugin-core-server.onpreroutinghandler.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [OnPreRoutingHandler](./kibana-plugin-core-server.onpreroutinghandler.md)
+
+## OnPreRoutingHandler type
+
+See [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md)<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+export declare type OnPreRoutingHandler = (request: KibanaRequest, response: LifecycleResponseFactory, toolkit: OnPreRoutingToolkit) => OnPreRoutingResult | KibanaResponse | Promise<OnPreRoutingResult | KibanaResponse>;
+```

docs/development/core/server/kibana-plugin-core-server.onpreroutingtoolkit.md

@@ -0,0 +1,21 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md)
+
+## OnPreRoutingToolkit interface
+
+A tool set defining an outcome of OnPreRouting interceptor for incoming request.
+
+<b>Signature:</b>
+
+```typescript
+export interface OnPreRoutingToolkit 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [next](./kibana-plugin-core-server.onpreroutingtoolkit.next.md) | <code>() =&gt; OnPreRoutingResult</code> | To pass request to the next handler |
+|  [rewriteUrl](./kibana-plugin-core-server.onpreroutingtoolkit.rewriteurl.md) | <code>(url: string) =&gt; OnPreRoutingResult</code> | Rewrite requested resources url before is was authenticated and routed to a handler |
+

docs/development/core/server/kibana-plugin-core-server.onpreroutingtoolkit.next.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md) &gt; [next](./kibana-plugin-core-server.onpreroutingtoolkit.next.md)
+
+## OnPreRoutingToolkit.next property
+
+To pass request to the next handler
+
+<b>Signature:</b>
+
+```typescript
+next: () => OnPreRoutingResult;
+```

docs/development/core/server/kibana-plugin-core-server.onpreroutingtoolkit.rewriteurl.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md) &gt; [rewriteUrl](./kibana-plugin-core-server.onpreroutingtoolkit.rewriteurl.md)
+
+## OnPreRoutingToolkit.rewriteUrl property
+
+Rewrite requested resources url before is was authenticated and routed to a handler
+
+<b>Signature:</b>
+
+```typescript
+rewriteUrl: (url: string) => OnPreRoutingResult;
+```

docs/migration/migrate_8_0.asciidoc

@@ -115,12 +115,17 @@ URL that it derived from the actual server address and `xpack.security.public` s
 
 *Impact:* Any workflow that involved manually clearing generated bundles will have to be updated with the new path.
 
+[float]]
+=== kibana.keystore has moved from the `data` folder to the `config` folder
+*Details:* By default, kibana.keystore has moved from the configured `path.data` folder to `<root>/config` for archive distributions
+and `/etc/kibana` for package distributions.  If a pre-existing keystore exists in the data directory that path will continue to be used.
+
 [float]
 [[breaking_80_user_role_changes]]
 === User role changes
 
 [float]
-==== `kibana_user` role has been removed and `kibana_admin` has been added.
+=== `kibana_user` role has been removed and `kibana_admin` has been added.
 
 *Details:* The `kibana_user` role has been removed and `kibana_admin` has been added to better
 reflect its intended use. This role continues to grant all access to every

docs/setup/production.asciidoc

@@ -167,9 +167,9 @@ These can be used to automatically update the list of hosts as a cluster is resi
 Kibana has a default maximum memory limit of 1.4 GB, and in most cases, we recommend leaving this unconfigured.  In some scenarios, such as large reporting jobs,
 it may make sense to tweak limits to meet more specific requirements.
 
-You can modify this limit by setting `--max-old-space-size` in the `NODE_OPTIONS` environment variable. For deb and rpm, packages this is passed in via `/etc/default/kibana` and can be appended to the bottom of the file.
+You can modify this limit by setting `--max-old-space-size` in the `node.options` config file that can be found inside `kibana/config` folder or any other configured with the environment variable `KIBANA_PATH_CONF` (for example in debian based system would be `/etc/kibana`).
 
 The option accepts a limit in MB:
 --------
-NODE_OPTIONS="--max-old-space-size=2048" bin/kibana
+--max-old-space-size=2048
 --------

package.json

@@ -67,7 +67,7 @@
     "uiFramework:documentComponent": "cd packages/kbn-ui-framework && yarn documentComponent",
     "kbn:watch": "node scripts/kibana --dev --logging.json=false",
     "build:types": "tsc --p tsconfig.types.json",
-    "docs:acceptApiChanges": "node  --max-old-space-size=6144  scripts/check_published_api_changes.js --accept",
+    "docs:acceptApiChanges": "node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept",
     "kbn:bootstrap": "node scripts/register_git_hook",
     "spec_to_console": "node scripts/spec_to_console",
     "backport-skip-ci": "backport --prDescription \"[skip-ci]\"",
@@ -87,7 +87,6 @@
     "**/@types/hoist-non-react-statics": "^3.3.1",
     "**/@types/chai": "^4.2.11",
     "**/cypress/@types/lodash": "^4.14.155",
-    "**/cypress/lodash": "^4.15.19",
     "**/typescript": "3.9.5",
     "**/graphql-toolkit/lodash": "^4.17.15",
     "**/hoist-non-react-statics": "^3.3.2",

packages/kbn-test/src/failed_tests_reporter/README.md

@@ -7,15 +7,15 @@ A little CLI that runs in CI to find the failed tests in the JUnit reports, then
 To fetch some JUnit reports from a recent build on CI, visit its `Google Cloud Storage Upload Report` and execute the following in the JS Console:
 
 ```js
-copy(`wget "${Array.from($$('a[href$=".xml"]')).filter(a => a.innerText === 'Download').map(a => a.href.replace('https://storage.cloud.google.com/', 'https://storage.googleapis.com/')).join('" "')}"`)
+copy(`wget -x -nH --cut-dirs 5 -P "target/downloaded_junit" "${Array.from($$('a[href$=".xml"]')).filter(a => a.innerText === 'Download').map(a => a.href.replace('https://storage.cloud.google.com/', 'https://storage.googleapis.com/')).join('" "')}"`)
 ```
 
-This copies a script to download the reports, which you should execute in the `test/junit` directory.
+This copies a script to download the reports, which you should execute in the root of the Kibana repository.
 
 Next, run the CLI in `--no-github-update` mode so that it doesn't actually communicate with Github and `--no-report-update` to prevent the script from mutating the reports on disk and instead log the updated report.
 
 ```sh
-node scripts/report_failed_tests.js --verbose --no-github-update --no-report-update
+node scripts/report_failed_tests.js --verbose --no-github-update --no-report-update target/downloaded_junit/**/*.xml
 ```
 
 Unless you specify the `GITHUB_TOKEN` environment variable requests to read existing issues will use anonymous access which is limited to 60 requests per hour.

packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import Path from 'path';
+
 import { REPO_ROOT, run, createFailError, createFlagError } from '@kbn/dev-utils';
 import globby from 'globby';
 
@@ -28,6 +30,8 @@ import { readTestReport } from './test_report';
 import { addMessagesToReport } from './add_messages_to_report';
 import { getReportMessageIter } from './report_metadata';
 
+const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')];
+
 export function runFailedTestsReporterCli() {
   run(
     async ({ log, flags }) => {
@@ -67,11 +71,15 @@ export function runFailedTestsReporterCli() {
         throw createFlagError('Missing --build-url or process.env.BUILD_URL');
       }
 
-      const reportPaths = await globby(['target/junit/**/*.xml'], {
-        cwd: REPO_ROOT,
+      const patterns = flags._.length ? flags._ : DEFAULT_PATTERNS;
+      const reportPaths = await globby(patterns, {
         absolute: true,
       });
 
+      if (!reportPaths.length) {
+        throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`);
+      }
+
       const newlyCreatedIssues: Array<{
         failure: TestFailure;
         newIssue: GithubIssueMini;

src/cli_keystore/cli_keystore.js

@@ -18,20 +18,16 @@
  */
 
 import _ from 'lodash';
-import { join } from 'path';
 
 import { pkg } from '../core/server/utils';
 import Command from '../cli/command';
-import { getDataPath } from '../core/server/path';
 import { Keystore } from '../legacy/server/keystore';
 
-const path = join(getDataPath(), 'kibana.keystore');
-const keystore = new Keystore(path);
-
 import { createCli } from './create';
 import { listCli } from './list';
 import { addCli } from './add';
 import { removeCli } from './remove';
+import { getKeystore } from './get_keystore';
 
 const argv = process.env.kbnWorkerArgv
   ? JSON.parse(process.env.kbnWorkerArgv)
@@ -42,6 +38,8 @@ program
   .version(pkg.version)
   .description('A tool for managing settings stored in the Kibana keystore');
 
+const keystore = new Keystore(getKeystore());
+
 createCli(program, keystore);
 listCli(program, keystore);
 addCli(program, keystore);