Merge branch '7.x' into backport/7.x/pr-55733

.ci/Jenkinsfile_coverage

@@ -0,0 +1,112 @@
+#!/bin/groovy
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load() // load from the Jenkins instance
+
+stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
+  timeout(time: 180, unit: 'MINUTES') {
+    timestamps {
+      ansiColor('xterm') {
+        catchError {
+          withEnv([
+            'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
+          ]) {
+            parallel([
+              'kibana-intake-agent': {
+                withEnv([
+                  'NODE_ENV=test' // Needed for jest tests only
+                ]) {
+                  kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh')()
+                }
+              },
+              'x-pack-intake-agent': {
+                withEnv([
+                  'NODE_ENV=test' // Needed for jest tests only
+                ]) {
+                  kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh')()
+                }
+              },
+              'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+                'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
+                'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
+                'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
+                'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
+                'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
+                'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
+                'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
+                'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
+                'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
+                'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
+                'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
+                'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
+              ]),
+              'kibana-xpack-agent-1': kibanaPipeline.withWorkers('kibana-xpack-tests-1', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
+                'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
+              ]),
+              'kibana-xpack-agent-2': kibanaPipeline.withWorkers('kibana-xpack-tests-2', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
+                'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
+              ]),
+
+              'kibana-xpack-agent-3': kibanaPipeline.withWorkers('kibana-xpack-tests-3', { kibanaPipeline.buildXpack() }, [
+                'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
+                'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
+                'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
+                'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
+                'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
+                'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
+              ]),
+            ])
+            kibanaPipeline.jobRunner('tests-l', false) {
+              kibanaPipeline.downloadCoverageArtifacts()
+              kibanaPipeline.bash(
+                '''
+                  # bootstrap from x-pack folder
+                  source src/dev/ci_setup/setup_env.sh
+                  cd x-pack
+                  yarn kbn bootstrap --prefer-offline
+                  cd ..
+                  # extract archives
+                  mkdir -p /tmp/extracted_coverage
+                  echo extracting intakes
+                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  echo extracting kibana-oss-tests
+                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  echo extracting kibana-xpack-tests
+                  for i in {1..3}; do
+                    tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests-${i}/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+                  done
+                  # replace path in json files to have valid html report
+                  pwd=$(pwd)
+                  du -sh /tmp/extracted_coverage/target/kibana-coverage/
+                  echo replacing path in json files
+                  for i in {1..9}; do
+                    sed -i "s|/dev/shm/workspace/kibana|$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/${i}*.json &
+                  done
+                  wait
+                  # merge oss & x-pack reports
+                  echo merging coverage reports
+                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary
+                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary
+                  echo copy mocha reports
+                  mkdir -p target/kibana-coverage/mocha-combined
+                  cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined
+                ''',
+                "run `yarn kbn bootstrap && merge coverage`"
+              )
+              sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*'
+              kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz')
+              sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*'
+              kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz')
+              sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*'
+              kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz')
+            }
+          }
+        }
+        kibanaPipeline.sendMail()
+      }
+    }
+  }
+}

.ci/es-snapshots/Jenkinsfile_verify_es

@@ -26,8 +26,8 @@ timeout(time: 120, unit: 'MINUTES') {
         withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
           parallel([
             // TODO we just need to run integration tests from intake?
-            'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'),
-            'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'),
+            'kibana-intake-agent': kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh'),
+            'x-pack-intake-agent': kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
             'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
               'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
               'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),

.ci/jobs.yml

@@ -1,3 +1,5 @@
+# This file is needed by functionalTests:ensureAllTestsInCiGroup for the list of ciGroups. That must be changed before this file can be removed
+
 JOB:
   - kibana-intake
   - x-pack-intake

.eslintrc.js

@@ -250,15 +250,15 @@ module.exports = {
               {
                 target: [
                   '(src|x-pack)/plugins/**/*',
-                  '!(src|x-pack)/plugins/*/server/**/*',
+                  '!(src|x-pack)/plugins/**/server/**/*',
 
                   'src/legacy/core_plugins/**/*',
-                  '!src/legacy/core_plugins/*/server/**/*',
-                  '!src/legacy/core_plugins/*/index.{js,ts,tsx}',
+                  '!src/legacy/core_plugins/**/server/**/*',
+                  '!src/legacy/core_plugins/**/index.{js,ts,tsx}',
 
                   'x-pack/legacy/plugins/**/*',
-                  '!x-pack/legacy/plugins/*/server/**/*',
-                  '!x-pack/legacy/plugins/*/index.{js,ts,tsx}',
+                  '!x-pack/legacy/plugins/**/server/**/*',
+                  '!x-pack/legacy/plugins/**/index.{js,ts,tsx}',
 
                   'examples/**/*',
                   '!examples/**/server/**/*',

.i18nrc.json

@@ -12,6 +12,7 @@
     "embeddableExamples": "examples/embeddable_examples",
     "share": "src/plugins/share",
     "home": "src/plugins/home",
+    "charts": "src/plugins/charts",
     "esUi": "src/plugins/es_ui_shared",
     "devTools": "src/plugins/dev_tools",
     "expressions": "src/plugins/expressions",

CONTRIBUTING.md

@@ -338,7 +338,7 @@ The `config/kibana.yml` file stores user configuration directives. Since this fi
 
 #### Setting Up SSL
 
-Kibana includes a self-signed certificate that can be used for development purposes: `yarn start --ssl`.
+Kibana includes self-signed certificates that can be used for development purposes in the browser and for communicating with Elasticsearch: `yarn start --ssl` & `yarn es snapshot --ssl`.
 
 ### Linting
 

Jenkinsfile

@@ -4,16 +4,21 @@ library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
 stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
-  timeout(time: 120, unit: 'MINUTES') {
+  timeout(time: 135, unit: 'MINUTES') {
     timestamps {
       ansiColor('xterm') {
         githubPr.withDefaultPrComments {
           catchError {
+            retryable.enable()
             parallel([
-              'kibana-intake-agent': kibanaPipeline.legacyJobRunner('kibana-intake'),
-              'x-pack-intake-agent': kibanaPipeline.legacyJobRunner('x-pack-intake'),
+              'kibana-intake-agent': kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh'),
+              'x-pack-intake-agent': kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
               'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-                'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') }),
+                'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', {
+                  retryable('kibana-firefoxSmoke') {
+                    runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke')
+                  }
+                }),
                 'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
                 'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
                 'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
@@ -26,11 +31,19 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
                 'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
                 'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
                 'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
-                'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { runbld('./test/scripts/jenkins_accessibility.sh', 'Execute accessibility tests') }),
+                'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', {
+                  retryable('kibana-accessibility') {
+                    runbld('./test/scripts/jenkins_accessibility.sh', 'Execute accessibility tests')
+                  }
+                }),
                 // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') }),
               ]),
               'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-                'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') }),
+                'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', {
+                  retryable('xpack-firefoxSmoke') {
+                    runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke')
+                  }
+                }),
                 'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
                 'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
                 'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
@@ -41,12 +54,18 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
                 'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
                 'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
                 'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
-                'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility tests') }),
+                'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', {
+                  retryable('xpack-accessibility') {
+                    runbld('./test/scripts/jenkins_xpack_accessibility.sh', 'Execute xpack-accessibility tests')
+                  }
+                }),
                 // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') }),
               ]),
             ])
           }
         }
+
+        retryable.printFlakyFailures()
         kibanaPipeline.sendMail()
       }
     }

NOTICE.txt

@@ -153,6 +153,40 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+---
+This product bundles rules based on https://github.com/BlueTeamLabs/sentinel-attack
+which is available under a "MIT" license. The files based on this license are:
+
+- windows_defense_evasion_via_filter_manager.json
+- windows_process_discovery_via_tasklist_command.json
+- windows_priv_escalation_via_accessibility_features.json
+- windows_persistence_via_application_shimming.json
+- windows_execution_via_trusted_developer_utilities.json
+- windows_execution_via_net_com_assemblies.json
+- windows_execution_via_connection_manager.json
+
+MIT License
+
+Copyright (c) 2019 Edoardo Gerosa, Olaf Hartong
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ---
 This product includes code that is adapted from mapbox-gl-js, which is
 available under a "BSD-3-Clause" license.

api-documenter.json

@@ -0,0 +1,4 @@
+{
+  "newlineKind": "lf",
+  "outputTarget": "markdown"
+}

docs/api/saved-objects/import.asciidoc

@@ -49,7 +49,7 @@ Import an index pattern and dashboard:
 
 [source,js]
 --------------------------------------------------
-$ curl -X POST "localhost:5601/api/saved_objects/_import" -H "kbn-xsrf: true" --form file=@file.ndjson <1>
+$ curl -X POST "localhost:5601/api/saved_objects/_import" -H "kbn-xsrf: true" --form file=@file.ndjson
 --------------------------------------------------
 
 The `file.ndjson` file contains the following:

docs/developer/add-data-guide.asciidoc

@@ -11,20 +11,24 @@ Each tutorial contains three sets of instructions:
 
 [float]
 === Creating a new tutorial
-1. Create a new directory in the link:https://github.com/elastic/kibana/tree/master/src/legacy/core_plugins/kibana/server/tutorials[tutorials directory].
-2. In the new directory, create a file called `index.js` that exports a function.
-The function must return a JavaScript object that conforms to the link:https://github.com/elastic/kibana/blob/master/src/legacy/core_plugins/kibana/common/tutorials/tutorial_schema.js[tutorial schema].
-3. Register the tutorial in link:https://github.com/elastic/kibana/blob/master/src/legacy/core_plugins/kibana/server/tutorials/register.js[register.js] by calling `server.registerTutorial(myFuncImportedFromIndexJs)`.
+1. Create a new directory in the link:https://github.com/elastic/kibana/tree/master/src/plugins/home/server/tutorials[tutorials directory].
+2. In the new directory, create a file called `index.ts` that exports a function.
+The function must return a function object that conforms to the `TutorialSchema` interface link:https://github.com/elastic/kibana/blob/master/src/plugins/home/server/services/tutorials/lib/tutorial_schema.ts[tutorial schema].
+3. Register the tutorial in link:https://github.com/elastic/kibana/blob/master/src/plugins/home/server/tutorials/register.ts[register.ts] by adding it to the `builtInTutorials`.
+// TODO update path once assets are migrated
 4. Add image assets to the link:https://github.com/elastic/kibana/tree/master/src/legacy/core_plugins/kibana/public/home/tutorial_resources[tutorial_resources directory].
 5. Run Kibana locally to preview the tutorial.
 6. Create a PR and go through the review process to get the changes approved.
 
+If you are creating a new plugin and the tutorial is only related to that plugin, you can also place the `TutorialSchema` object into your plugin folder. Add `home` to the `requiredPlugins` list in your `kibana.json` file.
+Then register the tutorial object by calling `home.tutorials.registerTutorial(tutorialObject)` in the `setup` lifecycle of your server plugin.
+
 [float]
 ==== Variables
 String values can contain variables that are substituted when rendered. Variables are specified by `{}`.
 For example: `{config.docs.version}` is rendered as `6.2` when running the tutorial in Kibana 6.2.
 
-link:https://github.com/elastic/kibana/blob/master/src/legacy/core_plugins/kibana/public/home/components/tutorial/replace_template_strings.js#L23[Provided variables]
+link:https://github.com/elastic/kibana/blob/master/src/legacy/core_plugins/kibana/public/home/np_ready/components/tutorial/replace_template_strings.js#L23[Provided variables]
 
 [float]
 ==== Markdown