Merge branch 'master' of github.com:elastic/kibana into implement/esl…

…int-no-async-foreach
elastic · Sep 13, 2021 · 4c94b18 · 4c94b18
2 parents 73e42fb + daf9cd5
commit 4c94b18
Show file tree

Hide file tree

Showing 57 changed files with 725 additions and 371 deletions.
diff --git a/docs/setup/access.asciidoc b/docs/setup/access.asciidoc
@@ -37,4 +37,21 @@ image::images/kibana-status-page-7_14_0.png[Kibana server status page]
 
 For JSON-formatted server status details, use the `localhost:5601/api/status` API endpoint. 
 
+[float]
+[[not-ready]]
+=== {kib} not ready
+
+If you receive an error that the {kib} `server is not ready`, check the following:
+
+* The {es} connectivity: 
++
+[source,sh]
+----
+`curl -XGET elasticsearch_ip_or_hostname:9200/`
+----
+* The {kib} logs:
+** Linux, DEB or RPM package: `/var/log/kibana/kibana.log`
+** Linux, tar.gz package: `$KIBANA_HOME/log/kibana.log`
+** Windows: `$KIBANA_HOME\log\kibana.log`
+* The health status of `.kibana*` indices
 
diff --git a/docs/setup/docker.asciidoc b/docs/setup/docker.asciidoc
@@ -93,6 +93,16 @@ services:
       - ./kibana.yml:/usr/share/kibana/config/kibana.yml
 --------------------------------------------
 
+==== Persist the {kib} keystore
+
+By default, {kib] auto-generates a keystore file for secure settings at startup. To persist your {kibana-ref}/secure-settings.html[secure settings], use the `kibana-keystore` utility to bind-mount the parent directory of the keystore to the container. For example:
+
+[source,sh]
+----
+docker run -it --rm -v full_path_to/config:/usr/share/kibana/config -v full_path_to/data:/usr/share/kibana/data docker.elastic.co/kibana/kibana:7.14.0 bin/kibana-keystore create
+docker run -it --rm -v full_path_to/config:/usr/share/kibana/config -v full_path_to/data:/usr/share/kibana/data docker.elastic.co/kibana/kibana:7.14.0 bin/kibana-keystore add test_keystore_setting
+----
+
 [float]
 [[environment-variable-config]]
 ==== Environment variable configuration

diff --git a/src/core/public/_variables.scss b/src/core/public/_variables.scss
@@ -1,7 +1,7 @@
 @import '@elastic/eui/src/global_styling/variables/header';
 
 // height of the header banner
-$kbnHeaderBannerHeight: $euiSizeXL;
+$kbnHeaderBannerHeight: $euiSizeXL;  // This value is also declared in `/x-pack/plugins/canvas/common/lib/constants.ts`
 // total height of the header (when the banner is *not* present)
 $kbnHeaderOffset: $euiHeaderHeightCompensation * 2;
 // total height of the header when the banner is present

diff --git a/src/core/public/rendering/_base.scss b/src/core/public/rendering/_base.scss
@@ -49,6 +49,12 @@
 
   &.kbnBody--hasHeaderBanner {
     @include kbnAffordForHeader($kbnHeaderOffsetWithBanner);
+
+    // Prevents banners from covering full screen data grids
+    .euiDataGrid--fullScreen {
+      height: calc(100vh - #{$kbnHeaderBannerHeight});
+      top: $kbnHeaderBannerHeight;
+    }
   }
   &.kbnBody--chromeHidden {
     @include kbnAffordForHeader(0);

diff --git a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts
@@ -57,9 +57,10 @@ describe('flattenHit', () => {
       fields: {
         date: ['1'],
         zzz: ['z'],
+        _abc: ['a'],
       },
     });
-    const expectedOrder = ['date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
+    const expectedOrder = ['_abc', 'date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
     expect(Object.keys(response)).toEqual(expectedOrder);
     expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder);
   });

diff --git a/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts b/src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts
@@ -66,7 +66,6 @@ function decorateFlattenedWrapper(hit: Record<string, any>, metaFields: Record<s
 
     // unwrap computed fields
     _.forOwn(hit.fields, function (val, key: any) {
-      if (key[0] === '_' && !_.includes(metaFields, key)) return;
       // Flatten an array with 0 or 1 elements to a single value.
       if (Array.isArray(val) && val.length <= 1) {
         flattened[key] = val[0];

diff --git a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts
@@ -25,6 +25,7 @@ export interface FieldDescriptor {
   type: string;
   esTypes: string[];
   subType?: FieldSubType;
+  metadata_field?: boolean;
 }
 
 interface FieldSubType {

diff --git a/...gins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js b/...gins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js
@@ -69,6 +69,17 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
   });
 
   describe('response order', () => {
+    it('supports fields that start with an underscore', async () => {
+      const fields = ['_field_a', '_field_b'];
+
+      stubDeps({
+        fieldsFromFieldCaps: fields.map((name) => ({ name })),
+      });
+
+      const fieldNames = (await getFieldCapabilities()).map((field) => field.name);
+      expect(fieldNames).toEqual(fields);
+    });
+
     it('always returns fields in alphabetical order', async () => {
       const letters = 'ambcdfjopngihkel'.split('');
       const sortedLetters = sortBy(letters);

diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts
@@ -34,20 +34,21 @@ export async function getFieldCapabilities(
   const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name');
 
   const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName)
-    .filter((name) => !name.startsWith('_'))
+    // not all meta fields are provided, so remove and manually add
+    .filter((name) => !fieldsFromFieldCapsByName[name].metadata_field)
     .concat(metaFields)
-    .reduce<{ names: string[]; hash: Record<string, string> }>(
+    .reduce<{ names: string[]; map: Map<string, string> }>(
       (agg, value) => {
-        // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
-        if (agg.hash[value] != null) {
+        // This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data
+        if (agg.map.get(value) != null) {
           return agg;
         } else {
-          agg.hash[value] = value;
+          agg.map.set(value, value);
           agg.names.push(value);
           return agg;
         }
       },
-      { names: [], hash: {} }
+      { names: [], map: new Map<string, string>() }
     )
     .names.map<FieldDescriptor>((name) =>
       defaults({}, fieldsFromFieldCapsByName[name], {
@@ -56,6 +57,7 @@ export async function getFieldCapabilities(
         searchable: false,
         aggregatable: false,
         readFromDocValues: false,
+        metadata_field: metaFields.includes(name),
       })
     )
     .map(mergeOverrides);

diff --git a/...ins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js b/...ins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js
@@ -48,6 +48,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
               'searchable',
               'aggregatable',
               'readFromDocValues',
+              'metadata_field',
             ]);
           });
         }

diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts
@@ -116,6 +116,8 @@ export function readFieldCapsResponse(
             }),
             {}
           ),
+          // @ts-expect-error
+          metadata_field: capsByType[types[0]].metadata_field,
         };
         // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
         agg.array.push(field);
@@ -131,6 +133,8 @@ export function readFieldCapsResponse(
         searchable: isSearchable,
         aggregatable: isAggregatable,
         readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType),
+        // @ts-expect-error
+        metadata_field: capsByType[types[0]].metadata_field,
       };
       // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
       agg.array.push(field);

diff --git a/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js b/test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js
@@ -38,6 +38,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan01',
@@ -46,6 +47,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan02',
@@ -54,6 +56,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
             ],
           });
@@ -77,6 +80,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan02',
@@ -85,6 +89,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
             ],
           });
@@ -109,6 +114,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'Jan02',
@@ -117,20 +123,23 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'meta1',
                 type: 'string',
                 aggregatable: false,
                 searchable: false,
                 readFromDocValues: false,
+                metadata_field: true,
               },
               {
                 name: 'meta2',
                 type: 'string',
                 aggregatable: false,
                 searchable: false,
                 readFromDocValues: false,
+                metadata_field: true,
               },
             ],
           });

diff --git a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js
@@ -35,6 +35,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'number_conflict',
@@ -43,6 +44,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: true,
+                metadata_field: false,
               },
               {
                 name: 'string_conflict',
@@ -51,6 +53,7 @@ export default function ({ getService }) {
                 aggregatable: true,
                 searchable: true,
                 readFromDocValues: false,
+                metadata_field: false,
               },
               {
                 name: 'success',
@@ -63,6 +66,7 @@ export default function ({ getService }) {
                   boolean: ['logs-2017.01.02'],
                   keyword: ['logs-2017.01.01'],
                 },
+                metadata_field: false,
               },
             ],
           });

diff --git a/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js b/test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js
@@ -25,6 +25,7 @@ export default function ({ getService }) {
       aggregatable: true,
       name: 'bar',
       readFromDocValues: true,
+      metadata_field: false,
     },
     {
       type: 'string',
@@ -33,6 +34,7 @@ export default function ({ getService }) {
       aggregatable: false,
       name: 'baz',
       readFromDocValues: false,
+      metadata_field: false,
     },
     {
       type: 'string',
@@ -42,6 +44,7 @@ export default function ({ getService }) {
       name: 'baz.keyword',
       readFromDocValues: true,
       subType: { multi: { parent: 'baz' } },
+      metadata_field: false,
     },
     {
       type: 'number',
@@ -50,6 +53,7 @@ export default function ({ getService }) {
       aggregatable: true,
       name: 'foo',
       readFromDocValues: true,
+      metadata_field: false,
     },
     {
       aggregatable: true,
@@ -63,6 +67,7 @@ export default function ({ getService }) {
         },
       },
       type: 'string',
+      metadata_field: false,
     },
   ];
 
@@ -100,6 +105,7 @@ export default function ({ getService }) {
               readFromDocValues: false,
               searchable: true,
               type: 'string',
+              metadata_field: true,
             },
             {
               aggregatable: false,
@@ -108,6 +114,7 @@ export default function ({ getService }) {
               readFromDocValues: false,
               searchable: false,
               type: '_source',
+              metadata_field: true,
             },
             {
               type: 'boolean',
@@ -116,6 +123,7 @@ export default function ({ getService }) {
               aggregatable: true,
               name: 'bar',
               readFromDocValues: true,
+              metadata_field: false,
             },
             {
               aggregatable: false,
@@ -124,6 +132,7 @@ export default function ({ getService }) {
               readFromDocValues: false,
               searchable: true,
               type: 'string',
+              metadata_field: false,
             },
             {
               type: 'string',
@@ -133,13 +142,15 @@ export default function ({ getService }) {
               name: 'baz.keyword',
               readFromDocValues: true,
               subType: { multi: { parent: 'baz' } },
+              metadata_field: false,
             },
             {
               aggregatable: false,
               name: 'crazy_meta_field',
               readFromDocValues: false,
               searchable: false,
               type: 'string',
+              metadata_field: true,
             },
             {
               type: 'number',
@@ -148,6 +159,7 @@ export default function ({ getService }) {
               aggregatable: true,
               name: 'foo',
               readFromDocValues: true,
+              metadata_field: false,
             },
             {
               aggregatable: true,
@@ -161,6 +173,7 @@ export default function ({ getService }) {
                 },
               },
               type: 'string',
+              metadata_field: false,
             },
           ],
         })