Merge branch 'master' into task-manager/schema-config

* master:
  Reorganize structure of component render to avoid errors. (elastic#54251)
  [DOCS] consolidate telemetry settings to core (elastic#52634)
  Add file upload to telemetry integration tests (elastic#53539)
  filters are back (elastic#54218)
  Add explanation of path.data to contribution guide (elastic#54235)
  fix ecommerce percentages in sample data (elastic#54200)

CONTRIBUTING.md

@@ -190,6 +190,19 @@ These snapshots are built on a nightly basis which expire after a couple weeks.
 yarn es snapshot
 ```
 
+##### Keeping data between snapshots
+
+If you want to keep the data inside your Elasticsearch between usages of this command,
+you should use the following command, to keep your data folder outside the downloaded snapshot 
+folder:
+
+```bash
+yarn es snapshot -E path.data=../data
+```
+
+The same parameter can be used with the source and archive command shown in the following
+paragraphs.
+
 #### Source
 
 By default, it will reference an [elasticsearch](https://github.com/elastic/elasticsearch) checkout which is a sibling to the Kibana directory named `elasticsearch`. If you wish to use a checkout in another location you can provide that by supplying `--source-path`

docs/settings/monitoring-settings.asciidoc

@@ -66,13 +66,6 @@ both the {es} monitoring cluster and the {es} production cluster.
 
 If not set, {kib} uses the value of the `elasticsearch.password` setting.
 
-`telemetry.enabled`::
-Set to `true` (default) to send cluster statistics to Elastic. Reporting your
-cluster statistics helps us improve your user experience. Your data is never
-shared with anyone. Set to `false` to disable statistics reporting from any
-browser connected to the {kib} instance. You can also opt out through the
-*Advanced Settings* in {kib}.
-
 `xpack.monitoring.elasticsearch.pingTimeout`::
 Specifies the time in milliseconds to wait for {es} to respond to internal
 health checks. By default, it matches the `elasticsearch.pingTimeout` setting,

docs/setup/settings.asciidoc

@@ -380,6 +380,11 @@ cannot be `false` at the same time.
  To enable telemetry and prevent users from disabling it,
  set `telemetry.allowChangingOptInStatus` to `false` and `telemetry.optIn` to `true`.
 
+`telemetry.enabled`:: *Default: true* Reporting your cluster statistics helps
+us improve your user experience. Your data is never shared with anyone. Set to
+`false` to disable telemetry capabilities entirely. You can alternatively opt
+out through the *Advanced Settings* in {kib}.
+
 `vega.enableExternalUrls:`:: *Default: false* Set this value to true to allow Vega to use any URL to access external data sources and images. If false, Vega can only get data from Elasticsearch.
 
 `xpack.license_management.enabled`:: *Default: true* Set this value to false to

x-pack/legacy/plugins/canvas/server/sample_data/ecommerce_saved_objects.json

@@ -194,7 +194,7 @@
                 "height": 56,
                 "angle": 0
               },
-              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" fields=\"customer_gender\" \n| ply by=\"customer_gender\" expression={rowCount | as \"percentage\"} \n| filterrows {getCell \"customer_gender\" | any {eq \"FEMALE\"}} \n| markdown {getCell \"percentage\"} \"%\" font={font family=\"Avenir\" size=48 align=\"center\" color=\"#eb6c66\" weight=\"normal\" underline=false italic=false}"
+              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" sort=\"order_date, desc\" fields=\"customer_gender\" | mapColumn \"femaleCount\" exp={getCell \"customer_gender\" | if {compare to=\"FEMALE\"} then=1 else=0} | math \"round(100 * sum(femaleCount) / count(femaleCount))\" | markdown {context} \"%\" font={font family=\"Avenir\" size=48 align=\"center\" color=\"#eb6c66\" weight=\"normal\" underline=false italic=false}"
             },
             {
               "id": "element-9e0b6230-2bc9-4995-8207-043e3063faeb",
@@ -205,7 +205,7 @@
                 "height": 56,
                 "angle": 0
               },
-              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" fields=\"customer_gender\" \n| ply by=\"customer_gender\" expression={rowCount | as \"percentage\"} \n| filterrows {getCell \"customer_gender\" | any {eq \"MALE\"}} \n| markdown {getCell \"percentage\"} \"%\" font={font family=\"Avenir\" size=48 align=\"center\" color=\"#f8bd4a\" weight=\"normal\" underline=false italic=false}"
+              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" sort=\"order_date, desc\" fields=\"customer_gender\" | mapColumn \"maleCount\" exp={getCell \"customer_gender\" | if {compare to=\"MALE\"} then=1 else=0} | math \"round(100 * sum(maleCount) / count(maleCount))\" | markdown {context} \"%\" font={font family=\"Avenir\" size=48 align=\"center\" color=\"#f8bd4a\" weight=\"normal\" underline=false italic=false}"
             },
             {
               "id": "element-2185edff-ac50-4162-b583-3bfd6469e925",
@@ -959,23 +959,23 @@
               "id": "element-b8f13a87-b781-42d5-a663-5dbd4f645d6d",
               "position": {
                 "left": 837,
-                "top": 472,
+                "top": 468,
                 "width": 91,
                 "height": 63,
                 "angle": 0
               },
-              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" fields=\"customer_gender\" \n| ply by=\"customer_gender\" expression={rowCount | as \"percentage\"} \n| filterrows {getCell \"customer_gender\" | any {eq \"FEMALE\"}} \n| markdown {getCell \"percentage\"} font={font family=\"nexa bold, Avenir\" size=48 align=\"center\" color=\"#F05A24\" weight=\"bold\" underline=false italic=false}"
+              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" sort=\"order_date, desc\" fields=\"customer_gender\" | mapColumn \"femaleCount\" exp={getCell \"customer_gender\" | if {compare to=\"FEMALE\"} then=1 else=0} | math \"round(100 * sum(femaleCount) / count(femaleCount))\" | markdown {context} font={font family=\"nexa bold, Avenir\" size=48 align=\"center\" color=\"#F05A24\" weight=\"bold\" underline=false italic=false}"
             },
             {
               "id": "element-86b06b67-893e-4555-ad38-7fba9ea3153b",
               "position": {
                 "left": 837,
-                "top": 219,
+                "top": 215,
                 "width": 90,
                 "height": 72,
                 "angle": 0
               },
-              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" fields=\"customer_gender\" \n| ply by=\"customer_gender\" expression={rowCount | as \"percentage\"} \n| filterrows {getCell \"customer_gender\" | any {eq \"MALE\"}} \n| markdown {getCell \"percentage\"} font={font family=\"nexa bold, Avenir\" size=48 align=\"center\" color=\"#00A89C\" weight=\"bold\" underline=false italic=false}"
+              "expression": "esdocs index=\"kibana_sample_data_ecommerce\" sort=\"order_date, desc\" fields=\"customer_gender\" | mapColumn \"maleCount\" exp={getCell \"customer_gender\" | if {compare to=\"MALE\"} then=1 else=0} | math \"round(100 * sum(maleCount) / count(maleCount))\" | markdown {context} font={font family=\"nexa bold, Avenir\" size=48 align=\"center\" color=\"#00A89C\" weight=\"bold\" underline=false italic=false}"
             },
             {
               "id": "element-507337d9-6e0e-4752-8770-6ebe88e9b3da",

x-pack/legacy/plugins/siem/public/pages/hosts/hosts.test.tsx

@@ -11,14 +11,18 @@ import { Router } from 'react-router-dom';
 import { MockedProvider } from 'react-apollo/test-utils';
 import { ActionCreator } from 'typescript-fsa';
 
+import { esFilters } from '../../../../../../../src/plugins/data/common/es_query';
 import '../../mock/match_media';
 import { mocksSource } from '../../containers/source/mock';
 import { wait } from '../../lib/helpers';
-import { TestProviders } from '../../mock';
+import { apolloClientObservable, TestProviders, mockGlobalState } from '../../mock';
 import { InputsModelId } from '../../store/inputs/constants';
 import { SiemNavigation } from '../../components/navigation';
+import { inputsActions } from '../../store/inputs';
+import { State, createStore } from '../../store';
 import { HostsComponentProps } from './types';
 import { Hosts } from './hosts';
+import { HostsTabs } from './hosts_tabs';
 
 // Test will fail because we will to need to mock some core services to make the test work
 // For now let's forget about SiemSearchBar and QueryBar
@@ -136,4 +140,58 @@ describe('Hosts - rendering', () => {
     wrapper.update();
     expect(wrapper.find(SiemNavigation).exists()).toBe(true);
   });
+
+  test('it should add the new filters after init', async () => {
+    const newFilters: esFilters.Filter[] = [
+      {
+        query: {
+          bool: {
+            filter: [
+              {
+                bool: {
+                  should: [
+                    {
+                      match_phrase: {
+                        'host.name': 'ItRocks',
+                      },
+                    },
+                  ],
+                  minimum_should_match: 1,
+                },
+              },
+            ],
+          },
+        },
+        meta: {
+          alias: '',
+          disabled: false,
+          key: 'bool',
+          negate: false,
+          type: 'custom',
+          value:
+            '{"query": {"bool": {"filter": [{"bool": {"should": [{"match_phrase": {"host.name": "ItRocks"}}],"minimum_should_match": 1}}]}}}',
+        },
+      },
+    ];
+    localSource[0].result.data.source.status.indicesExist = true;
+    const myState: State = mockGlobalState;
+    const myStore = createStore(myState, apolloClientObservable);
+    const wrapper = mount(
+      <TestProviders store={myStore}>
+        <MockedProvider mocks={localSource} addTypename={false}>
+          <Router history={mockHistory}>
+            <Hosts {...hostProps} />
+          </Router>
+        </MockedProvider>
+      </TestProviders>
+    );
+    await wait();
+    wrapper.update();
+
+    myStore.dispatch(inputsActions.setSearchBarFilter({ id: 'global', filters: newFilters }));
+    wrapper.update();
+    expect(wrapper.find(HostsTabs).props().filterQuery).toEqual(
+      '{"bool":{"must":[],"filter":[{"match_all":{}},{"bool":{"filter":[{"bool":{"should":[{"match_phrase":{"host.name":"ItRocks"}}],"minimum_should_match":1}}]}}],"should":[],"must_not":[]}}'
+    );
+  });
 });

x-pack/legacy/plugins/siem/public/pages/hosts/hosts.tsx

@@ -41,7 +41,7 @@ import { HostsTableType } from '../../store/hosts/model';
 
 const KpiHostsComponentManage = manageQuery(KpiHostsComponent);
 
-const HostsComponent = React.memo<HostsComponentProps>(
+export const HostsComponent = React.memo<HostsComponentProps>(
   ({
     deleteQuery,
     isInitializing,
@@ -56,13 +56,12 @@ const HostsComponent = React.memo<HostsComponentProps>(
     const capabilities = React.useContext(MlCapabilitiesContext);
     const kibana = useKibana();
     const { tabName } = useParams();
-
     const hostsFilters = React.useMemo(() => {
       if (tabName === HostsTableType.alerts) {
         return filters.length > 0 ? [...filters, ...filterAlertsHosts] : filterAlertsHosts;
       }
       return filters;
-    }, [tabName]);
+    }, [tabName, filters]);
     const narrowDateRange = useCallback(
       (min: number, max: number) => {
         setAbsoluteRangeDatePicker({ id: 'global', from: min, to: max });

x-pack/legacy/plugins/siem/public/pages/network/network.test.tsx

@@ -11,10 +11,13 @@ import { Router } from 'react-router-dom';
 import { MockedProvider } from 'react-apollo/test-utils';
 
 import '../../mock/match_media';
-
+import { esFilters } from '../../../../../../../src/plugins/data/common/es_query';
 import { mocksSource } from '../../containers/source/mock';
-import { TestProviders } from '../../mock';
+import { TestProviders, mockGlobalState, apolloClientObservable } from '../../mock';
+import { State, createStore } from '../../store';
+import { inputsActions } from '../../store/inputs';
 import { Network } from './network';
+import { NetworkRoutes } from './navigation';
 
 // Test will fail because we will to need to mock some core services to make the test work
 // For now let's forget about SiemSearchBar and QueryBar
@@ -111,4 +114,58 @@ describe('rendering - rendering', () => {
     wrapper.update();
     expect(wrapper.find('[data-test-subj="empty-page"]').exists()).toBe(false);
   });
+
+  test('it should add the new filters after init', async () => {
+    const newFilters: esFilters.Filter[] = [
+      {
+        query: {
+          bool: {
+            filter: [
+              {
+                bool: {
+                  should: [
+                    {
+                      match_phrase: {
+                        'host.name': 'ItRocks',
+                      },
+                    },
+                  ],
+                  minimum_should_match: 1,
+                },
+              },
+            ],
+          },
+        },
+        meta: {
+          alias: '',
+          disabled: false,
+          key: 'bool',
+          negate: false,
+          type: 'custom',
+          value:
+            '{"query": {"bool": {"filter": [{"bool": {"should": [{"match_phrase": {"host.name": "ItRocks"}}],"minimum_should_match": 1}}]}}}',
+        },
+      },
+    ];
+    localSource[0].result.data.source.status.indicesExist = true;
+    const myState: State = mockGlobalState;
+    const myStore = createStore(myState, apolloClientObservable);
+    const wrapper = mount(
+      <TestProviders store={myStore}>
+        <MockedProvider mocks={localSource} addTypename={false}>
+          <Router history={mockHistory}>
+            <Network {...getMockProps()} />
+          </Router>
+        </MockedProvider>
+      </TestProviders>
+    );
+    await new Promise(resolve => setTimeout(resolve));
+    wrapper.update();
+
+    myStore.dispatch(inputsActions.setSearchBarFilter({ id: 'global', filters: newFilters }));
+    wrapper.update();
+    expect(wrapper.find(NetworkRoutes).props().filterQuery).toEqual(
+      '{"bool":{"must":[],"filter":[{"match_all":{}},{"bool":{"filter":[{"bool":{"should":[{"match_phrase":{"host.name":"ItRocks"}}],"minimum_should_match":1}}]}}],"should":[],"must_not":[]}}'
+    );
+  });
 });

x-pack/legacy/plugins/siem/public/pages/network/network.tsx

@@ -59,7 +59,8 @@ const NetworkComponent = React.memo<NetworkComponentProps>(
         return filters.length > 0 ? [...filters, ...filterAlertsNetwork] : filterAlertsNetwork;
       }
       return filters;
-    }, [tabName]);
+    }, [tabName, filters]);
+
     const narrowDateRange = useCallback(
       (min: number, max: number) => {
         setAbsoluteRangeDatePicker({ id: 'global', from: min, to: max });

x-pack/legacy/plugins/uptime/public/components/functional/charts/duration_chart.tsx

@@ -66,32 +66,32 @@ export const DurationChart = ({
           </h4>
         </EuiTitle>
         <ChartWrapper height="400px" loading={loading}>
-          <Chart>
-            <Settings xDomain={{ min, max }} showLegend={true} legendPosition={Position.Bottom} />
-            <Axis
-              id="bottom"
-              position={Position.Bottom}
-              showOverlappingTicks={true}
-              tickFormat={timeFormatter(getChartDateLabel(min, max))}
-              title={i18n.translate('xpack.uptime.monitorCharts.durationChart.bottomAxis.title', {
-                defaultMessage: 'Timestamp',
-              })}
-            />
-            <Axis
-              domain={{ min: 0 }}
-              id="left"
-              position={Position.Left}
-              tickFormat={d => getTickFormat(d)}
-              title={i18n.translate('xpack.uptime.monitorCharts.durationChart.leftAxis.title', {
-                defaultMessage: 'Duration ms',
-              })}
-            />
-            {hasLines ? (
+          {hasLines ? (
+            <Chart>
+              <Settings xDomain={{ min, max }} showLegend={true} legendPosition={Position.Bottom} />
+              <Axis
+                id="bottom"
+                position={Position.Bottom}
+                showOverlappingTicks={true}
+                tickFormat={timeFormatter(getChartDateLabel(min, max))}
+                title={i18n.translate('xpack.uptime.monitorCharts.durationChart.bottomAxis.title', {
+                  defaultMessage: 'Timestamp',
+                })}
+              />
+              <Axis
+                domain={{ min: 0 }}
+                id="left"
+                position={Position.Left}
+                tickFormat={d => getTickFormat(d)}
+                title={i18n.translate('xpack.uptime.monitorCharts.durationChart.leftAxis.title', {
+                  defaultMessage: 'Duration ms',
+                })}
+              />
               <DurationLineSeriesList lines={locationDurationLines} meanColor={meanColor} />
-            ) : (
-              <DurationChartEmptyState />
-            )}
-          </Chart>
+            </Chart>
+          ) : (
+            <DurationChartEmptyState />
+          )}
         </ChartWrapper>
       </EuiPanel>
     </>

x-pack/test/api_integration/apis/telemetry/telemetry_local.js

@@ -86,6 +86,9 @@ export default function({ getService }) {
       expect(stats.stack_stats.kibana.plugins.reporting.enabled).to.be(true);
       expect(stats.stack_stats.kibana.plugins.rollups.index_patterns).to.be.an('object');
       expect(stats.stack_stats.kibana.plugins.spaces.available).to.be(true);
+      expect(stats.stack_stats.kibana.plugins.fileUploadTelemetry.filesUploadedTotalCount).to.be.a(
+        'number'
+      );
 
       expect(stats.stack_stats.kibana.os.platforms[0].platform).to.be.a('string');
       expect(stats.stack_stats.kibana.os.platforms[0].count).to.be(1);