From db635f78f8795dc45f4534367fa2e3e0b75d1264 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 12 Nov 2020 14:50:59 -0600 Subject: [PATCH] [1.7] Remove index parameter for wildcard fields (#1115) (#1120) --- CHANGELOG.md | 2 +- experimental/generated/beats/fields.ecs.yml | 1 - experimental/generated/ecs/ecs_flat.yml | 1 - experimental/generated/ecs/ecs_nested.yml | 1 - schemas/README.md | 3 ++- scripts/schema/cleaner.py | 3 +++ scripts/tests/unit/test_schema_cleaner.py | 4 ++++ 7 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 956b2a75b5..8e77fe6b0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ All notable changes to this project will be documented in this file based on the * Added `threat.technique.subtechnique` to capture MITRE ATT&CK® subtechniques. #951 * Added `configuration` as an allowed `event.category`. #963 * Added a new directory with experimental artifacts, which includes all changes - from RFCs that have reached stage 2. #993, #1053 + from RFCs that have reached stage 2. #993, #1053, #1115 #### Improvements diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 0431f4a01b..49b6c13543 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1160,7 +1160,6 @@ norms: false default_field: false description: The stack trace of this error in plain text. - index: true - name: type level: extended type: wildcard diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 62e67b84b1..46b44e8250 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -1599,7 +1599,6 @@ error.stack_trace: dashed_name: error-stack-trace description: The stack trace of this error in plain text. flat_name: error.stack_trace - index: true level: extended multi_fields: - flat_name: error.stack_trace.text diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 6fb9a81f55..8865448383 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -1971,7 +1971,6 @@ error: dashed_name: error-stack-trace description: The stack trace of this error in plain text. flat_name: error.stack_trace - index: true level: extended multi_fields: - flat_name: error.stack_trace.text diff --git a/schemas/README.md b/schemas/README.md index c87be195a3..11de7d6771 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -129,7 +129,8 @@ Supported keys to describe fields Example values that are composite types (array, object) should be quoted to avoid YAML interpretation in ECS-generated artifacts and other downstream projects depending on the schema. - multi\_fields (optional): Specify additional ways to index the field. -- index (optional): If `False`, means field is not indexed (overrides type) +- index (optional): If `False`, means field is not indexed (overrides type). This parameter has no effect + on a `wildcard` field. - format: Field format that can be used in a Kibana index template. - normalize: Normalization steps that should be applied at ingestion time. Supported values: - array: the content of the field should be an array (even when there's only one value). diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py index 5f15d459fe..e6eb050993 100644 --- a/scripts/schema/cleaner.py +++ b/scripts/schema/cleaner.py @@ -144,6 +144,9 @@ def field_or_multi_field_datatype_defaults(field_details): field_details.setdefault('ignore_above', 1024) if field_details['type'] == 'text': field_details.setdefault('norms', False) + # wildcard needs the index param stripped + if field_details['type'] == 'wildcard': + field_details.pop('index', None) if 'index' in field_details and not field_details['index']: field_details.setdefault('doc_values', False) diff --git a/scripts/tests/unit/test_schema_cleaner.py b/scripts/tests/unit/test_schema_cleaner.py index 8298a32bb3..491766e46a 100644 --- a/scripts/tests/unit/test_schema_cleaner.py +++ b/scripts/tests/unit/test_schema_cleaner.py @@ -209,6 +209,10 @@ def test_field_defaults(self): cleaner.field_defaults({'field_details': field_details}) self.assertEqual(field_details['doc_values'], False) + field_details = {**field_min_details, **{'type': 'wildcard', 'index': True}} + cleaner.field_defaults({'field_details': field_details}) + self.assertNotIn('index', field_details) + def test_field_defaults_dont_override(self): field_details = { 'description': 'description',