From 7ab30554b77447b58965b83b00dd4fd03e144878 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 20 Aug 2020 12:54:28 -0500 Subject: [PATCH] [1.x] Consolidate field-details doc template (#897) (#946) --- CHANGELOG.next.md | 2 + docs/field-details.asciidoc | 2 +- scripts/generators/asciidoc_fields.py | 270 ++++++------------ scripts/templates/field_details.j2 | 113 ++++++++ .../field_details/acceptable_value_names.j2 | 8 - .../field_details/field_reuse_section.j2 | 6 - .../templates/field_details/nestings_row.j2 | 7 - .../field_details/nestings_table_header.j2 | 11 - scripts/templates/field_details/row.j2 | 14 - .../templates/field_details/table_header.j2 | 14 - ...eld_values_template.j2 => field_values.j2} | 0 .../{fields_template.j2 => fields.j2} | 0 scripts/tests/test_asciidoc_fields.py | 132 +++++++++ 13 files changed, 328 insertions(+), 251 deletions(-) create mode 100644 scripts/templates/field_details.j2 delete mode 100644 scripts/templates/field_details/acceptable_value_names.j2 delete mode 100644 scripts/templates/field_details/field_reuse_section.j2 delete mode 100644 scripts/templates/field_details/nestings_row.j2 delete mode 100644 scripts/templates/field_details/nestings_table_header.j2 delete mode 100644 scripts/templates/field_details/row.j2 delete mode 100644 scripts/templates/field_details/table_header.j2 rename scripts/templates/{field_values_template.j2 => field_values.j2} (100%) rename scripts/templates/{fields_template.j2 => fields.j2} (100%) create mode 100644 scripts/tests/test_asciidoc_fields.py diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index ef52884095..85bbfa03ec 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -30,6 +30,8 @@ Thanks, you're awesome :-) --> #### Improvements +* Field details Jinja2 template components have been consolidated into one template #897 + #### Deprecated diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index c0c97dc14e..d6f3236892 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1,4 +1,3 @@ - [[ecs-base]] === Base Fields @@ -7085,3 +7084,4 @@ Note also that the `x509` fields are not expected to be used directly at the roo + diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index c25d7b8162..2aa6f4a8cd 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -5,11 +5,6 @@ from generators import ecs_helpers -# jinja2 setup -TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates') -template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) -template_env = jinja2.Environment(loader=template_loader) - def generate(nested, ecs_version, out_dir): save_asciidoc(path.join(out_dir, 'fields.asciidoc'), page_field_index(nested, ecs_version)) @@ -19,6 +14,64 @@ def generate(nested, ecs_version, out_dir): # Helpers +def render_fieldset_reuse_text(fieldset): + """Renders the expected nesting locations + if the the `reusable` object is present. + + :param fieldset: The fieldset to evaluate + """ + if not fieldset.get('reusable'): + return None + reusable_fields = fieldset['reusable']['expected'] + sorted_fields = sorted(reusable_fields, key=lambda k: k['full']) + return map(lambda f: f['full'], sorted_fields) + + +def render_nestings_reuse_section(fieldset): + """Renders the reuse section entries. + + :param fieldset: The target fieldset + """ + if not fieldset.get('reused_here'): + return None + rows = [] + for reused_here_entry in fieldset['reused_here']: + rows.append({ + 'flat_nesting': "{}.*".format(reused_here_entry['full']), + 'name': reused_here_entry['schema_name'], + 'short': reused_here_entry['short'] + }) + + return sorted(rows, key=lambda x: x['flat_nesting']) + + +def extract_allowed_values_key_names(field): + """Extracts the `name` keys from the field's + allowed_values if present in the field + object. + + :param field: The target field + """ + if not field.get('allowed_values'): + return [] + return ecs_helpers.list_extract_keys(field['allowed_values'], 'name') + + +def sort_fields(fieldset): + """Prepares a fieldset's fields for being + passed into the j2 template for rendering. This + includes sorting them into a list of objects and + adding a field for the names of any allowed values + for the field, if present. + + :param fieldset: The target fieldset + """ + fields_list = list(fieldset['fields'].values()) + for field in fields_list: + field['allowed_value_names'] = extract_allowed_values_key_names(field) + return sorted(fields_list, key=lambda field: field['name']) + + def templated(template_name): """Decorator function to simplify rendering a template. @@ -53,12 +106,19 @@ def save_asciidoc(f, text): with open(f, "w") as outfile: outfile.write(text) +# jinja2 setup + + +TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates') +template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) +template_env = jinja2.Environment(loader=template_loader) # Rendering schemas # Field Index -@templated('fields_template.j2') + +@templated('fields.j2') def page_field_index(nested, ecs_version): fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']) return dict(ecs_version=ecs_version, fieldsets=fieldsets) @@ -66,197 +126,27 @@ def page_field_index(nested, ecs_version): # Field Details Page - def page_field_details(nested): - page_text = '' - for fieldset in ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']): - page_text += render_fieldset(fieldset, nested) - return page_text - - -def render_fieldset(fieldset, nested): - text = field_details_table_header( - title=fieldset['title'], - name=fieldset['name'], - description=fieldset['description'] - ) - - text += render_fields(fieldset['fields']) - - text += table_footer() - - text += render_fieldset_reuse_section(fieldset, nested) - - return text - - -def render_fields(fields): - text = '' - for _, field in sorted(fields.items()): - # Skip fields nested in this field set - if 'original_fieldset' not in field: - text += render_field_details_row(field) - return text - - -def render_field_allowed_values(field): - if not 'allowed_values' in field: - return '' - allowed_values = ', '.join(ecs_helpers.list_extract_keys(field['allowed_values'], 'name')) - - return field_acceptable_value_names( - allowed_values=allowed_values, - flat_name=field['flat_name'], - dashed_name=field['dashed_name'] - ) - - -def render_field_details_row(field): - example = '' - if 'allowed_values' in field: - example = render_field_allowed_values(field) - elif 'example' in field: - example = "example: `{}`".format(str(field['example'])) - - field_type_with_mf = field['type'] - if 'multi_fields' in field: - field_type_with_mf += "\n\nMulti-fields:\n\n" - for mf in field['multi_fields']: - field_type_with_mf += "* {} (type: {})\n\n".format(mf['flat_name'], mf['type']) - - field_normalization = '' - if 'array' in field['normalize']: - field_normalization = "\nNote: this field should contain an array of values.\n\n" - - text = field_details_row( - flat_name=field['flat_name'], - description=field['description'], - field_type=field_type_with_mf, - example=example, - normalization=field_normalization, - level=field['level'] - ) - - return text - - -def render_fieldset_reuse_section(fieldset, nested): - '''Render the section on where field set can be nested, and which field sets can be nested here''' - if not ('nestings' in fieldset or 'reusable' in fieldset): - return '' - - text = field_reuse_section( - reuse_of_fieldset=render_fieldset_reuses_text(fieldset) - ) - - if 'nestings' in fieldset: - text += nestings_table_header( - name=fieldset['name'], - title=fieldset['title'] - ) - rows = [] - for reused_here_entry in fieldset['reused_here']: - rows.append({ - 'flat_nesting': "{}.*".format(reused_here_entry['full']), - 'name': reused_here_entry['schema_name'], - 'short': reused_here_entry['short'] - }) - - for row in sorted(rows, key=lambda x: x['flat_nesting']): - text += nestings_row( - nesting_name=row['name'], - flat_nesting=row['flat_nesting'], - nesting_short=row['short'] - ) - - text += table_footer() - return text - - -def render_fieldset_reuses_text(fieldset): - '''Render where a given field set is expected to be reused''' - if 'reusable' not in fieldset: - return '' - - section_name = fieldset['name'] - sorted_fields = sorted(fieldset['reusable']['expected'], key=lambda k: k['full']) - rendered_fields = map(lambda f: "`{}`".format(f['full']), sorted_fields) - text = "The `{}` fields are expected to be nested at: {}.\n\n".format( - section_name, ', '.join(rendered_fields)) - - if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: - template = "Note also that the `{}` fields may be used directly at the root of the events.\n\n" - else: - template = "Note also that the `{}` fields are not expected to " + \ - "be used directly at the root of the events.\n\n" - text += template.format(section_name) - return text - - -# Templates - -def table_footer(): - return ''' -|===== -''' - -# Field Details Page - -# Main Fields Table - - -@templated('field_details/table_header.j2') -def field_details_table_header(title, name, description): - return dict(name=name, title=title, description=description) - - -@templated('field_details/row.j2') -def field_details_row(flat_name, description, field_type, normalization, example, level): - return dict( - flat_name=flat_name, - description=description, - field_type=field_type, - normalization=normalization, - example=example, - level=level - ) - - -@templated('field_details/acceptable_value_names.j2') -def field_acceptable_value_names(allowed_values, dashed_name, flat_name): - return dict( - allowed_values=allowed_values, - dashed_name=dashed_name, - flat_name=flat_name - ) - - -# Field reuse - -@templated('field_details/field_reuse_section.j2') -def field_reuse_section(reuse_of_fieldset): - return dict(reuse_of_fieldset=reuse_of_fieldset) - - -# Nestings table - -@templated('field_details/nestings_table_header.j2') -def nestings_table_header(name, title): - return dict(name=name, title=title) + fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']) + results = (generate_field_details_page(fieldset) for fieldset in fieldsets) + return ''.join(results) -@templated('field_details/nestings_row.j2') -def nestings_row(nesting_name, flat_nesting, nesting_short): - return dict( - nesting_name=nesting_name, - flat_nesting=flat_nesting, - nesting_short=nesting_short - ) +@templated('field_details.j2') +def generate_field_details_page(fieldset): + # render field reuse text section + sorted_reuse_fields = render_fieldset_reuse_text(fieldset) + render_nestings_reuse_fields = render_nestings_reuse_section(fieldset) + sorted_fields = sort_fields(fieldset) + return dict(fieldset=fieldset, + sorted_reuse_fields=sorted_reuse_fields, + render_nestings_reuse_section=render_nestings_reuse_fields, + sorted_fields=sorted_fields) # Allowed values section -@templated('field_values_template.j2') +@templated('field_values.j2') def page_field_values(nested, template_name='field_values_template.j2'): category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome'] nested_fields = [] diff --git a/scripts/templates/field_details.j2 b/scripts/templates/field_details.j2 new file mode 100644 index 0000000000..0b1bb6e224 --- /dev/null +++ b/scripts/templates/field_details.j2 @@ -0,0 +1,113 @@ +{# Title & Description -#} +[[ecs-{{ fieldset['name'] }}]] +=== {{ fieldset['title'] }} Fields + +{{ fieldset['description']|replace("\n", "\n\n") }} + +{# Field Details Table Header -#} +==== {{ fieldset['title'] }} Field Details + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +{# Iterate through each field in the set -#} +{% for field in sorted_fields -%} +{% if 'original_fieldset' not in field -%} + +{# `Field` column -#} +| {{ field['flat_name'] }} +{# `Description` column -#} +| {{ field['description']|replace("\n", "\n\n") }} + +type: {{ field['type'] }} + +{% if 'multi_fields' in field -%} + +Multi-fields: + +{% for mf in field['multi_fields'] -%} + +* {{ mf['flat_name'] }} (type: {{ mf ['type'] }}) + + +{% endfor %}{# for mf #} +{% endif %}{# if 'multi_fields' #} +{% if 'array' in field['normalize'] -%} + +Note: this field should contain an array of values. + + +{% endif %} +{% if 'allowed_values' in field %} +*Important*: The field value must be one of the following: + +{{ field['allowed_value_names']|join(', ') }} + +To learn more about when to use which value, visit the page +<> +{% elif 'example' in field -%} + +example: `{{ field['example'] }}` + +{%- endif %}{# if 'allowed_values' elif 'example' #} + +{# `Level` column -#} +| {{ field['level'] }} + +// =============================================================== + +{% endif %}{# if 'original_fieldset' -#} +{% endfor %}{# for 'field' -#} + +|===== + +{# do we have `nestings` or `reusable` sections to worry about? -#} +{% if 'nestings' in fieldset or 'reusable' in fieldset -%} + +==== Field Reuse + +{% if 'reusable' in fieldset -%} + +The `{{ fieldset['name'] }}` fields are expected to be nested at: `{{ sorted_reuse_fields|join("`, `") }}`. + +{% if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level'] -%} + +Note also that the `{{ fieldset['name'] }}` fields may be used directly at the root of the events. + +{% else -%} + +Note also that the `{{ fieldset['name'] }}` fields are not expected to be used directly at the root of the events. + +{% endif %}{# if 'top_level' -#} +{% endif %}{# if 'reusable' #} + + +{% if 'nestings' in fieldset -%} + +[[ecs-{{ fieldset['name'] }}-nestings]] +===== Field sets that can be nested under {{ fieldset['title'] }} + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +{% for entry in render_nestings_reuse_section -%} + +| <> +| {{ entry['short'] }} + +// =============================================================== + + +{% endfor -%} + +|===== + +{% endif %}{# if 'nestings' #} +{%- endif %}{# if 'nestings' or 'reusable' in fieldset #} diff --git a/scripts/templates/field_details/acceptable_value_names.j2 b/scripts/templates/field_details/acceptable_value_names.j2 deleted file mode 100644 index 6080445742..0000000000 --- a/scripts/templates/field_details/acceptable_value_names.j2 +++ /dev/null @@ -1,8 +0,0 @@ - -*Important*: The field value must be one of the following: - -{{ allowed_values }} - -To learn more about when to use which value, visit the page -<> - diff --git a/scripts/templates/field_details/field_reuse_section.j2 b/scripts/templates/field_details/field_reuse_section.j2 deleted file mode 100644 index 37aa7ded45..0000000000 --- a/scripts/templates/field_details/field_reuse_section.j2 +++ /dev/null @@ -1,6 +0,0 @@ - -==== Field Reuse - -{{ reuse_of_fieldset }} - - diff --git a/scripts/templates/field_details/nestings_row.j2 b/scripts/templates/field_details/nestings_row.j2 deleted file mode 100644 index 826af848bb..0000000000 --- a/scripts/templates/field_details/nestings_row.j2 +++ /dev/null @@ -1,7 +0,0 @@ - -| <> -| {{ nesting_short }} - -// =============================================================== - - diff --git a/scripts/templates/field_details/nestings_table_header.j2 b/scripts/templates/field_details/nestings_table_header.j2 deleted file mode 100644 index 2ef25791d9..0000000000 --- a/scripts/templates/field_details/nestings_table_header.j2 +++ /dev/null @@ -1,11 +0,0 @@ - -[[ecs-{{ name }}-nestings]] -===== Field sets that can be nested under {{ title }} - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - diff --git a/scripts/templates/field_details/row.j2 b/scripts/templates/field_details/row.j2 deleted file mode 100644 index 90e2c8877f..0000000000 --- a/scripts/templates/field_details/row.j2 +++ /dev/null @@ -1,14 +0,0 @@ - -| {{ flat_name }} -| {{ description|replace("\n", "\n\n") }} - -type: {{ field_type }} - -{{ normalization }} - -{{ example }} - -| {{ level }} - -// =============================================================== - diff --git a/scripts/templates/field_details/table_header.j2 b/scripts/templates/field_details/table_header.j2 deleted file mode 100644 index 4496e8e768..0000000000 --- a/scripts/templates/field_details/table_header.j2 +++ /dev/null @@ -1,14 +0,0 @@ - -[[ecs-{{ name }}]] -=== {{ title }} Fields - -{{ description|replace("\n", "\n\n") }} - -==== {{ title }} Field Details - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - diff --git a/scripts/templates/field_values_template.j2 b/scripts/templates/field_values.j2 similarity index 100% rename from scripts/templates/field_values_template.j2 rename to scripts/templates/field_values.j2 diff --git a/scripts/templates/fields_template.j2 b/scripts/templates/fields.j2 similarity index 100% rename from scripts/templates/fields_template.j2 rename to scripts/templates/fields.j2 diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py new file mode 100644 index 0000000000..1a099a9958 --- /dev/null +++ b/scripts/tests/test_asciidoc_fields.py @@ -0,0 +1,132 @@ +import os +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + +from scripts.generators import asciidoc_fields +from scripts.generators import intermediate_files +from scripts.schema import cleaner +from scripts.schema import loader +from scripts.schema import finalizer + + +class TestGeneratorsAsciiFields(unittest.TestCase): + + def setUp(self): + self.foo_fieldset = self.dummy_fieldset() + + def dummy_fieldset(self): + return { + 'description': 'foo', + 'fields': { + 'foo.type': { + 'dashed_name': 'foo-type', + 'description': 'describes the foo', + 'example': '2016-05-23T08:05:34.853Z', + 'flat_name': 'foo.type', + 'level': 'core', + 'name': 'type', + 'normalize': ['array'], + 'short': 'describes the foo', + 'ignore_above': 1024, + 'type': 'keyword', + 'allowed_values': [{ + 'description': 'fluffy foo', + 'name': 'fluffy', + }, + { + 'description': 'coarse foo', + 'name': 'coarse', + } + ] + }, + 'foo.id': { + 'dashed_name': 'foo-id', + 'description': 'Unique ID of the foo.', + 'example': 'foo123', + 'flat_name': 'foo.id', + 'ignore_above': 1024, + 'level': 'core', + 'name': 'id', + 'normalize': [], + 'short': 'Unique ID of the foo.', + 'type': 'keyword' + } + }, + 'reusable': { + 'expected': [ + { + 'as': 'foo', + 'at': 'server', + 'full': 'server.foo' + }, + { + 'as': 'foo', + 'at': 'source', + 'full': 'source.foo' + }, + { + 'as': 'foo', + 'at': 'client', + 'full': 'client.foo', + }, + { + 'as': 'foo', + 'at': 'destination', + 'full': 'destination.foo' + } + ], + 'top_level': False, + }, + 'reused_here': [ + { + 'full': 'foo.as', + 'schema_name': 'as', + 'short': 'Fields describing an AS' + } + ], + 'group': 2, + 'name': 'foo', + 'prefix': 'foo.', + 'short': 'Foo fields', + 'title': 'Foo', + 'type': 'group' + } + + def test_validate_sort_fieldset(self): + sorted_foo_fields = asciidoc_fields.sort_fields(self.foo_fieldset) + #import pdb;pdb.set_trace() + self.assertIsInstance(sorted_foo_fields, list) + + # `allowed_value_names` always present + for field in sorted_foo_fields: + self.assertIsInstance(field.get('allowed_value_names'), list) + + self.assertFalse(sorted_foo_fields[0]['allowed_value_names']) + self.assertEqual('id', sorted_foo_fields[0]['name']) + self.assertEqual('type', sorted_foo_fields[1]['name']) + self.assertIn('fluffy', sorted_foo_fields[1]['allowed_value_names']) + self.assertIn('coarse', sorted_foo_fields[1]['allowed_value_names']) + + def test_rendering_fieldset_reuse(self): + foo_reuse_fields = asciidoc_fields.render_fieldset_reuse_text(self.foo_fieldset) + expected_sorted_reuse_fields = ( + 'client.foo', + 'destination.foo', + 'server.foo', + 'source.foo' + ) + + self.assertEqual(expected_sorted_reuse_fields, tuple(foo_reuse_fields)) + + def test_rendering_fieldset_nesting(self): + foo_nesting_fields = asciidoc_fields.render_nestings_reuse_section(self.foo_fieldset) + self.assertIsInstance(foo_nesting_fields, list) + self.assertEqual('foo.as.*', foo_nesting_fields[0]['flat_nesting']) + self.assertEqual('as', foo_nesting_fields[0]['name']) + self.assertEqual('Fields describing an AS', foo_nesting_fields[0]['short']) + + +if __name__ == '__main__': + unittest.main()