Skip to content

Commit

Permalink
[1.x] Consolidate field-details doc template (elastic#897) (elastic#946)
Browse files Browse the repository at this point in the history
  • Loading branch information
ebeahan committed Aug 25, 2020
1 parent f2eb3a4 commit 7ab3055
Show file tree
Hide file tree
Showing 13 changed files with 328 additions and 251 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.next.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ Thanks, you're awesome :-) -->

#### Improvements

* Field details Jinja2 template components have been consolidated into one template #897

#### Deprecated


Expand Down
2 changes: 1 addition & 1 deletion docs/field-details.asciidoc
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

[[ecs-base]]
=== Base Fields

Expand Down Expand Up @@ -7085,3 +7084,4 @@ Note also that the `x509` fields are not expected to be used directly at the roo




270 changes: 80 additions & 190 deletions scripts/generators/asciidoc_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,6 @@

from generators import ecs_helpers

# jinja2 setup
TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates')
template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR)
template_env = jinja2.Environment(loader=template_loader)


def generate(nested, ecs_version, out_dir):
save_asciidoc(path.join(out_dir, 'fields.asciidoc'), page_field_index(nested, ecs_version))
Expand All @@ -19,6 +14,64 @@ def generate(nested, ecs_version, out_dir):
# Helpers


def render_fieldset_reuse_text(fieldset):
"""Renders the expected nesting locations
if the the `reusable` object is present.
:param fieldset: The fieldset to evaluate
"""
if not fieldset.get('reusable'):
return None
reusable_fields = fieldset['reusable']['expected']
sorted_fields = sorted(reusable_fields, key=lambda k: k['full'])
return map(lambda f: f['full'], sorted_fields)


def render_nestings_reuse_section(fieldset):
"""Renders the reuse section entries.
:param fieldset: The target fieldset
"""
if not fieldset.get('reused_here'):
return None
rows = []
for reused_here_entry in fieldset['reused_here']:
rows.append({
'flat_nesting': "{}.*".format(reused_here_entry['full']),
'name': reused_here_entry['schema_name'],
'short': reused_here_entry['short']
})

return sorted(rows, key=lambda x: x['flat_nesting'])


def extract_allowed_values_key_names(field):
"""Extracts the `name` keys from the field's
allowed_values if present in the field
object.
:param field: The target field
"""
if not field.get('allowed_values'):
return []
return ecs_helpers.list_extract_keys(field['allowed_values'], 'name')


def sort_fields(fieldset):
"""Prepares a fieldset's fields for being
passed into the j2 template for rendering. This
includes sorting them into a list of objects and
adding a field for the names of any allowed values
for the field, if present.
:param fieldset: The target fieldset
"""
fields_list = list(fieldset['fields'].values())
for field in fields_list:
field['allowed_value_names'] = extract_allowed_values_key_names(field)
return sorted(fields_list, key=lambda field: field['name'])


def templated(template_name):
"""Decorator function to simplify rendering a template.
Expand Down Expand Up @@ -53,210 +106,47 @@ def save_asciidoc(f, text):
with open(f, "w") as outfile:
outfile.write(text)

# jinja2 setup


TEMPLATE_DIR = path.join(path.dirname(path.abspath(__file__)), '../templates')
template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR)
template_env = jinja2.Environment(loader=template_loader)

# Rendering schemas

# Field Index

@templated('fields_template.j2')

@templated('fields.j2')
def page_field_index(nested, ecs_version):
fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name'])
return dict(ecs_version=ecs_version, fieldsets=fieldsets)


# Field Details Page


def page_field_details(nested):
page_text = ''
for fieldset in ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name']):
page_text += render_fieldset(fieldset, nested)
return page_text


def render_fieldset(fieldset, nested):
text = field_details_table_header(
title=fieldset['title'],
name=fieldset['name'],
description=fieldset['description']
)

text += render_fields(fieldset['fields'])

text += table_footer()

text += render_fieldset_reuse_section(fieldset, nested)

return text


def render_fields(fields):
text = ''
for _, field in sorted(fields.items()):
# Skip fields nested in this field set
if 'original_fieldset' not in field:
text += render_field_details_row(field)
return text


def render_field_allowed_values(field):
if not 'allowed_values' in field:
return ''
allowed_values = ', '.join(ecs_helpers.list_extract_keys(field['allowed_values'], 'name'))

return field_acceptable_value_names(
allowed_values=allowed_values,
flat_name=field['flat_name'],
dashed_name=field['dashed_name']
)


def render_field_details_row(field):
example = ''
if 'allowed_values' in field:
example = render_field_allowed_values(field)
elif 'example' in field:
example = "example: `{}`".format(str(field['example']))

field_type_with_mf = field['type']
if 'multi_fields' in field:
field_type_with_mf += "\n\nMulti-fields:\n\n"
for mf in field['multi_fields']:
field_type_with_mf += "* {} (type: {})\n\n".format(mf['flat_name'], mf['type'])

field_normalization = ''
if 'array' in field['normalize']:
field_normalization = "\nNote: this field should contain an array of values.\n\n"

text = field_details_row(
flat_name=field['flat_name'],
description=field['description'],
field_type=field_type_with_mf,
example=example,
normalization=field_normalization,
level=field['level']
)

return text


def render_fieldset_reuse_section(fieldset, nested):
'''Render the section on where field set can be nested, and which field sets can be nested here'''
if not ('nestings' in fieldset or 'reusable' in fieldset):
return ''

text = field_reuse_section(
reuse_of_fieldset=render_fieldset_reuses_text(fieldset)
)

if 'nestings' in fieldset:
text += nestings_table_header(
name=fieldset['name'],
title=fieldset['title']
)
rows = []
for reused_here_entry in fieldset['reused_here']:
rows.append({
'flat_nesting': "{}.*".format(reused_here_entry['full']),
'name': reused_here_entry['schema_name'],
'short': reused_here_entry['short']
})

for row in sorted(rows, key=lambda x: x['flat_nesting']):
text += nestings_row(
nesting_name=row['name'],
flat_nesting=row['flat_nesting'],
nesting_short=row['short']
)

text += table_footer()
return text


def render_fieldset_reuses_text(fieldset):
'''Render where a given field set is expected to be reused'''
if 'reusable' not in fieldset:
return ''

section_name = fieldset['name']
sorted_fields = sorted(fieldset['reusable']['expected'], key=lambda k: k['full'])
rendered_fields = map(lambda f: "`{}`".format(f['full']), sorted_fields)
text = "The `{}` fields are expected to be nested at: {}.\n\n".format(
section_name, ', '.join(rendered_fields))

if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']:
template = "Note also that the `{}` fields may be used directly at the root of the events.\n\n"
else:
template = "Note also that the `{}` fields are not expected to " + \
"be used directly at the root of the events.\n\n"
text += template.format(section_name)
return text


# Templates

def table_footer():
return '''
|=====
'''

# Field Details Page

# Main Fields Table


@templated('field_details/table_header.j2')
def field_details_table_header(title, name, description):
return dict(name=name, title=title, description=description)


@templated('field_details/row.j2')
def field_details_row(flat_name, description, field_type, normalization, example, level):
return dict(
flat_name=flat_name,
description=description,
field_type=field_type,
normalization=normalization,
example=example,
level=level
)


@templated('field_details/acceptable_value_names.j2')
def field_acceptable_value_names(allowed_values, dashed_name, flat_name):
return dict(
allowed_values=allowed_values,
dashed_name=dashed_name,
flat_name=flat_name
)


# Field reuse

@templated('field_details/field_reuse_section.j2')
def field_reuse_section(reuse_of_fieldset):
return dict(reuse_of_fieldset=reuse_of_fieldset)


# Nestings table

@templated('field_details/nestings_table_header.j2')
def nestings_table_header(name, title):
return dict(name=name, title=title)
fieldsets = ecs_helpers.dict_sorted_by_keys(nested, ['group', 'name'])
results = (generate_field_details_page(fieldset) for fieldset in fieldsets)
return ''.join(results)


@templated('field_details/nestings_row.j2')
def nestings_row(nesting_name, flat_nesting, nesting_short):
return dict(
nesting_name=nesting_name,
flat_nesting=flat_nesting,
nesting_short=nesting_short
)
@templated('field_details.j2')
def generate_field_details_page(fieldset):
# render field reuse text section
sorted_reuse_fields = render_fieldset_reuse_text(fieldset)
render_nestings_reuse_fields = render_nestings_reuse_section(fieldset)
sorted_fields = sort_fields(fieldset)
return dict(fieldset=fieldset,
sorted_reuse_fields=sorted_reuse_fields,
render_nestings_reuse_section=render_nestings_reuse_fields,
sorted_fields=sorted_fields)


# Allowed values section

@templated('field_values_template.j2')
@templated('field_values.j2')
def page_field_values(nested, template_name='field_values_template.j2'):
category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome']
nested_fields = []
Expand Down
Loading

0 comments on commit 7ab3055

Please sign in to comment.