Skip to content

Commit

Permalink
Set strings to be keywords by default (#2688)
Browse files Browse the repository at this point in the history 
This adds a dynamic mapping to all our template files to set strings to be
keywords by default.

Previously we were using the default of `text` and only switching to `keyword`
when configured in `fields.yml`. This reverses the logic and sets mappings to
`text` only when requested in `fields.yml`.

The goal is to make upgrading the mapping template less painful. In the Beats
we have so far, unexpected fields are better of as keywords.
  • Loading branch information
@tsg @ruflin
tsg authored and ruflin committed Oct 5, 2016
1 parent cc989a4 commit 714d6eb
Show file tree
Hide file tree
Showing 10 changed files with 46 additions and 156 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ https://github.com/elastic/beats/compare/v5.0.0-beta1...master[Check the HEAD di

*Affecting all Beats*

- A dynamic mapping rule is added to the default Elasticsearch template to treat strings as keywords by default. {pull}2688[2688]

*Metricbeat*

*Packetbeat*
Expand Down
5 changes: 2 additions & 3 deletions filebeat/filebeat.template-es2x.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,13 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "fields.*"
"match_mapping_type": "string"
}
}
],
Expand Down
5 changes: 2 additions & 3 deletions filebeat/filebeat.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,12 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "fields.*"
"match_mapping_type": "string"
}
}
],
Expand Down
34 changes: 28 additions & 6 deletions libbeat/scripts/generate_template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,30 @@ def fields_to_es_template(args, input, output, index, version):

properties = {}
dynamic_templates = []

# Make strings keywords by default
if args.es2x:
dynamic_templates.append({
"strings_as_keyword": {
"mapping": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 1024
},
"match_mapping_type": "string",
}
})
else:
dynamic_templates.append({
"strings_as_keyword": {
"mapping": {
"type": "keyword",
"ignore_above": 1024
},
"match_mapping_type": "string",
}
})

for section in docs["fields"]:
prop, dynamic = fill_section_properties(args, section,
defaults, "")
Expand Down Expand Up @@ -200,9 +224,9 @@ def fill_field_properties(args, field, defaults, path):
field.get("scaling_factor", 1000)

elif field["type"] in ["dict", "list"]:
if field.get("dict-type") == "keyword":
if field.get("dict-type") == "text":
# add a dynamic template to set all members of
# the dict as keywords
# the dict as text
if len(path) > 0:
name = path + "." + field["name"]
else:
Expand All @@ -213,8 +237,7 @@ def fill_field_properties(args, field, defaults, path):
name: {
"mapping": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 1024
"index": "analyzed",
},
"match_mapping_type": "string",
"path_match": name + ".*"
Expand All @@ -224,8 +247,7 @@ def fill_field_properties(args, field, defaults, path):
dynamic_templates.append({
name: {
"mapping": {
"type": "keyword",
"ignore_above": 1024
"type": "text",
},
"match_mapping_type": "string",
"path_match": name + ".*"
Expand Down
5 changes: 2 additions & 3 deletions metricbeat/metricbeat.template-es2x.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,13 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "fields.*"
"match_mapping_type": "string"
}
}
],
Expand Down
5 changes: 2 additions & 3 deletions metricbeat/metricbeat.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,12 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "fields.*"
"match_mapping_type": "string"
}
}
],
Expand Down
49 changes: 2 additions & 47 deletions packetbeat/packetbeat.template-es2x.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,58 +11,13 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "fields.*"
}
},
{
"amqp.headers": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "amqp.headers.*"
}
},
{
"cassandra.response.supported": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "cassandra.response.supported.*"
}
},
{
"http.request.headers": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "http.request.headers.*"
}
},
{
"http.response.headers": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "http.response.headers.*"
"match_mapping_type": "string"
}
}
],
Expand Down
45 changes: 2 additions & 43 deletions packetbeat/packetbeat.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,53 +9,12 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "fields.*"
}
},
{
"amqp.headers": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "amqp.headers.*"
}
},
{
"cassandra.response.supported": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "cassandra.response.supported.*"
}
},
{
"http.request.headers": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "http.request.headers.*"
}
},
{
"http.response.headers": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "http.response.headers.*"
"match_mapping_type": "string"
}
}
],
Expand Down
27 changes: 2 additions & 25 deletions winlogbeat/winlogbeat.template-es2x.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,13 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "fields.*"
}
},
{
"event_data": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "event_data.*"
}
},
{
"user_data": {
"mapping": {
"ignore_above": 1024,
"index": "not_analyzed",
"type": "string"
},
"match_mapping_type": "string",
"path_match": "user_data.*"
"match_mapping_type": "string"
}
}
],
Expand Down
25 changes: 2 additions & 23 deletions winlogbeat/winlogbeat.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,12 @@
},
"dynamic_templates": [
{
"fields": {
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "fields.*"
}
},
{
"event_data": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "event_data.*"
}
},
{
"user_data": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string",
"path_match": "user_data.*"
"match_mapping_type": "string"
}
}
],
Expand Down

0 comments on commit 714d6eb

Please sign in to comment.