libbeat: Don't force an ignore_above limit on wildcard fields (#30668) (

#30708)

Modifies libbeat's template processor to stop hardcoding a default
`ignore_above` limit of 1024 on wildcard fields. This behavior was
inherited from keyword fields.

Closes #30096

(cherry picked from commit 677229f)

Co-authored-by: Adrian Serrano <adrisr83@gmail.com>

CHANGELOG-developer.next.asciidoc

@@ -59,6 +59,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Removed Beat generators. {pull}28816[28816]
 - libbeat.logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 - Removed deprecated disk spool from Beats. Use disk queue instead. {pull}28869[28869]
+- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -37,6 +37,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 *Affecting all Beats*
 
 - Log errors when parsing and applying config blocks and if the input is disabled. {pull}30534[30534]
+- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668]
 
 *Auditbeat*
 

libbeat/template/processor.go

@@ -317,11 +317,11 @@ func (p *Processor) wildcard(f *mapping.Field, analyzers common.MapStr) common.M
 
 	property["type"] = "wildcard"
 
-	switch f.IgnoreAbove {
-	case 0: // Use libbeat default
-		property["ignore_above"] = defaultIgnoreAbove
-	case -1: // Use ES default
-	default: // Use user value
+	/* For wildcard fields, unlike keywords, don't force a default ignore_above limit.
+	   The default in ES will be used unless an explicit limit is set.
+	   This is to take advantage of wildcard type benefits when indexing large strings.
+	*/
+	if f.IgnoreAbove > 0 {
 		property["ignore_above"] = f.IgnoreAbove
 	}
 

libbeat/template/processor_test.go

@@ -805,46 +805,77 @@ func TestProcessWildcardOSS(t *testing.T) {
 }
 
 func TestProcessWildcardElastic(t *testing.T) {
-	// Test common fields are combined even if they come from different objects
-	fields := mapping.Fields{
-		mapping.Field{
-			Name: "test",
-			Type: "group",
-			Fields: mapping.Fields{
+	for _, test := range []struct {
+		title    string
+		fields   mapping.Fields
+		expected common.MapStr
+	}{
+		{
+			title: "default",
+			fields: mapping.Fields{
 				mapping.Field{
-					Name: "one",
-					Type: "wildcard",
+					Name: "test",
+					Type: "group",
+					Fields: mapping.Fields{
+						mapping.Field{
+							Name: "one",
+							Type: "wildcard",
+						},
+					},
+				},
+			},
+			expected: common.MapStr{
+				"test": common.MapStr{
+					"properties": common.MapStr{
+						"one": common.MapStr{
+							"type": "wildcard",
+						},
+					},
 				},
 			},
 		},
-	}
-
-	output := common.MapStr{}
-	analyzers := common.MapStr{}
-	version, err := common.NewVersion("8.0.0")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	p := Processor{EsVersion: *version, ElasticLicensed: true}
-	err = p.Process(fields, nil, output, analyzers)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Make sure fields without a name are skipped during template generation
-	expectedOutput := common.MapStr{
-		"test": common.MapStr{
-			"properties": common.MapStr{
-				"one": common.MapStr{
-					"ignore_above": 1024,
-					"type":         "wildcard",
+		{
+			title: "explicit ignore_above",
+			fields: mapping.Fields{
+				mapping.Field{
+					Name: "test",
+					Type: "group",
+					Fields: mapping.Fields{
+						mapping.Field{
+							Name:        "one",
+							Type:        "wildcard",
+							IgnoreAbove: 4096,
+						},
+					},
+				},
+			},
+			expected: common.MapStr{
+				"test": common.MapStr{
+					"properties": common.MapStr{
+						"one": common.MapStr{
+							"ignore_above": 4096,
+							"type":         "wildcard",
+						},
+					},
 				},
 			},
 		},
+	} {
+		t.Run(test.title, func(t *testing.T) {
+			output := common.MapStr{}
+			analyzers := common.MapStr{}
+			version, err := common.NewVersion("8.0.0")
+			if err != nil {
+				t.Fatal(err)
+			}
+			p := Processor{EsVersion: *version, ElasticLicensed: true}
+			err = p.Process(test.fields, nil, output, analyzers)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.Equal(t, test.expected, output)
+		})
 	}
-
-	assert.Equal(t, expectedOutput, output)
 }
 
 func TestProcessWildcardPreSupport(t *testing.T) {