libbeat: Don't force an ignore_above limit on wildcard fields (#30668)

Modifies libbeat's template processor to stop hardcoding a default
`ignore_above` limit of 1024 on wildcard fields. This behavior was
inherited from keyword fields.

Closes #30096

(cherry picked from commit 677229f)

# Conflicts:
#	libbeat/template/processor_test.go

CHANGELOG-developer.next.asciidoc

@@ -52,6 +52,11 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
+- Removed the `common.Float` type. {issue}28279[28279] {pull}28280[28280] {pull}28376[28376]
+- Removed Beat generators. {pull}28816[28816]
+- libbeat.logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
+- Removed deprecated disk spool from Beats. Use disk queue instead. {pull}28869[28869]
+- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -34,6 +34,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix the ability for subcommands to be ran properly from the beats containers. {pull}30452[30452]
 - Log errors when parsing and applying config blocks and if the input is disabled. {pull}30534[30534]
+- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668]
 
 *Auditbeat*
 

libbeat/template/processor.go

@@ -317,11 +317,11 @@ func (p *Processor) wildcard(f *mapping.Field) common.MapStr {
 
 	property["type"] = "wildcard"
 
-	switch f.IgnoreAbove {
-	case 0: // Use libbeat default
-		property["ignore_above"] = defaultIgnoreAbove
-	case -1: // Use ES default
-	default: // Use user value
+	/* For wildcard fields, unlike keywords, don't force a default ignore_above limit.
+	   The default in ES will be used unless an explicit limit is set.
+	   This is to take advantage of wildcard type benefits when indexing large strings.
+	*/
+	if f.IgnoreAbove > 0 {
 		property["ignore_above"] = f.IgnoreAbove
 	}
 

libbeat/template/processor_test.go

@@ -812,18 +812,36 @@ func TestProcessWildcardOSS(t *testing.T) {
 }
 
 func TestProcessWildcardElastic(t *testing.T) {
-	// Test common fields are combined even if they come from different objects
-	fields := mapping.Fields{
-		mapping.Field{
-			Name: "test",
-			Type: "group",
-			Fields: mapping.Fields{
+	for _, test := range []struct {
+		title    string
+		fields   mapping.Fields
+		expected common.MapStr
+	}{
+		{
+			title: "default",
+			fields: mapping.Fields{
 				mapping.Field{
-					Name: "one",
-					Type: "wildcard",
+					Name: "test",
+					Type: "group",
+					Fields: mapping.Fields{
+						mapping.Field{
+							Name: "one",
+							Type: "wildcard",
+						},
+					},
+				},
+			},
+			expected: common.MapStr{
+				"test": common.MapStr{
+					"properties": common.MapStr{
+						"one": common.MapStr{
+							"type": "wildcard",
+						},
+					},
 				},
 			},
 		},
+<<<<<<< HEAD
 	}
 
 	output := common.MapStr{}
@@ -845,12 +863,50 @@ func TestProcessWildcardElastic(t *testing.T) {
 				"one": common.MapStr{
 					"ignore_above": 1024,
 					"type":         "wildcard",
+=======
+		{
+			title: "explicit ignore_above",
+			fields: mapping.Fields{
+				mapping.Field{
+					Name: "test",
+					Type: "group",
+					Fields: mapping.Fields{
+						mapping.Field{
+							Name:        "one",
+							Type:        "wildcard",
+							IgnoreAbove: 4096,
+						},
+					},
 				},
 			},
-		},
+			expected: common.MapStr{
+				"test": common.MapStr{
+					"properties": common.MapStr{
+						"one": common.MapStr{
+							"ignore_above": 4096,
+							"type":         "wildcard",
+						},
+					},
+>>>>>>> 677229fa34 (libbeat: Don't force an ignore_above limit on wildcard fields (#30668))
+				},
+			},
+		},
+	} {
+		t.Run(test.title, func(t *testing.T) {
+			output := common.MapStr{}
+			analyzers := common.MapStr{}
+			version, err := common.NewVersion("8.0.0")
+			if err != nil {
+				t.Fatal(err)
+			}
+			p := Processor{EsVersion: *version, ElasticLicensed: true}
+			err = p.Process(test.fields, nil, output, analyzers)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.Equal(t, test.expected, output)
+		})
 	}
-
-	assert.Equal(t, expectedOutput, output)
 }
 
 func TestProcessWildcardPreSupport(t *testing.T) {