Add regex support functionality

CHANGELOG-developer.next.asciidoc

@@ -137,6 +137,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Added TESTING_FILEBEAT_FILEPATTERN option for filebeat module pytests {pull}30103[30103]
 - Add gcp dataproc metricset. {pull}30008[30008]
 - Add Github action for linting
+- Add regex support for drop_fields processor.
 
 ==== Deprecated
 

NOTICE.txt

@@ -6039,11 +6039,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.2.1
+Version: v0.2.3
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.3/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -160,7 +160,7 @@ require (
 )
 
 require (
-	github.com/elastic/elastic-agent-libs v0.2.1
+	github.com/elastic/elastic-agent-libs v0.2.3
 	github.com/shirou/gopsutil/v3 v3.21.12
 	go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0
 	go.elastic.co/apm/module/apmhttp/v2 v2.0.0

go.sum

@@ -569,8 +569,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
-github.com/elastic/elastic-agent-libs v0.2.1 h1:v6dxLeyC7jsyUcRXLPRqRHxqREx2wYWp9epxDu1VasM=
-github.com/elastic/elastic-agent-libs v0.2.1/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s=
+github.com/elastic/elastic-agent-libs v0.2.3 h1:GY8M0fxOs/GBY2nIB+JOB91aoD72S87iEcm2qVGFUqI=
+github.com/elastic/elastic-agent-libs v0.2.3/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng=
 github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4 h1:ViJxdtOsHeO+SWVekzM82fYHH1xnvZ8CvGPXZj+G4YI=

libbeat/processors/actions/drop_fields.go

@@ -20,10 +20,13 @@ package actions
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 
-	"github.com/pkg/errors"
+	"errors"
 	"go.uber.org/multierr"
 
+	"github.com/elastic/beats/v7/libbeat/common/match"
+
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/processors"
 	"github.com/elastic/beats/v7/libbeat/processors/checks"
@@ -33,6 +36,7 @@ import (
 
 type dropFields struct {
 	Fields        []string
+	RegexpFields  []match.Matcher
 	IgnoreMissing bool
 }
 
@@ -50,10 +54,11 @@ func newDropFields(c *conf.C) (processors.Processor, error) {
 	}{}
 	err := c.Unpack(&config)
 	if err != nil {
-		return nil, fmt.Errorf("fail to unpack the drop_fields configuration: %s", err)
+		return nil, fmt.Errorf("fail to unpack the drop_fields configuration: %w", err)
 	}
 
 	/* remove read only fields */
+	// TODO: Is this implementation used? If so, there's a fix needed in removal of exported fields
 	for _, readOnly := range processors.MandatoryExportedFields {
 		for i, field := range config.Fields {
 			if readOnly == field {
@@ -62,25 +67,54 @@ func newDropFields(c *conf.C) (processors.Processor, error) {
 		}
 	}
 
-	f := &dropFields{Fields: config.Fields, IgnoreMissing: config.IgnoreMissing}
+	// Parse regexp containing fields and removes them from initial config
+	regexpFields := make([]match.Matcher, 0)
+	for i := len(config.Fields) - 1; i >= 0; i-- {
+		field := config.Fields[i]
+		if strings.HasPrefix(field, "/") && strings.HasSuffix(field, "/") && len(field) > 2 {
+			config.Fields = append(config.Fields[:i], config.Fields[i+1:]...)
+
+			matcher, err := match.Compile(field[1 : len(field)-1])
+			if err != nil {
+				return nil, fmt.Errorf("wrong configuration in drop_fields. %w", err)
+			}
+
+			regexpFields = append(regexpFields, matcher)
+		}
+	}
+
+	f := &dropFields{Fields: config.Fields, IgnoreMissing: config.IgnoreMissing, RegexpFields: regexpFields}
 	return f, nil
 }
 
 func (f *dropFields) Run(event *beat.Event) (*beat.Event, error) {
 	var errs []error
 
+	// remove exact match fields
 	for _, field := range f.Fields {
-		if err := event.Delete(field); err != nil {
-			if f.IgnoreMissing && err == mapstr.ErrKeyNotFound {
-				continue
+		f.deleteField(event, field, &errs)
+	}
+
+	// remove fields contained in regexp expressions
+	for _, regex := range f.RegexpFields {
+		for _, field := range *event.Fields.FlattenKeys() {
+			if regex.MatchString(field) {
+				f.deleteField(event, field, &errs)
 			}
-			errs = append(errs, errors.Wrapf(err, "failed to drop field [%v]", field))
 		}
 	}
 
 	return event, multierr.Combine(errs...)
 }
 
+func (f *dropFields) deleteField(event *beat.Event, field string, errs *[]error) {
+	if err := event.Delete(field); err != nil {
+		if !f.IgnoreMissing || !errors.Is(err, mapstr.ErrKeyNotFound) {
+			*errs = append(*errs, fmt.Errorf("failed to drop field [%v], error: %w", field, err))
+		}
+	}
+}
+
 func (f *dropFields) String() string {
 	json, _ := json.Marshal(f)
 	return "drop_fields=" + string(json)

libbeat/processors/actions/drop_fields_test.go

@@ -20,6 +20,9 @@ package actions
 import (
 	"testing"
 
+	"github.com/elastic/beats/v7/libbeat/common/match"
+	config2 "github.com/elastic/elastic-agent-libs/config"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/beats/v7/libbeat/beat"
@@ -57,4 +60,62 @@ func TestDropFieldRun(t *testing.T) {
 		assert.Equal(t, mapstr.M{}, newEvent.Meta)
 		assert.Equal(t, event.Fields, newEvent.Fields)
 	})
+
+	t.Run("supports a regexp field", func(t *testing.T) {
+		event = &beat.Event{
+			Fields: mapstr.M{
+				"field_1": mapstr.M{
+					"subfield_1": "sf_1_value",
+					"subfield_2": mapstr.M{
+						"subfield_2_1": "sf_2_1_value",
+						"subfield_2_2": "sf_2_2_value",
+					},
+					"subfield_3": mapstr.M{
+						"subfield_3_1": "sf_3_1_value",
+						"subfield_3_2": "sf_3_2_value",
+					},
+				},
+				"field_2": "f_2_value",
+			},
+		}
+
+		p := dropFields{
+			RegexpFields: []match.Matcher{match.MustCompile("field_2$"), match.MustCompile("field_1\\.(.*)\\.subfield_2_1"), match.MustCompile("field_1\\.subfield_3(.*)")},
+			Fields:       []string{},
+		}
+
+		newEvent, err := p.Run(event)
+		assert.NoError(t, err)
+		assert.Equal(t, mapstr.M{
+			"field_1": mapstr.M{
+				"subfield_1": "sf_1_value",
+			},
+		}, newEvent.Fields)
+	})
+}
+
+func TestNewDropFields(t *testing.T) {
+	t.Run("detects regexp fields and assign to RegexpFields property", func(t *testing.T) {
+		c := config2.MustNewConfigFrom(map[string]interface{}{
+			"fields": []string{"/field_.*1/", "/second/", "third"},
+		})
+
+		procInt, err := newDropFields(c)
+		assert.NoError(t, err)
+
+		processor := procInt.(*dropFields)
+		assert.Equal(t, []string{"third"}, processor.Fields)
+		assert.Equal(t, "<substring 'second'>", processor.RegexpFields[0].String())
+		assert.Equal(t, "field_(?-s:.)*1", processor.RegexpFields[1].String())
+	})
+
+	t.Run("returns error when regexp field is badly written", func(t *testing.T) {
+		c := config2.MustNewConfigFrom(map[string]interface{}{
+			"fields": []string{"/[//"},
+		})
+
+		_, err := newDropFields(c)
+
+		assert.Equal(t, "wrong configuration in drop_fields. error parsing regexp: missing closing ]: `[/`", err.Error())
+	})
 }