From 775f13428966aca51fa608641c4b957bd0ae794c Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 1 Jul 2020 12:32:00 +0200 Subject: [PATCH 01/35] [Filebeat][httpjson] Add date_cursor to httpjson input (#19483) * Add date_cursor to httpjson input * Add changelog entry * Fix tests * Default to UTC date * Add date_cursor validations and better error message * Run fmt update --- CHANGELOG.next.asciidoc | 1 + go.mod | 2 +- x-pack/filebeat/input/httpjson/config.go | 53 ++++++++++++++ x-pack/filebeat/input/httpjson/config_test.go | 27 +++++++ .../filebeat/input/httpjson/httpjson_test.go | 48 ++++++++++++ x-pack/filebeat/input/httpjson/input.go | 73 ++++++++++++++++++- 6 files changed, 199 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6bcd74743929..b8f05a254c1a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -53,6 +53,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Okta module now requires objects instead of JSON strings for the `http_headers`, `http_request_body`, `pagination`, `rate_limit`, and `ssl` variables. {pull}18953[18953] - Adds oauth support for httpjson input. {issue}18415[18415] {pull}18892[18892] - Adds `split_events_by` option to httpjson input. {pull}19246[19246] +- Adds `date_cursor` option to httpjson input. {pull}19483[19483] *Heartbeat* diff --git a/go.mod b/go.mod index f4867e4d91ea..901d13a13044 100644 --- a/go.mod +++ b/go.mod @@ -163,7 +163,7 @@ require ( golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 - golang.org/x/tools v0.0.0-20200630154851-b2d8b0336632 + golang.org/x/tools v0.0.0-20200701041122-1837592efa10 google.golang.org/api v0.15.0 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb google.golang.org/grpc v1.29.1 diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index 2fcc2fc8941c..63d20221de47 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -7,6 +7,7 @@ package httpjson import ( "regexp" "strings" + "text/template" "time" "github.com/pkg/errors" @@ -35,6 +36,7 @@ type config struct { RetryWaitMax time.Duration `config:"retry.wait_max"` TLS *tlscommon.Config `config:"ssl"` URL string `config:"url" validate:"required"` + DateCursor *DateCursor `config:"date_cursor"` } // Pagination contains information about httpjson pagination settings @@ -65,6 +67,54 @@ type RateLimit struct { Remaining string `config:"remaining"` } +type DateCursor struct { + Enabled *bool `config:"enabled"` + Field string `config:"field" validate:"required"` + URLField string `config:"url_field" validate:"required"` + ValueTemplate *Template `config:"value_template"` + DateFormat string `config:"date_format"` + InitialInterval time.Duration `config:"initial_interval"` +} + +type Template struct { + *template.Template +} + +func (t *Template) Unpack(in string) error { + tpl, err := template.New("tpl").Parse(in) + if err != nil { + return err + } + + *t = Template{Template: tpl} + + return nil +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (dc *DateCursor) IsEnabled() bool { + return dc != nil && (dc.Enabled == nil || *dc.Enabled) +} + +// IsEnabled returns true if the `enable` field is set to true in the yaml. +func (dc *DateCursor) GetDateFormat() string { + if dc.DateFormat == "" { + return time.RFC3339 + } + return dc.DateFormat +} + +func (dc *DateCursor) Validate() error { + if dc.DateFormat == "" { + return nil + } + now := time.Now().Format(dc.DateFormat) + if _, err := time.Parse(dc.DateFormat, now); err != nil { + return errors.New("invalid configuration: date_format is not a valid date layout") + } + return nil +} + func (c *config) Validate() error { switch strings.ToUpper(c.HTTPMethod) { case "GET", "POST": @@ -81,6 +131,9 @@ func (c *config) Validate() error { } } if c.Pagination != nil { + if c.DateCursor.IsEnabled() { + return errors.Errorf("invalid configuration: date_cursor cannnot be set in combination with other pagination mechanisms") + } if c.Pagination.Header != nil { if c.Pagination.RequestField != "" || c.Pagination.IDField != "" || len(c.Pagination.ExtraBodyContent) > 0 { return errors.Errorf("invalid configuration: both pagination.header and pagination.req_field or pagination.id_field or pagination.extra_body_content cannot be set simultaneously") diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index cfec6a2440be..a86c2aa76db6 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -8,6 +8,7 @@ import ( "context" "os" "testing" + "time" "github.com/pkg/errors" "golang.org/x/oauth2/google" @@ -350,6 +351,32 @@ func TestConfigOauth2Validation(t *testing.T) { "url": "localhost", }, }, + { + name: "date_cursor must fail in combination with pagination", + expectedErr: "invalid configuration: date_cursor cannnot be set in combination with other pagination mechanisms accessing config", + input: map[string]interface{}{ + "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo"}, + "pagination": map[string]interface{}{ + "header": map[string]interface{}{"field_name": "foo", "regex_pattern": "bar"}, + }, + "url": "localhost", + }, + }, + { + name: "date_cursor.date_format will fail if invalid", + expectedErr: "invalid configuration: date_format is not a valid date layout accessing 'date_cursor'", + input: map[string]interface{}{ + "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo", "date_format": "1234"}, + "url": "localhost", + }, + }, + { + name: "date_cursor must work with a valid date_format", + input: map[string]interface{}{ + "date_cursor": map[string]interface{}{"field": "foo", "url_field": "foo", "date_format": time.RFC3339}, + "url": "localhost", + }, + }, } for _, c := range cases { diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/httpjson_test.go index 4e70fe72472f..75374404eeae 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/httpjson_test.go @@ -7,6 +7,7 @@ package httpjson import ( "context" "encoding/json" + "fmt" "io/ioutil" "log" "math/rand" @@ -727,3 +728,50 @@ func TestArrayWithSplitResponse(t *testing.T) { } }) } + +func TestCursor(t *testing.T) { + m := map[string]interface{}{ + "http_method": "GET", + "date_cursor.field": "@timestamp", + "date_cursor.url_field": "$filter", + "date_cursor.value_template": "alertCreationTime ge {{.}}", + "date_cursor.initial_interval": "10m", + "date_cursor.date_format": "2006-01-02T15:04:05Z", + } + + timeNow = func() time.Time { + t, _ := time.Parse("2006-01-02T15:04:05Z", "2002-10-02T15:10:00Z") + return t + } + + const ( + expectedQuery = "%24filter=alertCreationTime+ge+2002-10-02T15%3A00%3A00Z" + expectedNextCursorValue = "2002-10-02T15:00:01Z" + expectedNextQuery = "%24filter=alertCreationTime+ge+2002-10-02T15%3A00%3A01Z" + ) + var gotQuery string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + gotQuery = r.URL.Query().Encode() + w.Write([]byte(`[{"@timestamp":"2002-10-02T15:00:00Z"},{"@timestamp":"2002-10-02T15:00:01Z"}]`)) + })) + + runTest(t, ts, m, func(input *HttpjsonInput, out *stubOutleter, t *testing.T) { + group, _ := errgroup.WithContext(context.Background()) + group.Go(input.run) + + events, ok := out.waitForEvents(2) + if !ok { + t.Fatalf("Expected 2 events, but got %d.", len(events)) + } + input.Stop() + + if err := group.Wait(); err != nil { + t.Fatal(err) + } + + assert.Equal(t, expectedQuery, gotQuery) + assert.Equal(t, expectedNextCursorValue, input.nextCursorValue) + assert.Equal(t, fmt.Sprintf("%s?%s", ts.URL, expectedNextQuery), input.getURL()) + }) +} diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 78f5eba5344e..fe65f63ebf83 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "net" "net/http" + "net/url" "regexp" "strconv" "sync" @@ -37,6 +38,9 @@ const ( var userAgent = useragent.UserAgent("Filebeat") +// for testing +var timeNow = time.Now + func init() { err := input.Register(inputName, NewInput) if err != nil { @@ -55,6 +59,8 @@ type HttpjsonInput struct { workerCancel context.CancelFunc // Used to signal that the worker should stop. workerOnce sync.Once // Guarantees that the worker goroutine is only started once. workerWg sync.WaitGroup // Waits on worker goroutine. + + nextCursorValue string } // RequestInfo struct has the information for generating an HTTP request @@ -343,6 +349,7 @@ func createRequestInfoFromBody(m common.MapStr, idField string, requestField str // processHTTPRequest processes HTTP request, and handles pagination if enabled func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Client, ri *RequestInfo) error { + ri.URL = in.getURL() for { req, err := in.createHTTPRequest(ctx, ri) if err != nil { @@ -407,8 +414,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl in.log.Debug("http.response.body is not a valid JSON object", string(responseData)) return errors.Errorf("http.response.body is not a valid JSON object, but a %T", obj) } - - if mm != nil && in.config.Pagination != nil && in.config.Pagination.IsEnabled() { + if mm != nil && in.config.Pagination.IsEnabled() { if in.config.Pagination.Header != nil { // Pagination control using HTTP Header url, err := getNextLinkFromHeader(header, in.config.Pagination.Header.FieldName, in.config.Pagination.Header.RegexPattern) @@ -427,7 +433,7 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl continue } else { // Pagination control using HTTP Body fields - ri, err := createRequestInfoFromBody(common.MapStr(mm), in.config.Pagination.IDField, in.config.Pagination.RequestField, common.MapStr(in.config.Pagination.ExtraBodyContent), in.config.Pagination.URL, ri) + ri, err = createRequestInfoFromBody(common.MapStr(mm), in.config.Pagination.IDField, in.config.Pagination.RequestField, common.MapStr(in.config.Pagination.ExtraBodyContent), in.config.Pagination.URL, ri) if err != nil { return err } @@ -441,10 +447,70 @@ func (in *HttpjsonInput) processHTTPRequest(ctx context.Context, client *http.Cl continue } } + if mm != nil && in.config.DateCursor.IsEnabled() { + in.advanceCursor(common.MapStr(mm)) + } return nil } } +func (in *HttpjsonInput) getURL() string { + if !in.config.DateCursor.IsEnabled() { + return in.config.URL + } + + var dateStr string + if in.nextCursorValue == "" { + t := timeNow().UTC().Add(-in.config.DateCursor.InitialInterval) + dateStr = t.Format(in.config.DateCursor.GetDateFormat()) + } else { + dateStr = in.nextCursorValue + } + + url, err := url.Parse(in.config.URL) + if err != nil { + return in.config.URL + } + + q := url.Query() + + var value string + if in.config.DateCursor.ValueTemplate == nil { + value = dateStr + } else { + buf := new(bytes.Buffer) + if err := in.config.DateCursor.ValueTemplate.Execute(buf, dateStr); err != nil { + return in.config.URL + } + value = buf.String() + } + + q.Set(in.config.DateCursor.URLField, value) + + url.RawQuery = q.Encode() + + return url.String() +} + +func (in *HttpjsonInput) advanceCursor(m common.MapStr) { + v, err := m.GetValue(in.config.DateCursor.Field) + if err != nil { + in.log.Warnf("date_cursor field: %q", err) + return + } + switch t := v.(type) { + case string: + _, err := time.Parse(in.config.DateCursor.GetDateFormat(), t) + if err != nil { + return + } + in.nextCursorValue = t + default: + in.log.Warn("date_cursor field must be a string, cursor will not advance") + return + } +} + func (in *HttpjsonInput) run() error { ctx, cancel := context.WithCancel(in.workerCtx) defer cancel() @@ -455,7 +521,6 @@ func (in *HttpjsonInput) run() error { } ri := &RequestInfo{ - URL: in.URL, ContentMap: common.MapStr{}, Headers: in.HTTPHeaders, } From 9619361ee28a10c0240f194ff57829679f3f1ae0 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 1 Jul 2020 13:07:52 +0200 Subject: [PATCH 02/35] Use Go install for getting goimports (#19535) * go get off mode * use go install --- dev-tools/mage/fmt.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/mage/fmt.go b/dev-tools/mage/fmt.go index 02a8001895e2..b4463696b816 100644 --- a/dev-tools/mage/fmt.go +++ b/dev-tools/mage/fmt.go @@ -74,8 +74,8 @@ func GoImports() error { return err } } else { - if err := gotool.Get( - gotool.Get.Package(filepath.Join(GoImportsImportPath)), + if err := gotool.Install( + gotool.Install.Package(filepath.Join(GoImportsImportPath)), ); err != nil { return err } From c45ced3bd04f6fd448c88d629a1a34f116fb2dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 1 Jul 2020 15:03:59 +0200 Subject: [PATCH 03/35] Add support for Thread ID in Filebeat Kafka module (#19463) Closes #18164 --- CHANGELOG.asciidoc | 1 + filebeat/docs/fields.asciidoc | 10 +++++ filebeat/docs/modules/kafka.asciidoc | 2 + filebeat/module/kafka/_meta/docs.asciidoc | 2 + filebeat/module/kafka/fields.go | 2 +- filebeat/module/kafka/log/_meta/fields.yml | 4 ++ filebeat/module/kafka/log/ingest/pipeline.yml | 1 + .../kafka/log/test/state-change-2.2.2.log | 23 +++++++++++ .../test/state-change-2.2.2.log-expected.json | 39 +++++++++++++++++++ 9 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/kafka/log/test/state-change-2.2.2.log create mode 100644 filebeat/module/kafka/log/test/state-change-2.2.2.log-expected.json diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d2274bb0080c..d1ab5687dc9c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -156,6 +156,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit - Add Kibana Dashboard for MISP module. {pull}14147[14147] - Add support for gzipped files in S3 input {pull}13980[13980] - Add Filebeat Azure Dashboards {pull}14127[14127] +- Add support for thread ID in Filebeat Kafka module. {pull}19463[19463] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index a695efaab3e9..8562e3da0e2e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -24755,6 +24755,16 @@ type: keyword Java class the log is coming from. +type: keyword + +-- + +*`kafka.log.thread`*:: ++ +-- +Thread name the log is coming from. + + type: keyword -- diff --git a/filebeat/docs/modules/kafka.asciidoc b/filebeat/docs/modules/kafka.asciidoc index d9319b43b508..5e2145a348e2 100644 --- a/filebeat/docs/modules/kafka.asciidoc +++ b/filebeat/docs/modules/kafka.asciidoc @@ -11,6 +11,8 @@ This file is generated! See scripts/docs_collector.py The +{modulename}+ module collects and parses the logs created by https://kafka.apache.org/[Kafka]. +The module has additional support for parsing thread ID from logs. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/filebeat/module/kafka/_meta/docs.asciidoc b/filebeat/module/kafka/_meta/docs.asciidoc index 4e199f98b4be..4940b08d54e4 100644 --- a/filebeat/module/kafka/_meta/docs.asciidoc +++ b/filebeat/module/kafka/_meta/docs.asciidoc @@ -6,6 +6,8 @@ The +{modulename}+ module collects and parses the logs created by https://kafka.apache.org/[Kafka]. +The module has additional support for parsing thread ID from logs. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/filebeat/module/kafka/fields.go b/filebeat/module/kafka/fields.go index 86950968bf0b..6d0aadfd8673 100644 --- a/filebeat/module/kafka/fields.go +++ b/filebeat/module/kafka/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKafka returns asset data. // This is the base64 encoded gzipped contents of module/kafka. func AssetKafka() string { - return "eJykkjtuwzAQRHudYuDeOoCKNOkSpMsFFtZKJsQfyLUT3z6gGP8Y2paR7cQRZ55Gu8bEhw4TDRM1gCjR3GH1np5XDdBz3ATlRTnb4aUBgFmDcf1OcwMMinUfu1law5Lhs10aOXjuMAa3878nFc9rm0sr7cbTWc3spmGeDKvdCK0sx/ZCLBOvUnnP+ko5ZpNWFAvFk2xn0rZ2z6gxUGaTsONqnuEYaeQnE+u3buVVgzfOeGfZSjV64sOXC32h3ak7zevRErLluXoVU46yI4bgTFsH0RTLr/wHxBvtKXs+RSGBNvW/UO7cAgbgM9lB2RNDWsK2eLG2h4+aedTOIro/LUnGvdMTHu7smUz4u9yqhVjAR/aGpyBww5mubX4CAAD//40BKYM=" + return "eJykkztywyAQhnud4h/31gFUpEmXTDpfYMdayYwQMLB24ttngPhF8GtMZ9Z8/6eFXWLifYeJhokaQJRo7rD4jL8XDdBzWHvlRFnT4a0BgFTDbPut5gYYFOs+dKm0hKGZT7i4ZO+4w+jt1v3tVJiXmHOUtuNxrwa7Cswry2o7QivDoT0rlokXqbxjfVE5ZJNWFIqKI9kk07Z2blajp+wmfsvVvJlDoJGfTKyfupZXDV7b2VnDRqrRE++/re+L2o12x/V+QEI2nFqvQsxRZsTg7dzWRTSF8itfkPigHWXmUxay8Uxl1Asaq8RL8Oc8PK3rr6F8+w9IAKuIgzJHhzgMbfHH2jzgzg3da89Ddv9uS7LujT7h7uyczIR/ytf9oBbwldlw5AV2ONm1zW8AAAD//95/TD8=" } diff --git a/filebeat/module/kafka/log/_meta/fields.yml b/filebeat/module/kafka/log/_meta/fields.yml index c06f44d0bf49..32b9008f7483 100644 --- a/filebeat/module/kafka/log/_meta/fields.yml +++ b/filebeat/module/kafka/log/_meta/fields.yml @@ -20,6 +20,10 @@ type: keyword description: > Java class the log is coming from. + - name: thread + type: keyword + description: > + Thread name the log is coming from. - name: trace type: group description: > diff --git a/filebeat/module/kafka/log/ingest/pipeline.yml b/filebeat/module/kafka/log/ingest/pipeline.yml index 41db8f4f1973..a10724891225 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.yml +++ b/filebeat/module/kafka/log/ingest/pipeline.yml @@ -6,6 +6,7 @@ processors: patterns: - (?m)%{TIMESTAMP_ISO8601:kafka.log.timestamp}. %{LOGLEVEL:log.level} +%{JAVALOGMESSAGE:message} \(%{JAVACLASS:kafka.log.class}\)$[ \n]*(?'kafka.log.trace.full'.*) + - (?m)\[%{TIMESTAMP_ISO8601:kafka.log.timestamp}\] \[%{LOGLEVEL:log.level} ?\] \[%{NOTSPACE:kafka.log.thread}\] \[%{NOTSPACE:kafka.log.class}\] \- %{GREEDYDATA:message} - grok: field: message pattern_definitions: diff --git a/filebeat/module/kafka/log/test/state-change-2.2.2.log b/filebeat/module/kafka/log/test/state-change-2.2.2.log new file mode 100644 index 000000000000..69d64c90d22a --- /dev/null +++ b/filebeat/module/kafka/log/test/state-change-2.2.2.log @@ -0,0 +1,23 @@ +[2020-04-23 00:28:21,796] [WARN ] [kafka-request-handler-7] [state.change.logger] - [Broker id=1] Ignoring LeaderAndIsr request from controller 1 with correlation id 1 epoch 27 for partition xxx.xxx.inventory.test.1-0-2 since its associated leader epoch 5 is not higher than the current leader epoch 5 +[2020-01-20 01:32:00,705] [ERROR] [controller-event-thread] [state.change.logger] - [Controller id=1 epoch=25] Controller 1 epoch 25 failed to change state for partition xxx.xxx.na.och.aud.1-0-2 from OfflinePartition to OnlinePartition +kafka.common.StateChangeFailedException: Failed to elect leader for partition xxx.xxx.na.och.aud.1-0-2 under strategy OfflinePartitionLeaderElectionStrategy + at kafka.controller.PartitionStateMachine.$anonfun$doElectLeaderForPartitions$9(PartitionStateMachine.scala:390) + at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) + at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) + at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) + at kafka.controller.PartitionStateMachine.doElectLeaderForPartitions(PartitionStateMachine.scala:388) + at kafka.controller.PartitionStateMachine.electLeaderForPartitions(PartitionStateMachine.scala:315) + at kafka.controller.PartitionStateMachine.doHandleStateChanges(PartitionStateMachine.scala:225) + at kafka.controller.PartitionStateMachine.handleStateChanges(PartitionStateMachine.scala:141) + at kafka.controller.PartitionStateMachine.triggerOnlinePartitionStateChange(PartitionStateMachine.scala:123) + at kafka.controller.PartitionStateMachine.triggerOnlinePartitionStateChange(PartitionStateMachine.scala:109) + at kafka.controller.PartitionStateMachine.startup(PartitionStateMachine.scala:66) + at kafka.controller.KafkaController.onControllerFailover(KafkaController.scala:266) + at kafka.controller.KafkaController.kafka$controller$KafkaController$$elect(KafkaController.scala:1271) + at kafka.controller.KafkaController$Startup$.process(KafkaController.scala:1184) + at kafka.controller.ControllerEventManager$ControllerEventThread.$anonfun$doWork$1(ControllerEventManager.scala:94) + at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) + at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) + at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:94) + at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) + diff --git a/filebeat/module/kafka/log/test/state-change-2.2.2.log-expected.json b/filebeat/module/kafka/log/test/state-change-2.2.2.log-expected.json new file mode 100644 index 000000000000..a8ad3645ec70 --- /dev/null +++ b/filebeat/module/kafka/log/test/state-change-2.2.2.log-expected.json @@ -0,0 +1,39 @@ +[ + { + "@timestamp": "2020-04-23T00:28:21.796-02:00", + "event.dataset": "kafka.log", + "event.kind": "event", + "event.module": "kafka", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "kafka.log.class": "state.change.logger", + "kafka.log.component": "Broker id=1", + "kafka.log.thread": "kafka-request-handler-7", + "log.level": "WARN", + "log.offset": 0, + "message": "Ignoring LeaderAndIsr request from controller 1 with correlation id 1 epoch 27 for partition xxx.xxx.inventory.test.1-0-2 since its associated leader epoch 5 is not higher than the current leader epoch 5 ", + "service.type": "kafka" + }, + { + "@timestamp": "2020-01-20T01:32:00.705-02:00", + "event.dataset": "kafka.log", + "event.kind": "event", + "event.module": "kafka", + "event.timezone": "-02:00", + "event.type": "error", + "fileset.name": "log", + "input.type": "log", + "kafka.log.class": "state.change.logger", + "kafka.log.component": "Controller id=1 epoch=25", + "kafka.log.thread": "controller-event-thread", + "log.flags": [ + "multiline" + ], + "log.level": "ERROR", + "log.offset": 303, + "message": "Controller 1 epoch 25 failed to change state for partition xxx.xxx.na.och.aud.1-0-2 from OfflinePartition to OnlinePartition", + "service.type": "kafka" + } +] \ No newline at end of file From 0800ab1cd35b95ae55934b611628176c0c739fa2 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 1 Jul 2020 15:05:00 +0200 Subject: [PATCH 04/35] Implement storage handling for cursor inputs (#19518) This change provide the store implementation that is used by the cursor input to track ephemeral and persistent state. The full list of changes will include: - Introduce v2 API interfaces - Introduce [compatibility layer](https://github.com/urso/beats/tree/fb-input-v2-combined/filebeat/input/v2/compat) to integrate API with existing functionality - Introduce helpers for writing [stateless](https://github.com/urso/beats/blob/fb-input-v2-combined/filebeat/input/v2/input-stateless/stateless.go) inputs. - Introduce helpers for writing [inputs that store a state](https://github.com/urso/beats/tree/fb-input-v2-combined/filebeat/input/v2/input-cursor) between restarts. - Integrate new API with [existing inputs and modules](https://github.com/urso/beats/blob/fb-input-v2-combined/filebeat/beater/filebeat.go#L301) in filebeat. --- filebeat/input/v2/input-cursor/publish.go | 36 ++ filebeat/input/v2/input-cursor/store.go | 131 ++++++- filebeat/input/v2/input-cursor/store_test.go | 351 +++++++++++++++++++ 3 files changed, 513 insertions(+), 5 deletions(-) create mode 100644 filebeat/input/v2/input-cursor/store_test.go diff --git a/filebeat/input/v2/input-cursor/publish.go b/filebeat/input/v2/input-cursor/publish.go index 437c4e07373f..b5d33a7fce4d 100644 --- a/filebeat/input/v2/input-cursor/publish.go +++ b/filebeat/input/v2/input-cursor/publish.go @@ -22,6 +22,7 @@ import ( input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" ) // Publisher is used to publish an event and update the cursor in a single call to Publish. @@ -70,3 +71,38 @@ func (c *cursorPublisher) Publish(event beat.Event, cursorUpdate interface{}) er func (op *updateOp) Execute(numEvents uint) { panic("TODO: implement me") } + +func createUpdateOp(store *store, resource *resource, updates interface{}) (*updateOp, error) { + ts := time.Now() + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + cursor := resource.pendingCursor + if resource.activeCursorOperations == 0 { + var tmp interface{} + typeconv.Convert(&tmp, cursor) + resource.pendingCursor = tmp + cursor = tmp + } + if err := typeconv.Convert(&cursor, updates); err != nil { + return nil, err + } + resource.pendingCursor = cursor + + resource.Retain() + resource.activeCursorOperations++ + return &updateOp{ + resource: resource, + store: store, + timestamp: ts, + delta: updates, + }, nil +} + +// done releases resources held by the last N updateOps. +func (op *updateOp) done(n uint) { + op.resource.UpdatesReleaseN(n) + op.resource = nil + *op = updateOp{} +} diff --git a/filebeat/input/v2/input-cursor/store.go b/filebeat/input/v2/input-cursor/store.go index f44cf6760dca..66d5b4509363 100644 --- a/filebeat/input/v2/input-cursor/store.go +++ b/filebeat/input/v2/input-cursor/store.go @@ -18,10 +18,12 @@ package cursor import ( + "strings" "sync" "time" "github.com/elastic/beats/v7/libbeat/common/atomic" + "github.com/elastic/beats/v7/libbeat/common/cleanup" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore" @@ -126,7 +128,25 @@ type ( var closeStore = (*store).close func openStore(log *logp.Logger, statestore StateStore, prefix string) (*store, error) { - panic("TODO: implement me") + ok := false + + persistentStore, err := statestore.Access() + if err != nil { + return nil, err + } + defer cleanup.IfNot(&ok, func() { persistentStore.Close() }) + + states, err := readStates(log, persistentStore, prefix) + if err != nil { + return nil, err + } + + ok = true + return &store{ + log: log, + persistentStore: persistentStore, + ephemeralStore: states, + }, nil } func (s *store) Retain() { s.refCount.Retain() } @@ -137,7 +157,9 @@ func (s *store) Release() { } func (s *store) close() { - panic("TODO: implement me") + if err := s.persistentStore.Close(); err != nil { + s.log.Errorf("Closing registry store did report an error: %+v", err) + } } // Get returns the resource for the key. @@ -152,18 +174,62 @@ func (s *store) Get(key string) *resource { // On update the resource its `cursor` state is used, to keep the cursor state in sync with the current known // on disk store state. func (s *store) UpdateTTL(resource *resource, ttl time.Duration) { - panic("TODO: implement me") + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + if resource.stored && resource.internalState.TTL == ttl { + return + } + + resource.internalState.TTL = ttl + if resource.internalState.Updated.IsZero() { + resource.internalState.Updated = time.Now() + } + + err := s.persistentStore.Set(resource.key, state{ + TTL: resource.internalState.TTL, + Updated: resource.internalState.Updated, + Cursor: resource.cursor, + }) + if err != nil { + s.log.Errorf("Failed to update resource management fields for '%v'", resource.key) + resource.internalInSync = false + } else { + resource.stored = true + resource.internalInSync = true + } } // Find returns the resource for a given key. If the key is unknown and create is set to false nil will be returned. // The resource returned by Find is marked as active. (*resource).Release must be called to mark the resource as inactive again. func (s *states) Find(key string, create bool) *resource { - panic("TODO: implement me") + s.mu.Lock() + defer s.mu.Unlock() + + if resource := s.table[key]; resource != nil { + resource.Retain() + return resource + } + + if !create { + return nil + } + + // resource is owned by table(session) and input that uses the resource. + resource := &resource{ + stored: false, + key: key, + lock: unison.MakeMutex(), + } + s.table[key] = resource + resource.Retain() + return resource } // IsNew returns true if we have no state recorded for the current resource. func (r *resource) IsNew() bool { - panic("TODO: implement me") + r.stateMutex.Lock() + defer r.stateMutex.Unlock() + return r.pendingCursor == nil && r.cursor == nil } // Retain is used to indicate that 'resource' gets an additional 'owner'. @@ -201,3 +267,58 @@ func (r *resource) inSyncStateSnapshot() state { Cursor: r.cursor, } } + +// stateSnapshot returns the current in memory state, that already contains state updates +// not yet ACKed. +func (r *resource) stateSnapshot() state { + cursor := r.pendingCursor + if r.activeCursorOperations == 0 { + cursor = r.cursor + } + + return state{ + TTL: r.internalState.TTL, + Updated: r.internalState.Updated, + Cursor: cursor, + } +} + +func readStates(log *logp.Logger, store *statestore.Store, prefix string) (*states, error) { + keyPrefix := prefix + "::" + states := &states{ + table: map[string]*resource{}, + } + + err := store.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + if !strings.HasPrefix(string(key), keyPrefix) { + return true, nil + } + + var st state + if err := dec.Decode(&st); err != nil { + log.Errorf("Failed to read regisry state for '%v', cursor state will be ignored. Error was: %+v", + key, err) + return true, nil + } + + resource := &resource{ + key: key, + stored: true, + lock: unison.MakeMutex(), + internalInSync: true, + internalState: stateInternal{ + TTL: st.TTL, + Updated: st.Updated, + }, + cursor: st.Cursor, + } + states.table[resource.key] = resource + + return true, nil + }) + + if err != nil { + return nil, err + } + return states, nil +} diff --git a/filebeat/input/v2/input-cursor/store_test.go b/filebeat/input/v2/input-cursor/store_test.go new file mode 100644 index 000000000000..2fa3f6eaa3e8 --- /dev/null +++ b/filebeat/input/v2/input-cursor/store_test.go @@ -0,0 +1,351 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/storetest" +) + +type testStateStore struct { + Store *statestore.Store + GCPeriod time.Duration +} + +func TestStore_OpenClose(t *testing.T) { + t.Run("releasing store closes", func(t *testing.T) { + var closed bool + cleanup := closeStoreWith(func(s *store) { + closed = true + s.close() + }) + defer cleanup() + + store := testOpenStore(t, "test", nil) + store.Release() + + require.True(t, closed) + }) + + t.Run("fail if persistent store can not be accessed", func(t *testing.T) { + _, err := openStore(logp.NewLogger("test"), testStateStore{}, "test") + require.Error(t, err) + }) + + t.Run("load from empty", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + require.Equal(t, 0, len(storeMemorySnapshot(store))) + require.Equal(t, 0, len(storeInSyncSnapshot(store))) + }) + + t.Run("already available state is loaded", func(t *testing.T) { + states := map[string]state{ + "test::key0": {Cursor: "1"}, + "test::key1": {Cursor: "2"}, + } + + store := testOpenStore(t, "test", createSampleStore(t, states)) + defer store.Release() + + checkEqualStoreState(t, states, storeMemorySnapshot(store)) + checkEqualStoreState(t, states, storeInSyncSnapshot(store)) + }) + + t.Run("ignore entries with wrong index on open", func(t *testing.T) { + states := map[string]state{ + "test::key0": {Cursor: "1"}, + "other::key": {Cursor: "2"}, + } + + store := testOpenStore(t, "test", createSampleStore(t, states)) + defer store.Release() + + want := map[string]state{ + "test::key0": {Cursor: "1"}, + } + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) +} + +func TestStore_Get(t *testing.T) { + t.Run("find existing resource", func(t *testing.T) { + cursorState := state{Cursor: "1"} + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key0": cursorState, + })) + defer store.Release() + + res := store.Get("test::key0") + require.NotNil(t, res) + defer res.Release() + + // check in memory state matches matches original persistent state + require.Equal(t, cursorState, res.stateSnapshot()) + // check assumed in-sync state matches matches original persistent state + require.Equal(t, cursorState, res.inSyncStateSnapshot()) + }) + + t.Run("access unknown resource", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + require.NotNil(t, res) + defer res.Release() + + // new resource has empty state + require.Equal(t, state{}, res.stateSnapshot()) + }) + + t.Run("same resource is returned", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res1 := store.Get("test::key") + require.NotNil(t, res1) + defer res1.Release() + + res2 := store.Get("test::key") + require.NotNil(t, res2) + defer res2.Release() + + assert.Equal(t, res1, res2) + }) +} + +func TestStore_UpdateTTL(t *testing.T) { + t.Run("add TTL for new entry to store", func(t *testing.T) { + // when creating a resource we set the TTL and insert a new key value pair without cursor value into the store: + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + store.UpdateTTL(res, 60*time.Second) + + want := map[string]state{ + "test::key": { + TTL: 60 * time.Second, + Updated: res.internalState.Updated, + Cursor: nil, + }, + } + + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) + + t.Run("update TTL for in-sync resource does not overwrite state", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Second, + Cursor: "test", + }, + })) + defer store.Release() + + res := store.Get("test::key") + store.UpdateTTL(res, 60*time.Second) + want := map[string]state{ + "test::key": { + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test", + }, + } + + checkEqualStoreState(t, want, storeMemorySnapshot(store)) + checkEqualStoreState(t, want, storeInSyncSnapshot(store)) + }) + + t.Run("update TTL for resource with pending updates", func(t *testing.T) { + // This test updates the resource TTL while update operations are still + // pending, but not synced to the persistent store yet. + // UpdateTTL changes the state in the persistent store immediately, and must therefore + // serialize the old in-sync state with update meta-data. + + // create store + backend := createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Second, + Cursor: "test", + }, + }) + store := testOpenStore(t, "test", backend) + defer store.Release() + + // create pending update operation + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + // Update key/value pair TTL. This will update the internal state in the + // persistent store only, not modifying the old cursor state yet. + store.UpdateTTL(res, 60*time.Second) + + // validate + wantMemoryState := state{ + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test-state-update", + } + wantInSyncState := state{ + Updated: res.internalState.Updated, + TTL: 60 * time.Second, + Cursor: "test", + } + + checkEqualStoreState(t, map[string]state{"test::key": wantMemoryState}, storeMemorySnapshot(store)) + checkEqualStoreState(t, map[string]state{"test::key": wantInSyncState}, storeInSyncSnapshot(store)) + checkEqualStoreState(t, map[string]state{"test::key": wantInSyncState}, backend.snapshot()) + }) +} + +func closeStoreWith(fn func(s *store)) func() { + old := closeStore + closeStore = fn + return func() { + closeStore = old + } +} + +func testOpenStore(t *testing.T, prefix string, persistentStore StateStore) *store { + if persistentStore == nil { + persistentStore = createSampleStore(t, nil) + } + + store, err := openStore(logp.NewLogger("test"), persistentStore, prefix) + if err != nil { + t.Fatalf("failed to open the store") + } + return store +} + +func createSampleStore(t *testing.T, data map[string]state) testStateStore { + storeReg := statestore.NewRegistry(storetest.NewMemoryStoreBackend()) + store, err := storeReg.Get("test") + if err != nil { + t.Fatalf("Failed to access store: %v", err) + } + + for k, v := range data { + if err := store.Set(k, v); err != nil { + t.Fatalf("Error when populating the sample store: %v", err) + } + } + + return testStateStore{ + Store: store, + } +} + +func (ts testStateStore) WithGCPeriod(d time.Duration) testStateStore { ts.GCPeriod = d; return ts } +func (ts testStateStore) CleanupInterval() time.Duration { return ts.GCPeriod } +func (ts testStateStore) Access() (*statestore.Store, error) { + if ts.Store == nil { + return nil, errors.New("no store configured") + } + return ts.Store, nil +} + +// snapshot copies all key/value pairs from the persistent store into a table for inspection. +func (ts testStateStore) snapshot() map[string]state { + states := map[string]state{} + err := ts.Store.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + var st state + if err := dec.Decode(&st); err != nil { + return false, err + } + states[key] = st + return true, nil + }) + + if err != nil { + panic("unexpected decode error from persistent test store") + } + return states +} + +// storeMemorySnapshot copies all key/value pairs into a table for inspection. +// The state returned reflects the in memory state, which can be ahead of the +// persistent state. +// +// Note: The state returned by storeMemorySnapshot is always ahead of the state returned by storeInSyncSnapshot. +// All key value pairs are fully in-sync, if both snapshot functions return the same state. +func storeMemorySnapshot(store *store) map[string]state { + store.ephemeralStore.mu.Lock() + defer store.ephemeralStore.mu.Unlock() + + states := map[string]state{} + for k, res := range store.ephemeralStore.table { + states[k] = res.stateSnapshot() + } + return states +} + +// storeInSyncSnapshot copies all key/value pairs into the table for inspection. +// The state returned reflects the current state that the in-memory tables assumed to be +// written to the persistent store already. + +// Note: The state returned by storeMemorySnapshot is always ahead of the state returned by storeInSyncSnapshot. +// All key value pairs are fully in-sync, if both snapshot functions return the same state. +func storeInSyncSnapshot(store *store) map[string]state { + store.ephemeralStore.mu.Lock() + defer store.ephemeralStore.mu.Unlock() + + states := map[string]state{} + for k, res := range store.ephemeralStore.table { + states[k] = res.inSyncStateSnapshot() + } + return states +} + +// checkEqualStoreState compares 2 store snapshot tables for equality. The test +// fails with Errorf if the state differ. +// +// Note: testify is too strict when comparing timestamp, better use checkEqualStoreState. +func checkEqualStoreState(t *testing.T, want, got map[string]state) bool { + if d := cmp.Diff(want, got); d != "" { + t.Errorf("store state mismatch (-want +got):\n%s", d) + return false + } + return true +} + +// requireEqualStoreState compares 2 store snapshot tables for equality. The test +// fails with Fatalf if the state differ. +// +// Note: testify is too strict when comparing timestamp, better use checkEqualStoreState. +func requireEqualStoreState(t *testing.T, want, got map[string]state) bool { + if d := cmp.Diff(want, got); d != "" { + t.Fatalf("store state mismatch (-want +got):\n%s", d) + return false + } + return true +} From 2a1b4f49dc318f275dc42b96326a967b608bb04f Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 1 Jul 2020 16:17:04 +0200 Subject: [PATCH 05/35] [Ingest Manager] Use libbeat logger (#19211) [Ingest Manager] Use libbeat logger (#19211) --- NOTICE.txt | 3369 ++++------------- dev-tools/notice/overrides.json | 1 + go.mod | 2 +- go.sum | 31 +- libbeat/common/file/rotator.go | 3 +- libbeat/logp/config.go | 28 +- libbeat/logp/level.go | 20 + vendor/modules.txt | 1457 +++++++ .../_meta/config/common.p2.yml.tmpl | 71 +- .../_meta/config/common.reference.p2.yml.tmpl | 71 +- .../config/elastic-agent.docker.yml.tmpl | 71 +- x-pack/elastic-agent/_meta/elastic-agent.yml | 71 +- .../elastic-agent_configuration_example.yml | 73 +- x-pack/elastic-agent/elastic-agent.docker.yml | 71 +- .../elastic-agent/elastic-agent.reference.yml | 71 +- x-pack/elastic-agent/elastic-agent.yml | 71 +- .../agent/application/action_dispatcher.go | 2 +- .../agent/application/action_store_test.go | 2 +- .../pkg/agent/application/enroll_cmd_test.go | 2 +- .../pkg/agent/application/fleet_acker_test.go | 2 +- .../agent/application/fleet_gateway_test.go | 6 +- .../handler_action_policy_change_test.go | 4 +- .../application/introspect_output_cmd.go | 12 +- .../pkg/agent/application/lazy_acker_test.go | 2 +- .../pkg/agent/application/local_mode.go | 2 +- .../agent/application/managed_mode_test.go | 2 +- .../pkg/agent/application/router.go | 2 +- x-pack/elastic-agent/pkg/agent/cmd/enroll.go | 2 +- x-pack/elastic-agent/pkg/agent/cmd/run.go | 2 +- .../pkg/agent/operation/common_test.go | 2 +- .../pkg/agent/program/program_test.go | 2 +- .../agent/stateresolver/stateresolver_test.go | 4 +- .../elastic-agent/pkg/core/logger/logger.go | 121 +- .../pkg/core/plugin/process/start.go | 2 - .../pkg/core/server/server_test.go | 2 +- .../elastic-agent/pkg/filewatcher/watcher.go | 2 +- x-pack/elastic-agent/pkg/fleetapi/client.go | 2 +- .../elastic-agent/pkg/fleetapi/helper_test.go | 2 +- x-pack/elastic-agent/pkg/kibana/client.go | 2 +- .../elastic-agent/pkg/kibana/client_test.go | 4 +- .../pkg/reporter/fleet/reporter_test.go | 2 +- 41 files changed, 2806 insertions(+), 2864 deletions(-) create mode 100644 vendor/modules.txt diff --git a/NOTICE.txt b/NOTICE.txt index 7cf8cd112fe4..1f1b68965d41 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13547,217 +13547,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/urso/ecslog -Version: v0.0.1 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/urso/ecslog@v0.0.1/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/vmware/govmomi Version: v0.0.0-20170802214208-2cad15190b41 @@ -19995,37 +19784,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/alcortesm/tgz -Version: v0.0.0-20161220082320-9c5fe88206d7 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/alcortesm/tgz@v0.0.0-20161220082320-9c5fe88206d7/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2016 Alberto Cortés - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/alecthomas/template Version: v0.0.0-20190718012654-fb15b899a751 @@ -20093,49 +19851,19 @@ SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/anmitsu/go-shlex -Version: v0.0.0-20161002113705-648efa622239 +Dependency : github.com/antihax/optional +Version: v0.0.0-20180407024304-ca021399b1a6 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/anmitsu/go-shlex@v0.0.0-20161002113705-648efa622239/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/antihax/optional@v0.0.0-20180407024304-ca021399b1a6/LICENSE: -Copyright (c) anmitsu +The MIT License (MIT) +Copyright (c) 2016 Adam Hintz -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/antihax/optional -Version: v0.0.0-20180407024304-ca021399b1a6 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/antihax/optional@v0.0.0-20180407024304-ca021399b1a6/LICENSE: - -The MIT License (MIT) -Copyright (c) 2016 Adam Hintz - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @@ -22917,39 +22645,6 @@ third-party archives. limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/creack/pty -Version: v1.1.7 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/creack/pty@v1.1.7/LICENSE: - -Copyright (c) 2011 Keith Rarick - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, -sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall -be included in all copies or substantial portions of the -Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS -OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/cucumber/godog Version: v0.8.1 @@ -24344,57 +24039,6 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/emirpasic/gods -Version: v1.12.0 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/emirpasic/gods@v1.12.0/LICENSE: - -Copyright (c) 2015, Emir Pasic -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -------------------------------------------------------------------------------- - -AVL Tree: - -Copyright (c) 2017 Benjamin Scher Purcell - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/envoyproxy/go-control-plane Version: v0.9.4 @@ -24854,314 +24498,102 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/flynn/go-shlex -Version: v0.0.0-20150515145356-3f9db97f8568 -Licence type (autodetected): Apache-2.0 +Dependency : github.com/fortytw2/leaktest +Version: v1.3.0 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/flynn/go-shlex@v0.0.0-20150515145356-3f9db97f8568/COPYING: +Contents of probable licence file $GOMODCACHE/github.com/fortytw2/leaktest@v1.3.0/LICENSE: +Copyright (c) 2012 The Go Authors. All rights reserved. - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - 1. Definitions. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +-------------------------------------------------------------------------------- +Dependency : github.com/frankban/quicktest +Version: v1.7.2 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.7.2/LICENSE: - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +MIT License - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +Copyright (c) 2017 Canonical Ltd. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. +-------------------------------------------------------------------------------- +Dependency : github.com/ghodss/yaml +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. +Contents of probable licence file $GOMODCACHE/github.com/ghodss/yaml@v1.0.0/LICENSE: - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +The MIT License (MIT) - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +Copyright (c) 2014 Sam Ghods - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/fortytw2/leaktest -Version: v1.3.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/fortytw2/leaktest@v1.3.0/LICENSE: - -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/frankban/quicktest -Version: v1.7.2 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.7.2/LICENSE: - -MIT License - -Copyright (c) 2017 Canonical Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/ghodss/yaml -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/ghodss/yaml@v1.0.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Sam Ghods - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. Copyright (c) 2012 The Go Authors. All rights reserved. @@ -25193,43 +24625,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/gliderlabs/ssh -Version: v0.2.2 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/gliderlabs/ssh@v0.2.2/LICENSE: - -Copyright (c) 2016 Glider Labs. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Glider Labs nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/go-gl/glfw/v3.3/glfw Version: v0.0.0-20191125211704-12ad95a8df72 @@ -31758,37 +31153,6 @@ See the License for the specific language governing permissions and limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/jbenet/go-context -Version: v0.0.0-20150711004518-d14ea06fba99 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/jbenet/go-context@v0.0.0-20150711004518-d14ea06fba99/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Juan Batiz-Benet - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/jcmturner/gofork Version: v1.0.0 @@ -32106,65 +31470,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/kevinburke/ssh_config -Version: v0.0.0-20190725054713-01f96b0aa0cd -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/kevinburke/ssh_config@v0.0.0-20190725054713-01f96b0aa0cd/LICENSE: - -Copyright (c) 2017 Kevin Burke. - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -=================== - -The lexer and parser borrow heavily from github.com/pelletier/go-toml. The -license for that project is copied below. - -The MIT License (MIT) - -Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/kisielk/errcheck Version: v1.2.0 @@ -32349,13 +31654,13 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/kr/pty -Version: v1.1.8 +Version: v1.1.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/kr/pty@v1.1.8/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/kr/pty@v1.1.1/License: -Copyright (c) 2019 Keith Rarick +Copyright (c) 2011 Keith Rarick Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated @@ -33055,37 +32360,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/mitchellh/go-wordwrap -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mitchellh/go-wordwrap@v1.0.0/LICENSE.md: - -The MIT License (MIT) - -Copyright (c) 2014 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -------------------------------------------------------------------------------- Dependency : github.com/mitchellh/iochan Version: v1.0.0 @@ -35105,14 +34379,6 @@ Contents of probable licence file $GOMODCACHE/github.com/oxtoacart/bpool@v0.0.0- limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/pelletier/go-buffruneio -Version: v0.2.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -No licence file provided. - -------------------------------------------------------------------------------- Dependency : github.com/peterbourgon/diskv Version: v2.0.1+incompatible @@ -36364,44 +35630,6 @@ Contents of probable licence file $GOMODCACHE/github.com/spf13/afero@v1.2.2/LICE of your accepting any such warranty or additional liability. --------------------------------------------------------------------------------- -Dependency : github.com/src-d/gcfg -Version: v1.4.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/src-d/gcfg@v1.4.0/LICENSE: - -Copyright (c) 2012 Péter Surányi. Portions Copyright (c) 2009 The Go -Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/stretchr/objx Version: v0.2.0 @@ -36711,12 +35939,12 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/diag@v0.0.0-202002 -------------------------------------------------------------------------------- -Dependency : github.com/urso/diag-ecs -Version: v0.0.0-20200210114345-ab085841dcb9 +Dependency : github.com/urso/go-bin +Version: v0.0.0-20180220135811-781c575c9f0e Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/diag-ecs@v0.0.0-20200210114345-ab085841dcb9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/urso/go-bin@v0.0.0-20180220135811-781c575c9f0e/LICENSE: Apache License Version 2.0, January 2004 @@ -36922,12 +36150,20 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/diag-ecs@v0.0.0-20 -------------------------------------------------------------------------------- -Dependency : github.com/urso/go-bin -Version: v0.0.0-20180220135811-781c575c9f0e +Dependency : github.com/urso/magetools +Version: v0.0.0-20190919040553-290c89e0c230 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/go-bin@v0.0.0-20180220135811-781c575c9f0e/LICENSE: +No licence file provided. + +-------------------------------------------------------------------------------- +Dependency : github.com/urso/qcgen +Version: v0.0.0-20180131103024-0b059e7db4f4 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180131103024-0b059e7db4f4/LICENSE: Apache License Version 2.0, January 2004 @@ -37133,12 +36369,12 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/go-bin@v0.0.0-2018 -------------------------------------------------------------------------------- -Dependency : github.com/urso/magetools -Version: v0.0.0-20200125210132-c2e338f92f3a +Dependency : github.com/urso/sderr +Version: v0.0.0-20200210124243-c2a16f3d43ec Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/magetools@v0.0.0-20200125210132-c2e338f92f3a/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: Apache License Version 2.0, January 2004 @@ -37344,14 +36580,52 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/magetools@v0.0.0-2 -------------------------------------------------------------------------------- -Dependency : github.com/urso/qcgen -Version: v0.0.0-20180131103024-0b059e7db4f4 +Dependency : github.com/vbatts/tar-split +Version: v0.11.1 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/vbatts/tar-split@v0.11.1/LICENSE: + +Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/xanzy/go-gitlab +Version: v0.22.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180131103024-0b059e7db4f4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xanzy/go-gitlab@v0.22.3/LICENSE: - Apache License +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -37531,7 +36805,7 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180 APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -37539,7 +36813,7 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -37554,13 +36828,15 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180 limitations under the License. + -------------------------------------------------------------------------------- -Dependency : github.com/urso/sderr -Version: v0.0.0-20200210124243-c2a16f3d43ec +Dependency : github.com/xdg/stringprep +Version: v1.0.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xdg/stringprep@v1.0.0/LICENSE: + Apache License Version 2.0, January 2004 @@ -37737,104 +37013,40 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200 incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. +-------------------------------------------------------------------------------- +Dependency : github.com/xeipuuv/gojsonpointer +Version: v0.0.0-20190905194746-02993c407bfb +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. +Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v0.0.0-20190905194746-02993c407bfb/LICENSE-APACHE-2.0.txt: - Copyright [yyyy] [name of copyright owner] - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ - http://www.apache.org/licenses/LICENSE-2.0 + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + 1. Definitions. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. --------------------------------------------------------------------------------- -Dependency : github.com/vbatts/tar-split -Version: v0.11.1 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. -Contents of probable licence file $GOMODCACHE/github.com/vbatts/tar-split@v0.11.1/LICENSE: + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. -Copyright (c) 2015 Vincent Batts, Raleigh, NC, USA - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/xanzy/go-gitlab -Version: v0.22.3 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/xanzy/go-gitlab@v0.22.3/LICENSE: - -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation @@ -37991,7 +37203,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -37999,7 +37211,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2015 xeipuuv Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38014,14 +37226,14 @@ Apache License limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/xanzy/ssh-agent -Version: v0.2.1 +Dependency : github.com/xeipuuv/gojsonreference +Version: v0.0.0-20180127040603-bd5ef7bd5415 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonreference@v0.0.0-20180127040603-bd5ef7bd5415/LICENSE-APACHE-2.0.txt: + Apache License Version 2.0, January 2004 @@ -38203,7 +37415,7 @@ Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -38211,7 +37423,7 @@ Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2015 xeipuuv Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38226,14 +37438,13 @@ Contents of probable licence file $GOMODCACHE/github.com/xanzy/ssh-agent@v0.2.1/ limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/xdg/stringprep -Version: v1.0.0 +Dependency : github.com/xeipuuv/gojsonschema +Version: v0.0.0-20181112162635-ac52e6811b56 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xdg/stringprep@v1.0.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonschema@v0.0.0-20181112162635-ac52e6811b56/LICENSE-APACHE-2.0.txt: Apache License @@ -38411,14 +37622,105 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg/stringprep@v1.0.0/L incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015 xeipuuv + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + -------------------------------------------------------------------------------- -Dependency : github.com/xeipuuv/gojsonpointer -Version: v0.0.0-20190905194746-02993c407bfb +Dependency : github.com/yuin/gopher-lua +Version: v0.0.0-20170403160031-b402f3114ec7 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/yuin/gopher-lua@v0.0.0-20170403160031-b402f3114ec7/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 Yusuke Inuzuka + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : go.elastic.co/fastjson +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.0.0/LICENSE: + +Copyright 2018 Elasticsearch BV + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +--- + +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : go.opencensus.io +Version: v0.22.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v0.0.0-20190905194746-02993c407bfb/LICENSE-APACHE-2.0.txt: +Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.22.2/LICENSE: Apache License @@ -38609,7 +37911,7 @@ Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2015 xeipuuv + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38623,1209 +37925,75 @@ Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonpointer@v See the License for the specific language governing permissions and limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/xeipuuv/gojsonreference -Version: v0.0.0-20180127040603-bd5ef7bd5415 -Licence type (autodetected): Apache-2.0 +Dependency : go.uber.org/goleak +Version: v1.0.0 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonreference@v0.0.0-20180127040603-bd5ef7bd5415/LICENSE-APACHE-2.0.txt: +Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.0.0/LICENSE: +The MIT License (MIT) - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Copyright (c) 2018 Uber Technologies, Inc. - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - 1. Definitions. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +-------------------------------------------------------------------------------- +Dependency : go.uber.org/tools +Version: v0.0.0-20190618225709-2cfd321de3ee +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +Contents of probable licence file $GOMODCACHE/go.uber.org/tools@v0.0.0-20190618225709-2cfd321de3ee/LICENSE: - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +Copyright (c) 2017 Uber Technologies, Inc. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2015 xeipuuv - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/xeipuuv/gojsonschema -Version: v0.0.0-20181112162635-ac52e6811b56 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonschema@v0.0.0-20181112162635-ac52e6811b56/LICENSE-APACHE-2.0.txt: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2015 xeipuuv - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/yuin/gopher-lua -Version: v0.0.0-20170403160031-b402f3114ec7 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/yuin/gopher-lua@v0.0.0-20170403160031-b402f3114ec7/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 Yusuke Inuzuka - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : go.elastic.co/fastjson -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.0.0/LICENSE: - -Copyright 2018 Elasticsearch BV - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - ---- - -Copyright (c) 2016 Mail.Ru Group - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : go.opencensus.io -Version: v0.22.2 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.opencensus.io@v0.22.2/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --------------------------------------------------------------------------------- -Dependency : go.uber.org/goleak -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.uber.org/goleak@v1.0.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2018 Uber Technologies, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : go.uber.org/tools -Version: v0.0.0-20190618225709-2cfd321de3ee -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/go.uber.org/tools@v0.0.0-20190618225709-2cfd321de3ee/LICENSE: - -Copyright (c) 2017 Uber Technologies, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/exp -Version: v0.0.0-20191227195350-da58074b4299 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20191227195350-da58074b4299/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/image -Version: v0.0.0-20190802002840-cff245a6509b -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/image@v0.0.0-20190802002840-cff245a6509b/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/mobile -Version: v0.0.0-20190719004257-d2bd2a29d028 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/mobile@v0.0.0-20190719004257-d2bd2a29d028/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/mod -Version: v0.3.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.3.0/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : golang.org/x/xerrors -Version: v0.0.0-20191204190536-9bdfabe68543 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/LICENSE: - -Copyright (c) 2019 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : google.golang.org/appengine -Version: v1.6.5 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/google.golang.org/appengine@v1.6.5/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : google.golang.org/protobuf -Version: v1.23.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.23.0/LICENSE: - -Copyright (c) 2018 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : gopkg.in/airbrake/gobrake.v2 -Version: v2.0.9 +Dependency : golang.org/x/exp +Version: v0.0.0-20191227195350-da58074b4299 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/airbrake/gobrake.v2@v2.0.9/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20191227195350-da58074b4299/LICENSE: -Copyright (c) 2014 The Gobrake Authors. All rights reserved. +Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -39846,124 +38014,23 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/alecthomas/kingpin.v2 -Version: v2.2.6 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/alecthomas/kingpin.v2@v2.2.6/COPYING: - -Copyright (C) 2014 Alec Thomas - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/check.v1 -Version: v1.0.0-20190902080502-41f04d3bba15 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20190902080502-41f04d3bba15/LICENSE: - -Gocheck - A rich testing framework for Go - -Copyright (c) 2010-2013 Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : gopkg.in/errgo.v2 -Version: v2.1.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/errgo.v2@v2.1.0/LICENSE: - -Copyright © 2013, Roger Peppe -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of this project nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : gopkg.in/fsnotify.v1 -Version: v1.4.7 +Dependency : golang.org/x/image +Version: v0.0.0-20190802002840-cff245a6509b Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/fsnotify.v1@v1.4.7/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/image@v0.0.0-20190802002840-cff245a6509b/LICENSE: -Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2012 fsnotify Authors. All rights reserved. +Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -39993,254 +38060,124 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : gopkg.in/gemnasium/logrus-airbrake-hook.v2 -Version: v2.1.2 -Licence type (autodetected): MIT +Dependency : golang.org/x/mobile +Version: v0.0.0-20190719004257-d2bd2a29d028 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/gemnasium/logrus-airbrake-hook.v2@v2.1.2/LICENSE: - -The MIT License (MIT) +Contents of probable licence file $GOMODCACHE/golang.org/x/mobile@v0.0.0-20190719004257-d2bd2a29d028/LICENSE: -Copyright (c) 2015 Gemnasium +Copyright (c) 2009 The Go Authors. All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/aescts.v1 -Version: v1.0.1 -Licence type (autodetected): Apache-2.0 +Dependency : golang.org/x/mod +Version: v0.3.0 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/aescts.v1@v1.0.1/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.3.0/LICENSE: - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. +Copyright (c) 2009 The Go Authors. All rights reserved. - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - END OF TERMS AND CONDITIONS +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - APPENDIX: How to apply the Apache License to your work. - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/xerrors +Version: v0.0.0-20191204190536-9bdfabe68543 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - Copyright {yyyy} {name of copyright owner} +Contents of probable licence file $GOMODCACHE/golang.org/x/xerrors@v0.0.0-20191204190536-9bdfabe68543/LICENSE: - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Copyright (c) 2019 The Go Authors. All rights reserved. - http://www.apache.org/licenses/LICENSE-2.0 +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/dnsutils.v1 -Version: v1.0.1 +Dependency : google.golang.org/appengine +Version: v1.6.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/dnsutils.v1@v1.0.1/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/appengine@v1.6.5/LICENSE: + Apache License Version 2.0, January 2004 @@ -40446,223 +38383,255 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/dnsutils.v1@v1. -------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/goidentity.v3 -Version: v3.0.0 -Licence type (autodetected): Apache-2.0 +Dependency : google.golang.org/protobuf +Version: v1.23.0 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/goidentity.v3@v3.0.0/LICENSE: +Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.23.0/LICENSE: - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Copyright (c) 2018 The Go Authors. All rights reserved. - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - 1. Definitions. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/airbrake/gobrake.v2 +Version: v2.0.9 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +Contents of probable licence file $GOMODCACHE/gopkg.in/airbrake/gobrake.v2@v2.0.9/LICENSE: - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +Copyright (c) 2014 The Gobrake Authors. All rights reserved. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/alecthomas/kingpin.v2 +Version: v2.2.6 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/alecthomas/kingpin.v2@v2.2.6/COPYING: + +Copyright (C) 2014 Alec Thomas + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/check.v1 +Version: v1.0.0-20190902080502-41f04d3bba15 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/gopkg.in/check.v1@v1.0.0-20190902080502-41f04d3bba15/LICENSE: + +Gocheck - A rich testing framework for Go + +Copyright (c) 2010-2013 Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : gopkg.in/errgo.v2 +Version: v2.1.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. +Contents of probable licence file $GOMODCACHE/gopkg.in/errgo.v2@v2.1.0/LICENSE: - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +Copyright © 2013, Roger Peppe +All rights reserved. - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of this project nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. +-------------------------------------------------------------------------------- +Dependency : gopkg.in/fsnotify.v1 +Version: v1.4.7 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. +Contents of probable licence file $GOMODCACHE/gopkg.in/fsnotify.v1@v1.4.7/LICENSE: - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2012 fsnotify Authors. All rights reserved. - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS +-------------------------------------------------------------------------------- +Dependency : gopkg.in/gemnasium/logrus-airbrake-hook.v2 +Version: v2.1.2 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - APPENDIX: How to apply the Apache License to your work. +Contents of probable licence file $GOMODCACHE/gopkg.in/gemnasium/logrus-airbrake-hook.v2@v2.1.2/LICENSE: - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. +The MIT License (MIT) - Copyright {yyyy} {name of copyright owner} +Copyright (c) 2015 Gemnasium - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - http://www.apache.org/licenses/LICENSE-2.0 +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : gopkg.in/jcmturner/rpc.v1 -Version: v1.1.0 +Dependency : gopkg.in/jcmturner/aescts.v1 +Version: v1.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/aescts.v1@v1.0.1/LICENSE: Apache License Version 2.0, January 2004 @@ -40844,7 +38813,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -40852,7 +38821,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -40868,12 +38837,12 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/L -------------------------------------------------------------------------------- -Dependency : gopkg.in/src-d/go-billy.v4 -Version: v4.3.2 +Dependency : gopkg.in/jcmturner/dnsutils.v1 +Version: v1.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/dnsutils.v1@v1.0.1/LICENSE: Apache License Version 2.0, January 2004 @@ -41055,7 +39024,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -41063,7 +39032,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2017 Sourced Technologies S.L. + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41079,12 +39048,12 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-billy.v4@v4.3.2/ -------------------------------------------------------------------------------- -Dependency : gopkg.in/src-d/go-git-fixtures.v3 -Version: v3.5.0 +Dependency : gopkg.in/jcmturner/goidentity.v3 +Version: v3.0.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git-fixtures.v3@v3.5.0/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/goidentity.v3@v3.0.0/LICENSE: Apache License Version 2.0, January 2004 @@ -41274,7 +39243,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git-fixtures.v3@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2017 Sourced Technologies, S.L. + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41290,12 +39259,12 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git-fixtures.v3@ -------------------------------------------------------------------------------- -Dependency : gopkg.in/src-d/go-git.v4 -Version: v4.13.1 +Dependency : gopkg.in/jcmturner/rpc.v1 +Version: v1.1.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git.v4@v4.13.1/LICENSE: +Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/rpc.v1@v1.1.0/LICENSE: Apache License Version 2.0, January 2004 @@ -41477,7 +39446,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git.v4@v4.13.1/L APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -41485,7 +39454,7 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/src-d/go-git.v4@v4.13.1/L same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2018 Sourced Technologies, S.L. + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41539,40 +39508,6 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : gopkg.in/warnings.v0 -Version: v0.1.2 -Licence type (autodetected): BSD-2-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/gopkg.in/warnings.v0@v0.1.2/LICENSE: - -Copyright (c) 2016 Péter Surányi. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : gopkg.in/yaml.v3 Version: v3.0.0-20200313102051-9f266ea9e77c diff --git a/dev-tools/notice/overrides.json b/dev-tools/notice/overrides.json index c40ec350c70d..16c8447a13d4 100644 --- a/dev-tools/notice/overrides.json +++ b/dev-tools/notice/overrides.json @@ -9,3 +9,4 @@ {"name": "github.com/chzyer/logex", "licenceType": "MIT"} {"name": "github.com/munnerz/goautoneg", "licenceType": "BSD-3-Clause"} {"name": "github.com/pelletier/go-buffruneio", "licenceType": "MIT"} +{"name": "github.com/urso/magetools", "licenceType": "Apache-2.0"} diff --git a/go.mod b/go.mod index 901d13a13044..f5cb0cde6fdb 100644 --- a/go.mod +++ b/go.mod @@ -139,10 +139,10 @@ require ( github.com/shirou/gopsutil v2.19.11+incompatible github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.5 + github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.6.1 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786 - github.com/urso/ecslog v0.0.1 github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 // indirect diff --git a/go.sum b/go.sum index 31b5a70a086e..214b18ae45d9 100644 --- a/go.sum +++ b/go.sum @@ -98,7 +98,6 @@ github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc h github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= github.com/akavel/rsrc v0.8.0 h1:zjWn7ukO9Kc5Q62DOJCcxGpXC18RawVtYAGdz2aLlfw= github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -107,7 +106,6 @@ github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 h1:7rj9qZ63knnVo2Z github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20/go.mod h1:cI59GRkC2FRaFYtgbYEqMlgnnfvAwXzjojyZKXwklNg= github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 h1:WFwa9pqou0Nb4DdfBOyaBTH0GqLE74Qwdf61E7ITHwQ= github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43/go.mod h1:tJPYQG4mnMeUtQvQKNkbsFrnmZOg59Qnf8CcctFv5v4= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5 h1:nkZ9axP+MvUFCu8JRN/MCY+DmTfs6lY7hE0QnJbxSdI= github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= @@ -175,7 +173,6 @@ github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQa github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cucumber/godog v0.8.1 h1:lVb+X41I4YDreE+ibZ50bdXmySxgRviYFgKY6Aw4XE8= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= @@ -247,8 +244,6 @@ github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f h1:Fvsq github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f/go.mod h1:OPGqFNdTS34kMReS5hPFtBhD9J8itmSDurs1ix2wx7c= github.com/elastic/go-seccomp-bpf v1.1.0 h1:jUzzDc6LyCtdolZdvL/26dad6rZ9vsc7xZ2eadKECAU= github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8= -github.com/elastic/go-structform v0.0.6 h1:wqeK4LwD2NNDOoRGTImE24S6pkCDVr8+oUSIkmChzLk= -github.com/elastic/go-structform v0.0.6/go.mod h1:QrMyP3oM9Sjk92EVGLgRaL2lKt0Qx7ZNDRWDxB6khVs= github.com/elastic/go-structform v0.0.7 h1:ihszOJQryNuIIHE2ZgsbiDq+agKO6V4yK0JYAI3tjzc= github.com/elastic/go-structform v0.0.7/go.mod h1:QrMyP3oM9Sjk92EVGLgRaL2lKt0Qx7ZNDRWDxB6khVs= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= @@ -269,7 +264,6 @@ github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e/go.mod h github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -278,7 +272,6 @@ github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.5.0 h1:vBh+kQp8lg9XPr56u1CPrWjFXtdphMoGWVHr9/1c+A0= github.com/fatih/color v1.5.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= @@ -287,7 +280,6 @@ github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24 h1:nREVDi4H8mwnN github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= @@ -441,7 +433,6 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -472,7 +463,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/justinas/nosurf v1.1.0/go.mod h1:ALpWdSbuNGy2lZWtyXdjkYv4edL23oSEgfBT1gPJ5BQ= github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA= github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -488,7 +478,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -523,7 +512,6 @@ github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v1.0.1 h1:x0jD3dcHk9a9xPSDN6YEL4xL6Qz0dvNYm8yZqui5chI= github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4= github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 h1:qdHlMllk/PTLUrX3XdtXDrLL1lPSfcqUmJD1eYfbapg= @@ -572,7 +560,6 @@ github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -652,7 +639,6 @@ github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzu github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -673,18 +659,12 @@ github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786 h1:B/IVHYiI0d04dudYw+CvCAGqSMq8d0yWy56eD6p85BQ= github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786/go.mod h1:RIkfovP3Y7my19aXEjjbNd9E5TlHozzAyt7B8AaEcwg= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urso/diag v0.0.0-20200125202105-ffdc32ff5518/go.mod h1:pNWFTeQ+V1OYT/TzWpnWb6eQBdoXpdx+H+lrH97/Oyo= github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 h1:OHNw/6pXODJAB32NujjdQO/KIYQ3KAbHQfCzH81XdCs= github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797/go.mod h1:pNWFTeQ+V1OYT/TzWpnWb6eQBdoXpdx+H+lrH97/Oyo= -github.com/urso/diag-ecs v0.0.0-20200210114345-ab085841dcb9 h1:GzsakegdLNhw0mF2fcFd+BgdY8owV+4Y+6MvbgRXIWg= -github.com/urso/diag-ecs v0.0.0-20200210114345-ab085841dcb9/go.mod h1:+1Ug5A104KCMD8ZZ4YarKGNSt8ANJWA7nWqji37BmrQ= -github.com/urso/ecslog v0.0.1 h1:Candk+au3CbAT5SpVMhTE3VIq0r09UJely7vUzVAfF0= -github.com/urso/ecslog v0.0.1/go.mod h1:wky/kxUyw0VJw/HlXal+7oTT2YDU1KAWrxbuKRFblEI= github.com/urso/go-bin v0.0.0-20180220135811-781c575c9f0e h1:NiofbjIUI5gR+ybDsGSVH1fWyjSeDYiYVJHT1+kcsak= github.com/urso/go-bin v0.0.0-20180220135811-781c575c9f0e/go.mod h1:6GfHrdWBQYjFRIznu7XuQH4lYB2w8nO4bnImVKkzPOM= +github.com/urso/magetools v0.0.0-20190919040553-290c89e0c230 h1:Ft1EJ6JL0F/RV6o2qJ1Be+wYxjYUSfRA3srfHgSgojc= github.com/urso/magetools v0.0.0-20190919040553-290c89e0c230/go.mod h1:DFxTNgS/ExCGmmjVjSOgS2WjtfjKXgCyDzAFgbtovSA= -github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a h1:jWAaRFnay3H2e6S0GGCl5nKrkgQNlarCE/kvcutzBmw= -github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a/go.mod h1:DbaJnRzkGaWrMWm5Hz6QVnUj//x9/zjrfx8bF3J+GJY= github.com/urso/qcgen v0.0.0-20180131103024-0b059e7db4f4 h1:hhA8EBThzz9PztawVTycKvfETVuBqxAQ5keFlAVtbAw= github.com/urso/qcgen v0.0.0-20180131103024-0b059e7db4f4/go.mod h1:RspW+E2Yb7Fs7HclB2tiDaiu6Rp41BiIG4Wo1YaoXGc= github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec h1:HkZIDJrMKZHPsYhmH2XjTTSk1pbMCFfpxSnyzZUFm+k= @@ -694,7 +674,6 @@ github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 h1:NeNpIvfvaFOh0BH7 github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/xanzy/go-gitlab v0.22.3 h1:/rNlZ2hquUWNc6rJdntVM03tEOoTmnZ1lcNyJCl0WlU= github.com/xanzy/go-gitlab v0.22.3/go.mod h1:t4Bmvnxj7k37S4Y17lfLx+nLqkf/oQwT2HagfWKv5Og= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= @@ -740,12 +719,10 @@ go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -830,7 +807,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -840,7 +816,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -937,12 +912,8 @@ gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 h1:/saqWwm73dLmuzbNhe92F0QsZ/KiFND+esHco2v1hiY= gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 0e8e369ff5fc..14ea85cd0703 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -18,6 +18,7 @@ package file import ( + "fmt" "os" "path/filepath" "strconv" @@ -301,7 +302,7 @@ func (r *Rotator) openFile() error { r.file, err = os.OpenFile(r.filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, r.permissions) if err != nil { - return errors.Wrap(err, "failed to open new file") + return errors.Wrap(err, fmt.Sprintf("failed to open new file '%s'", r.filename)) } if r.redirectStderr { RedirectStandardError(r.file) diff --git a/libbeat/logp/config.go b/libbeat/logp/config.go index 55948aacc117..3c2f2cde381b 100644 --- a/libbeat/logp/config.go +++ b/libbeat/logp/config.go @@ -24,18 +24,18 @@ import ( // Config contains the configuration options for the logger. To create a Config // from a common.Config use logp/config.Build. type Config struct { - Beat string `config:",ignore"` // Name of the Beat (for default file name). - JSON bool `config:"json"` // Write logs as JSON. - Level Level `config:"level"` // Logging level (error, warning, info, debug). - Selectors []string `config:"selectors"` // Selectors for debug level logging. - ECSEnabled bool `config:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line + Beat string `config:",ignore"` // Name of the Beat (for default file name). + JSON bool `config:"json"` // Write logs as JSON. + Level Level `config:"level"` // Logging level (error, warning, info, debug). + Selectors []string `config:"selectors"` // Selectors for debug level logging. + ECSEnabled bool `config:"ecs" yaml:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line toObserver bool toIODiscard bool - ToStderr bool `config:"to_stderr"` - ToSyslog bool `config:"to_syslog"` - ToFiles bool `config:"to_files"` - ToEventLog bool `config:"to_eventlog"` + ToStderr bool `config:"to_stderr" yaml:"to_stderr"` + ToSyslog bool `config:"to_syslog" yaml:"to_syslog"` + ToFiles bool `config:"to_files" yaml:"to_files"` + ToEventLog bool `config:"to_eventlog" yaml:"to_eventlog"` Files FileConfig `config:"files"` @@ -46,14 +46,14 @@ type Config struct { // FileConfig contains the configuration options for the file output. type FileConfig struct { - Path string `config:"path"` - Name string `config:"name"` - MaxSize uint `config:"rotateeverybytes" validate:"min=1"` - MaxBackups uint `config:"keepfiles" validate:"max=1024"` + Path string `config:"path" yaml:"path"` + Name string `config:"name" yaml:"name"` + MaxSize uint `config:"rotateeverybytes" yaml:"rotateeverybytes" validate:"min=1"` + MaxBackups uint `config:"keepfiles" yaml:"keepfiles" validate:"max=1024"` Permissions uint32 `config:"permissions"` Interval time.Duration `config:"interval"` RotateOnStartup bool `config:"rotateonstartup"` - RedirectStderr bool `config:"redirect_stderr"` + RedirectStderr bool `config:"redirect_stderr" yaml:"redirect_stderr"` } const defaultLevel = InfoLevel diff --git a/libbeat/logp/level.go b/libbeat/logp/level.go index 6890192da03b..95e7c5c77947 100644 --- a/libbeat/logp/level.go +++ b/libbeat/logp/level.go @@ -81,6 +81,26 @@ func (l *Level) Unpack(str string) error { return errors.Errorf("invalid level '%v'", str) } +// MarshalYAML marshals level in a correct form +func (l Level) MarshalYAML() (interface{}, error) { + s, found := levelStrings[l] + if found { + return s, nil + } + + return nil, errors.Errorf("invalid level '%d'", l) +} + +// MarshalJSON marshals level in a correct form +func (l Level) MarshalJSON() ([]byte, error) { + s, found := levelStrings[l] + if found { + return []byte(s), nil + } + + return nil, errors.Errorf("invalid level '%d'", l) +} + func (l Level) zapLevel() zapcore.Level { z, found := zapLevels[l] if found { diff --git a/vendor/modules.txt b/vendor/modules.txt new file mode 100644 index 000000000000..7ee98f1384e8 --- /dev/null +++ b/vendor/modules.txt @@ -0,0 +1,1457 @@ +# 4d63.com/embedfiles v0.0.0-20190311033909-995e0740726f +4d63.com/embedfiles +# 4d63.com/tz v1.1.1-0.20191124060701-6d37baae851b +4d63.com/tz +# cloud.google.com/go v0.51.0 +cloud.google.com/go +cloud.google.com/go/compute/metadata +cloud.google.com/go/functions/metadata +cloud.google.com/go/iam +cloud.google.com/go/internal +cloud.google.com/go/internal/fields +cloud.google.com/go/internal/optional +cloud.google.com/go/internal/trace +cloud.google.com/go/internal/version +cloud.google.com/go/monitoring/apiv3 +# cloud.google.com/go/datastore v1.0.0 +cloud.google.com/go/datastore +cloud.google.com/go/datastore/internal/gaepb +# cloud.google.com/go/pubsub v1.0.1 +cloud.google.com/go/pubsub +cloud.google.com/go/pubsub/apiv1 +cloud.google.com/go/pubsub/internal/distribution +# cloud.google.com/go/storage v1.0.0 +cloud.google.com/go/storage +# code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee +code.cloudfoundry.org/go-diodes +# code.cloudfoundry.org/go-loggregator v7.4.0+incompatible +code.cloudfoundry.org/go-loggregator +code.cloudfoundry.org/go-loggregator/conversion +code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2 +# code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f +code.cloudfoundry.org/gofileutils/fileutils +# code.cloudfoundry.org/rfc5424 v0.0.0-20180905210152-236a6d29298a +code.cloudfoundry.org/rfc5424 +# github.com/Azure/azure-amqp-common-go/v3 v3.0.0 +github.com/Azure/azure-amqp-common-go/v3 +github.com/Azure/azure-amqp-common-go/v3/aad +github.com/Azure/azure-amqp-common-go/v3/auth +github.com/Azure/azure-amqp-common-go/v3/cbs +github.com/Azure/azure-amqp-common-go/v3/conn +github.com/Azure/azure-amqp-common-go/v3/internal +github.com/Azure/azure-amqp-common-go/v3/internal/tracing +github.com/Azure/azure-amqp-common-go/v3/rpc +github.com/Azure/azure-amqp-common-go/v3/sas +github.com/Azure/azure-amqp-common-go/v3/uuid +# github.com/Azure/azure-event-hubs-go/v3 v3.1.2 +github.com/Azure/azure-event-hubs-go/v3 +github.com/Azure/azure-event-hubs-go/v3/atom +github.com/Azure/azure-event-hubs-go/v3/eph +github.com/Azure/azure-event-hubs-go/v3/persist +github.com/Azure/azure-event-hubs-go/v3/storage +# github.com/Azure/azure-pipeline-go v0.2.1 +github.com/Azure/azure-pipeline-go/pipeline +# github.com/Azure/azure-sdk-for-go v37.1.0+incompatible +github.com/Azure/azure-sdk-for-go/services/eventhub/mgmt/2017-04-01/eventhub +github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights +github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources +github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage +github.com/Azure/azure-sdk-for-go/version +# github.com/Azure/azure-storage-blob-go v0.8.0 +github.com/Azure/azure-storage-blob-go/azblob +# github.com/Azure/go-amqp v0.12.6 +github.com/Azure/go-amqp +github.com/Azure/go-amqp/internal/testconn +# github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 +github.com/Azure/go-ansiterm +github.com/Azure/go-ansiterm/winterm +# github.com/Azure/go-autorest/autorest v0.9.4 +github.com/Azure/go-autorest/autorest +github.com/Azure/go-autorest/autorest/azure +# github.com/Azure/go-autorest/autorest/adal v0.8.1 +github.com/Azure/go-autorest/autorest/adal +# github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 +github.com/Azure/go-autorest/autorest/azure/auth +# github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 +github.com/Azure/go-autorest/autorest/azure/cli +# github.com/Azure/go-autorest/autorest/date v0.2.0 +github.com/Azure/go-autorest/autorest/date +# github.com/Azure/go-autorest/autorest/to v0.3.0 +github.com/Azure/go-autorest/autorest/to +# github.com/Azure/go-autorest/autorest/validation v0.2.0 +github.com/Azure/go-autorest/autorest/validation +# github.com/Azure/go-autorest/logger v0.1.0 +github.com/Azure/go-autorest/logger +# github.com/Azure/go-autorest/tracing v0.5.0 +github.com/Azure/go-autorest/tracing +# github.com/BurntSushi/toml v0.3.1 +github.com/BurntSushi/toml +# github.com/Masterminds/semver v1.4.2 +github.com/Masterminds/semver +# github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 +github.com/Microsoft/go-winio +github.com/Microsoft/go-winio/pkg/guid +# github.com/Microsoft/hcsshim v0.8.7 +github.com/Microsoft/hcsshim/osversion +# github.com/Shopify/sarama v0.0.0-00010101000000-000000000000 => github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e +github.com/Shopify/sarama +# github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 +github.com/StackExchange/wmi +# github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc +github.com/aerospike/aerospike-client-go +github.com/aerospike/aerospike-client-go/internal/lua +github.com/aerospike/aerospike-client-go/internal/lua/resources +github.com/aerospike/aerospike-client-go/logger +github.com/aerospike/aerospike-client-go/pkg/bcrypt +github.com/aerospike/aerospike-client-go/pkg/ripemd160 +github.com/aerospike/aerospike-client-go/types +github.com/aerospike/aerospike-client-go/types/atomic +github.com/aerospike/aerospike-client-go/types/particle_type +github.com/aerospike/aerospike-client-go/types/rand +github.com/aerospike/aerospike-client-go/utils/buffer +# github.com/akavel/rsrc v0.8.0 +github.com/akavel/rsrc/binutil +github.com/akavel/rsrc/coff +github.com/akavel/rsrc/ico +# github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 +github.com/andrewkroh/sys/windows/svc/eventlog +# github.com/antlr/antlr4 v0.0.0-20200225173536-225249fdaef5 +github.com/antlr/antlr4/runtime/Go/antlr +# github.com/armon/go-radix v1.0.0 +github.com/armon/go-radix +# github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 +github.com/armon/go-socks5 +# github.com/aws/aws-lambda-go v1.6.0 +github.com/aws/aws-lambda-go/events +github.com/aws/aws-lambda-go/lambda +github.com/aws/aws-lambda-go/lambda/messages +github.com/aws/aws-lambda-go/lambdacontext +# github.com/aws/aws-sdk-go-v2 v0.9.0 +github.com/aws/aws-sdk-go-v2/aws +github.com/aws/aws-sdk-go-v2/aws/arn +github.com/aws/aws-sdk-go-v2/aws/awserr +github.com/aws/aws-sdk-go-v2/aws/defaults +github.com/aws/aws-sdk-go-v2/aws/ec2metadata +github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds +github.com/aws/aws-sdk-go-v2/aws/endpointcreds +github.com/aws/aws-sdk-go-v2/aws/endpoints +github.com/aws/aws-sdk-go-v2/aws/external +github.com/aws/aws-sdk-go-v2/aws/signer/v4 +github.com/aws/aws-sdk-go-v2/aws/stscreds +github.com/aws/aws-sdk-go-v2/internal/awsutil +github.com/aws/aws-sdk-go-v2/internal/ini +github.com/aws/aws-sdk-go-v2/internal/sdk +github.com/aws/aws-sdk-go-v2/private/protocol +github.com/aws/aws-sdk-go-v2/private/protocol/ec2query +github.com/aws/aws-sdk-go-v2/private/protocol/json/jsonutil +github.com/aws/aws-sdk-go-v2/private/protocol/jsonrpc +github.com/aws/aws-sdk-go-v2/private/protocol/query +github.com/aws/aws-sdk-go-v2/private/protocol/query/queryutil +github.com/aws/aws-sdk-go-v2/private/protocol/rest +github.com/aws/aws-sdk-go-v2/private/protocol/restxml +github.com/aws/aws-sdk-go-v2/private/protocol/xml +github.com/aws/aws-sdk-go-v2/private/protocol/xml/xmlutil +github.com/aws/aws-sdk-go-v2/service/cloudformation +github.com/aws/aws-sdk-go-v2/service/cloudformation/cloudformationiface +github.com/aws/aws-sdk-go-v2/service/cloudwatch +github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface +github.com/aws/aws-sdk-go-v2/service/ec2 +github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface +github.com/aws/aws-sdk-go-v2/service/iam +github.com/aws/aws-sdk-go-v2/service/rds +github.com/aws/aws-sdk-go-v2/service/rds/rdsiface +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface +github.com/aws/aws-sdk-go-v2/service/s3 +github.com/aws/aws-sdk-go-v2/service/s3/s3iface +github.com/aws/aws-sdk-go-v2/service/sqs +github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface +github.com/aws/aws-sdk-go-v2/service/sts +# github.com/awslabs/goformation/v4 v4.1.0 +github.com/awslabs/goformation/v4/cloudformation +github.com/awslabs/goformation/v4/cloudformation/accessanalyzer +github.com/awslabs/goformation/v4/cloudformation/amazonmq +github.com/awslabs/goformation/v4/cloudformation/amplify +github.com/awslabs/goformation/v4/cloudformation/apigateway +github.com/awslabs/goformation/v4/cloudformation/apigatewayv2 +github.com/awslabs/goformation/v4/cloudformation/applicationautoscaling +github.com/awslabs/goformation/v4/cloudformation/appmesh +github.com/awslabs/goformation/v4/cloudformation/appstream +github.com/awslabs/goformation/v4/cloudformation/appsync +github.com/awslabs/goformation/v4/cloudformation/ask +github.com/awslabs/goformation/v4/cloudformation/athena +github.com/awslabs/goformation/v4/cloudformation/autoscaling +github.com/awslabs/goformation/v4/cloudformation/autoscalingplans +github.com/awslabs/goformation/v4/cloudformation/backup +github.com/awslabs/goformation/v4/cloudformation/batch +github.com/awslabs/goformation/v4/cloudformation/budgets +github.com/awslabs/goformation/v4/cloudformation/certificatemanager +github.com/awslabs/goformation/v4/cloudformation/cloud9 +github.com/awslabs/goformation/v4/cloudformation/cloudformation +github.com/awslabs/goformation/v4/cloudformation/cloudfront +github.com/awslabs/goformation/v4/cloudformation/cloudtrail +github.com/awslabs/goformation/v4/cloudformation/cloudwatch +github.com/awslabs/goformation/v4/cloudformation/codebuild +github.com/awslabs/goformation/v4/cloudformation/codecommit +github.com/awslabs/goformation/v4/cloudformation/codedeploy +github.com/awslabs/goformation/v4/cloudformation/codepipeline +github.com/awslabs/goformation/v4/cloudformation/codestar +github.com/awslabs/goformation/v4/cloudformation/codestarnotifications +github.com/awslabs/goformation/v4/cloudformation/cognito +github.com/awslabs/goformation/v4/cloudformation/config +github.com/awslabs/goformation/v4/cloudformation/datapipeline +github.com/awslabs/goformation/v4/cloudformation/dax +github.com/awslabs/goformation/v4/cloudformation/directoryservice +github.com/awslabs/goformation/v4/cloudformation/dlm +github.com/awslabs/goformation/v4/cloudformation/dms +github.com/awslabs/goformation/v4/cloudformation/docdb +github.com/awslabs/goformation/v4/cloudformation/dynamodb +github.com/awslabs/goformation/v4/cloudformation/ec2 +github.com/awslabs/goformation/v4/cloudformation/ecr +github.com/awslabs/goformation/v4/cloudformation/ecs +github.com/awslabs/goformation/v4/cloudformation/efs +github.com/awslabs/goformation/v4/cloudformation/eks +github.com/awslabs/goformation/v4/cloudformation/elasticache +github.com/awslabs/goformation/v4/cloudformation/elasticbeanstalk +github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancing +github.com/awslabs/goformation/v4/cloudformation/elasticloadbalancingv2 +github.com/awslabs/goformation/v4/cloudformation/elasticsearch +github.com/awslabs/goformation/v4/cloudformation/emr +github.com/awslabs/goformation/v4/cloudformation/events +github.com/awslabs/goformation/v4/cloudformation/eventschemas +github.com/awslabs/goformation/v4/cloudformation/fsx +github.com/awslabs/goformation/v4/cloudformation/gamelift +github.com/awslabs/goformation/v4/cloudformation/glue +github.com/awslabs/goformation/v4/cloudformation/greengrass +github.com/awslabs/goformation/v4/cloudformation/guardduty +github.com/awslabs/goformation/v4/cloudformation/iam +github.com/awslabs/goformation/v4/cloudformation/inspector +github.com/awslabs/goformation/v4/cloudformation/iot +github.com/awslabs/goformation/v4/cloudformation/iot1click +github.com/awslabs/goformation/v4/cloudformation/iotanalytics +github.com/awslabs/goformation/v4/cloudformation/iotevents +github.com/awslabs/goformation/v4/cloudformation/iotthingsgraph +github.com/awslabs/goformation/v4/cloudformation/kinesis +github.com/awslabs/goformation/v4/cloudformation/kinesisanalytics +github.com/awslabs/goformation/v4/cloudformation/kinesisanalyticsv2 +github.com/awslabs/goformation/v4/cloudformation/kinesisfirehose +github.com/awslabs/goformation/v4/cloudformation/kms +github.com/awslabs/goformation/v4/cloudformation/lakeformation +github.com/awslabs/goformation/v4/cloudformation/lambda +github.com/awslabs/goformation/v4/cloudformation/logs +github.com/awslabs/goformation/v4/cloudformation/managedblockchain +github.com/awslabs/goformation/v4/cloudformation/mediaconvert +github.com/awslabs/goformation/v4/cloudformation/medialive +github.com/awslabs/goformation/v4/cloudformation/mediastore +github.com/awslabs/goformation/v4/cloudformation/msk +github.com/awslabs/goformation/v4/cloudformation/neptune +github.com/awslabs/goformation/v4/cloudformation/opsworks +github.com/awslabs/goformation/v4/cloudformation/opsworkscm +github.com/awslabs/goformation/v4/cloudformation/pinpoint +github.com/awslabs/goformation/v4/cloudformation/pinpointemail +github.com/awslabs/goformation/v4/cloudformation/policies +github.com/awslabs/goformation/v4/cloudformation/qldb +github.com/awslabs/goformation/v4/cloudformation/ram +github.com/awslabs/goformation/v4/cloudformation/rds +github.com/awslabs/goformation/v4/cloudformation/redshift +github.com/awslabs/goformation/v4/cloudformation/robomaker +github.com/awslabs/goformation/v4/cloudformation/route53 +github.com/awslabs/goformation/v4/cloudformation/route53resolver +github.com/awslabs/goformation/v4/cloudformation/s3 +github.com/awslabs/goformation/v4/cloudformation/sagemaker +github.com/awslabs/goformation/v4/cloudformation/sdb +github.com/awslabs/goformation/v4/cloudformation/secretsmanager +github.com/awslabs/goformation/v4/cloudformation/securityhub +github.com/awslabs/goformation/v4/cloudformation/serverless +github.com/awslabs/goformation/v4/cloudformation/servicecatalog +github.com/awslabs/goformation/v4/cloudformation/servicediscovery +github.com/awslabs/goformation/v4/cloudformation/ses +github.com/awslabs/goformation/v4/cloudformation/sns +github.com/awslabs/goformation/v4/cloudformation/sqs +github.com/awslabs/goformation/v4/cloudformation/ssm +github.com/awslabs/goformation/v4/cloudformation/stepfunctions +github.com/awslabs/goformation/v4/cloudformation/tags +github.com/awslabs/goformation/v4/cloudformation/transfer +github.com/awslabs/goformation/v4/cloudformation/utils +github.com/awslabs/goformation/v4/cloudformation/waf +github.com/awslabs/goformation/v4/cloudformation/wafregional +github.com/awslabs/goformation/v4/cloudformation/wafv2 +github.com/awslabs/goformation/v4/cloudformation/workspaces +github.com/awslabs/goformation/v4/intrinsics +# github.com/beorn7/perks v1.0.1 +github.com/beorn7/perks/quantile +# github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 +github.com/blakesmith/ar +# github.com/bradleyfalzon/ghinstallation v1.1.0 +github.com/bradleyfalzon/ghinstallation +# github.com/bsm/sarama-cluster v2.1.14-0.20180625083203-7e67d87a6b3f+incompatible +github.com/bsm/sarama-cluster +# github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e +github.com/cavaliercoder/go-rpm +github.com/cavaliercoder/go-rpm/version +# github.com/cespare/xxhash/v2 v2.1.1 +github.com/cespare/xxhash/v2 +# github.com/cloudfoundry-community/go-cfclient v0.0.0-20190808214049-35bcce23fc5f +github.com/cloudfoundry-community/go-cfclient +# github.com/cloudfoundry/noaa v2.1.0+incompatible +github.com/cloudfoundry/noaa +github.com/cloudfoundry/noaa/consumer +github.com/cloudfoundry/noaa/consumer/internal +github.com/cloudfoundry/noaa/errors +# github.com/cloudfoundry/sonde-go v0.0.0-20171206171820-b33733203bb4 +github.com/cloudfoundry/sonde-go/events +# github.com/containerd/containerd v1.3.3 +github.com/containerd/containerd/errdefs +# github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 +github.com/containerd/continuity/fs +github.com/containerd/continuity/pathdriver +github.com/containerd/continuity/syscallx +github.com/containerd/continuity/sysx +# github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c +github.com/containerd/fifo +# github.com/coreos/bbolt v1.3.1-coreos.6.0.20180318001526-af9db2027c98 +github.com/coreos/bbolt +# github.com/coreos/go-systemd/v22 v22.0.0 +github.com/coreos/go-systemd/v22/activation +github.com/coreos/go-systemd/v22/dbus +github.com/coreos/go-systemd/v22/internal/dlopen +github.com/coreos/go-systemd/v22/sdjournal +# github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea +github.com/coreos/pkg/dlopen +# github.com/davecgh/go-spew v1.1.1 +github.com/davecgh/go-spew/spew +# github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 +github.com/davecgh/go-xdr/xdr2 +# github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e +github.com/denisenkom/go-mssqldb +github.com/denisenkom/go-mssqldb/internal/cp +github.com/denisenkom/go-mssqldb/internal/decimal +github.com/denisenkom/go-mssqldb/internal/querytext +# github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 +github.com/devigned/tab +# github.com/dgrijalva/jwt-go v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible +github.com/dgrijalva/jwt-go +# github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1 +github.com/digitalocean/go-libvirt +github.com/digitalocean/go-libvirt/internal/constants +github.com/digitalocean/go-libvirt/libvirttest +# github.com/dimchansky/utfbom v1.1.0 +github.com/dimchansky/utfbom +# github.com/dlclark/regexp2 v1.1.7-0.20171009020623-7632a260cbaf +github.com/dlclark/regexp2 +github.com/dlclark/regexp2/syntax +# github.com/docker/distribution v2.7.1+incompatible +github.com/docker/distribution/digestset +github.com/docker/distribution/reference +github.com/docker/distribution/registry/api/errcode +# github.com/docker/docker v1.4.2-0.20170802015333-8af4db6f002a => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 +github.com/docker/docker/api +github.com/docker/docker/api/types +github.com/docker/docker/api/types/backend +github.com/docker/docker/api/types/blkiodev +github.com/docker/docker/api/types/container +github.com/docker/docker/api/types/events +github.com/docker/docker/api/types/filters +github.com/docker/docker/api/types/image +github.com/docker/docker/api/types/mount +github.com/docker/docker/api/types/network +github.com/docker/docker/api/types/plugins/logdriver +github.com/docker/docker/api/types/registry +github.com/docker/docker/api/types/strslice +github.com/docker/docker/api/types/swarm +github.com/docker/docker/api/types/swarm/runtime +github.com/docker/docker/api/types/time +github.com/docker/docker/api/types/versions +github.com/docker/docker/api/types/volume +github.com/docker/docker/client +github.com/docker/docker/daemon/logger +github.com/docker/docker/errdefs +github.com/docker/docker/pkg/archive +github.com/docker/docker/pkg/fileutils +github.com/docker/docker/pkg/idtools +github.com/docker/docker/pkg/ioutils +github.com/docker/docker/pkg/jsonmessage +github.com/docker/docker/pkg/longpath +github.com/docker/docker/pkg/mount +github.com/docker/docker/pkg/plugingetter +github.com/docker/docker/pkg/plugins +github.com/docker/docker/pkg/plugins/transport +github.com/docker/docker/pkg/pools +github.com/docker/docker/pkg/progress +github.com/docker/docker/pkg/streamformatter +github.com/docker/docker/pkg/stringid +github.com/docker/docker/pkg/system +github.com/docker/docker/pkg/term +github.com/docker/docker/pkg/term/windows +# github.com/docker/go-connections v0.4.0 +github.com/docker/go-connections/nat +github.com/docker/go-connections/sockets +github.com/docker/go-connections/tlsconfig +# github.com/docker/go-metrics v0.0.1 +github.com/docker/go-metrics +# github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8 => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f +github.com/docker/go-plugins-helpers/sdk +# github.com/docker/go-units v0.4.0 +github.com/docker/go-units +# github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 +github.com/docker/spdystream +github.com/docker/spdystream/spdy +# github.com/dop251/goja v0.0.0-00010101000000-000000000000 => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 +github.com/dop251/goja +github.com/dop251/goja/ast +github.com/dop251/goja/file +github.com/dop251/goja/parser +github.com/dop251/goja/token +# github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 +github.com/dop251/goja_nodejs/require +github.com/dop251/goja_nodejs/util +# github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 +github.com/dustin/go-humanize +# github.com/eapache/go-resiliency v1.2.0 +github.com/eapache/go-resiliency/breaker +# github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 +github.com/eapache/go-xerial-snappy +# github.com/eapache/queue v1.1.0 +github.com/eapache/queue +# github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 +github.com/eclipse/paho.mqtt.golang +github.com/eclipse/paho.mqtt.golang/packets +# github.com/elastic/ecs v1.5.0 +github.com/elastic/ecs/code/go/ecs +# github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07 +github.com/elastic/elastic-agent-client/v7/pkg/client +github.com/elastic/elastic-agent-client/v7/pkg/proto +github.com/elastic/elastic-agent-client/v7/pkg/utils +# github.com/elastic/go-concert v0.0.2 +github.com/elastic/go-concert +github.com/elastic/go-concert/atomic +github.com/elastic/go-concert/chorus +github.com/elastic/go-concert/unison +# github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 +github.com/elastic/go-libaudit/v2 +github.com/elastic/go-libaudit/v2/aucoalesce +github.com/elastic/go-libaudit/v2/auparse +github.com/elastic/go-libaudit/v2/rule +github.com/elastic/go-libaudit/v2/rule/flags +github.com/elastic/go-libaudit/v2/sys +# github.com/elastic/go-licenser v0.2.1 +github.com/elastic/go-licenser +github.com/elastic/go-licenser/licensing +# github.com/elastic/go-lookslike v0.3.0 +github.com/elastic/go-lookslike +github.com/elastic/go-lookslike/internal/llreflect +github.com/elastic/go-lookslike/isdef +github.com/elastic/go-lookslike/llpath +github.com/elastic/go-lookslike/llresult +github.com/elastic/go-lookslike/testslike +github.com/elastic/go-lookslike/validator +# github.com/elastic/go-lumber v0.1.0 +github.com/elastic/go-lumber/client/v2 +github.com/elastic/go-lumber/lj +github.com/elastic/go-lumber/log +github.com/elastic/go-lumber/protocol/v2 +github.com/elastic/go-lumber/server/internal +github.com/elastic/go-lumber/server/v2 +# github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 +github.com/elastic/go-perf +# github.com/elastic/go-seccomp-bpf v1.1.0 +github.com/elastic/go-seccomp-bpf +github.com/elastic/go-seccomp-bpf/arch +# github.com/elastic/go-structform v0.0.7 +github.com/elastic/go-structform +github.com/elastic/go-structform/cborl +github.com/elastic/go-structform/gotype +github.com/elastic/go-structform/internal/unsafe +github.com/elastic/go-structform/json +github.com/elastic/go-structform/ubjson +github.com/elastic/go-structform/visitors +# github.com/elastic/go-sysinfo v1.3.0 +github.com/elastic/go-sysinfo +github.com/elastic/go-sysinfo/internal/registry +github.com/elastic/go-sysinfo/providers/darwin +github.com/elastic/go-sysinfo/providers/linux +github.com/elastic/go-sysinfo/providers/shared +github.com/elastic/go-sysinfo/providers/windows +github.com/elastic/go-sysinfo/types +# github.com/elastic/go-txfile v0.0.7 +github.com/elastic/go-txfile +github.com/elastic/go-txfile/dev-tools/lib/mage/xbuild +github.com/elastic/go-txfile/internal/cleanup +github.com/elastic/go-txfile/internal/invariant +github.com/elastic/go-txfile/internal/iter +github.com/elastic/go-txfile/internal/strbld +github.com/elastic/go-txfile/internal/tracelog +github.com/elastic/go-txfile/internal/vfs +github.com/elastic/go-txfile/internal/vfs/osfs +github.com/elastic/go-txfile/internal/vfs/osfs/osfstest +github.com/elastic/go-txfile/pq +github.com/elastic/go-txfile/txerr +github.com/elastic/go-txfile/txfiletest +# github.com/elastic/go-ucfg v0.8.3 +github.com/elastic/go-ucfg +github.com/elastic/go-ucfg/cfgutil +github.com/elastic/go-ucfg/flag +github.com/elastic/go-ucfg/json +github.com/elastic/go-ucfg/parse +github.com/elastic/go-ucfg/yaml +# github.com/elastic/go-windows v1.0.1 +github.com/elastic/go-windows +# github.com/elastic/gosigar v0.10.5 +github.com/elastic/gosigar +github.com/elastic/gosigar/cgroup +github.com/elastic/gosigar/sys +github.com/elastic/gosigar/sys/linux +github.com/elastic/gosigar/sys/windows +# github.com/evanphx/json-patch v4.2.0+incompatible +github.com/evanphx/json-patch +# github.com/fatih/color v1.5.0 +github.com/fatih/color +# github.com/fsnotify/fsevents v0.0.0-00010101000000-000000000000 => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 +github.com/fsnotify/fsevents +# github.com/fsnotify/fsnotify v1.4.7 => github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d +github.com/fsnotify/fsnotify +# github.com/garyburd/redigo v1.0.1-0.20160525165706-b8dc90050f24 +github.com/garyburd/redigo/internal +github.com/garyburd/redigo/redis +# github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 +github.com/go-ole/go-ole +github.com/go-ole/go-ole/oleutil +# github.com/go-sourcemap/sourcemap v2.1.2+incompatible +github.com/go-sourcemap/sourcemap +github.com/go-sourcemap/sourcemap/internal/base64vlq +# github.com/go-sql-driver/mysql v1.4.1 +github.com/go-sql-driver/mysql +# github.com/gocarina/gocsv v0.0.0-20170324095351-ffef3ffc77be +github.com/gocarina/gocsv +# github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e +github.com/godbus/dbus +# github.com/godbus/dbus/v5 v5.0.3 +github.com/godbus/dbus/v5 +# github.com/godror/godror v0.10.4 +github.com/godror/godror +# github.com/gofrs/flock v0.7.2-0.20190320160742-5135e617513b +github.com/gofrs/flock +# github.com/gofrs/uuid v3.3.0+incompatible +github.com/gofrs/uuid +# github.com/gogo/protobuf v1.3.1 +github.com/gogo/protobuf/gogoproto +github.com/gogo/protobuf/proto +github.com/gogo/protobuf/protoc-gen-gogo/descriptor +github.com/gogo/protobuf/sortkeys +github.com/gogo/protobuf/types +# github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe +github.com/golang-sql/civil +# github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 +github.com/golang/groupcache/lru +# github.com/golang/protobuf v1.4.2 +github.com/golang/protobuf/descriptor +github.com/golang/protobuf/internal/gengogrpc +github.com/golang/protobuf/jsonpb +github.com/golang/protobuf/proto +github.com/golang/protobuf/protoc-gen-go +github.com/golang/protobuf/protoc-gen-go/descriptor +github.com/golang/protobuf/ptypes +github.com/golang/protobuf/ptypes/any +github.com/golang/protobuf/ptypes/duration +github.com/golang/protobuf/ptypes/empty +github.com/golang/protobuf/ptypes/struct +github.com/golang/protobuf/ptypes/timestamp +github.com/golang/protobuf/ptypes/wrappers +# github.com/golang/snappy v0.0.1 +github.com/golang/snappy +# github.com/google/flatbuffers v1.7.2-0.20170925184458-7a6b2bf521e9 +github.com/google/flatbuffers/go +# github.com/google/go-cmp v0.4.0 +github.com/google/go-cmp/cmp +github.com/google/go-cmp/cmp/internal/diff +github.com/google/go-cmp/cmp/internal/flags +github.com/google/go-cmp/cmp/internal/function +github.com/google/go-cmp/cmp/internal/value +# github.com/google/go-github/v28 v28.1.1 +github.com/google/go-github/v28/github +# github.com/google/go-github/v29 v29.0.2 +github.com/google/go-github/v29/github +# github.com/google/go-querystring v1.0.0 +github.com/google/go-querystring/query +# github.com/google/gofuzz v1.1.0 +github.com/google/gofuzz +# github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34 => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 +github.com/google/gopacket +github.com/google/gopacket/afpacket +github.com/google/gopacket/layers +# github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f +github.com/google/uuid +# github.com/googleapis/gax-go/v2 v2.0.5 +github.com/googleapis/gax-go/v2 +# github.com/googleapis/gnostic v0.3.1-0.20190624222214-25d8b0b66985 +github.com/googleapis/gnostic/OpenAPIv2 +github.com/googleapis/gnostic/compiler +github.com/googleapis/gnostic/extensions +# github.com/gorhill/cronexpr v0.0.0-20161205141322-d520615e531a +github.com/gorhill/cronexpr +# github.com/gorilla/websocket v1.4.1 +github.com/gorilla/websocket +# github.com/grpc-ecosystem/grpc-gateway v1.13.0 +github.com/grpc-ecosystem/grpc-gateway/internal +github.com/grpc-ecosystem/grpc-gateway/runtime +github.com/grpc-ecosystem/grpc-gateway/utilities +# github.com/h2non/filetype v1.0.12 +github.com/h2non/filetype +github.com/h2non/filetype/matchers +github.com/h2non/filetype/matchers/isobmff +github.com/h2non/filetype/types +# github.com/hashicorp/errwrap v1.0.0 +github.com/hashicorp/errwrap +# github.com/hashicorp/go-cleanhttp v0.5.1 +github.com/hashicorp/go-cleanhttp +# github.com/hashicorp/go-multierror v1.1.0 +github.com/hashicorp/go-multierror +# github.com/hashicorp/go-retryablehttp v0.6.6 +github.com/hashicorp/go-retryablehttp +# github.com/hashicorp/go-uuid v1.0.2 +github.com/hashicorp/go-uuid +# github.com/hashicorp/go-version v1.0.0 +github.com/hashicorp/go-version +# github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d +github.com/hashicorp/golang-lru +github.com/hashicorp/golang-lru/simplelru +# github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01 +github.com/haya14busa/go-actions-toolkit/core +# github.com/imdario/mergo v0.3.6 +github.com/imdario/mergo +# github.com/inconshreveable/mousetrap v1.0.0 +github.com/inconshreveable/mousetrap +# github.com/insomniacslk/dhcp v0.0.0-20180716145214-633285ba52b2 => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 +github.com/insomniacslk/dhcp/dhcpv4 +github.com/insomniacslk/dhcp/iana +# github.com/jcmturner/gofork v1.0.0 +github.com/jcmturner/gofork/encoding/asn1 +github.com/jcmturner/gofork/x/crypto/pbkdf2 +# github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af +github.com/jmespath/go-jmespath +# github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 +github.com/jmoiron/sqlx +github.com/jmoiron/sqlx/reflectx +# github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 +github.com/joeshaw/multierror +# github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd +github.com/josephspurrier/goversioninfo +# github.com/jpillora/backoff v1.0.0 +github.com/jpillora/backoff +# github.com/json-iterator/go v1.1.8 +github.com/json-iterator/go +# github.com/jstemmer/go-junit-report v0.9.1 +github.com/jstemmer/go-junit-report +github.com/jstemmer/go-junit-report/formatter +github.com/jstemmer/go-junit-report/parser +# github.com/klauspost/compress v1.9.8 +github.com/klauspost/compress/flate +github.com/klauspost/compress/fse +github.com/klauspost/compress/huff0 +github.com/klauspost/compress/snappy +github.com/klauspost/compress/zlib +github.com/klauspost/compress/zstd +github.com/klauspost/compress/zstd/internal/xxhash +# github.com/konsorten/go-windows-terminal-sequences v1.0.2 +github.com/konsorten/go-windows-terminal-sequences +# github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01 +github.com/lib/pq +github.com/lib/pq/oid +github.com/lib/pq/scram +# github.com/magefile/mage v1.9.0 +github.com/magefile/mage +github.com/magefile/mage/internal +github.com/magefile/mage/mage +github.com/magefile/mage/mg +github.com/magefile/mage/parse +github.com/magefile/mage/sh +github.com/magefile/mage/target +# github.com/mailru/easyjson v0.7.1 +github.com/mailru/easyjson +github.com/mailru/easyjson/buffer +github.com/mailru/easyjson/jlexer +github.com/mailru/easyjson/jwriter +# github.com/mattn/go-colorable v0.0.8 +github.com/mattn/go-colorable +# github.com/mattn/go-ieproxy v0.0.0-20191113090002-7c0f6868bffe +github.com/mattn/go-ieproxy +# github.com/mattn/go-isatty v0.0.2 +github.com/mattn/go-isatty +# github.com/mattn/go-shellwords v1.0.7 +github.com/mattn/go-shellwords +# github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 +github.com/matttproud/golang_protobuf_extensions/pbutil +# github.com/miekg/dns v1.1.15 +github.com/miekg/dns +# github.com/mitchellh/go-homedir v1.1.0 +github.com/mitchellh/go-homedir +# github.com/mitchellh/gox v1.0.1 +github.com/mitchellh/gox +# github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 +github.com/mitchellh/hashstructure +# github.com/mitchellh/iochan v1.0.0 +github.com/mitchellh/iochan +# github.com/mitchellh/mapstructure v1.1.2 +github.com/mitchellh/mapstructure +# github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd +github.com/modern-go/concurrent +# github.com/modern-go/reflect2 v1.0.1 +github.com/modern-go/reflect2 +# github.com/morikuni/aec v1.0.0 +github.com/morikuni/aec +# github.com/oklog/ulid v1.3.1 +github.com/oklog/ulid +# github.com/opencontainers/go-digest v1.0.0-rc1.0.20190228220655-ac19fd6e7483 +github.com/opencontainers/go-digest +# github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 +github.com/opencontainers/image-spec/specs-go +github.com/opencontainers/image-spec/specs-go/v1 +# github.com/opencontainers/runc v1.0.0-rc9 +github.com/opencontainers/runc/libcontainer/system +github.com/opencontainers/runc/libcontainer/user +# github.com/pierrec/lz4 v2.4.1+incompatible +github.com/pierrec/lz4 +github.com/pierrec/lz4/internal/xxh32 +# github.com/pierrre/gotestcover v0.0.0-20160113212533-7b94f124d338 +github.com/pierrre/gotestcover +# github.com/pkg/errors v0.9.1 +github.com/pkg/errors +# github.com/pmezard/go-difflib v1.0.0 +github.com/pmezard/go-difflib/difflib +# github.com/prometheus/client_golang v1.1.1-0.20190913103102-20428fa0bffc +github.com/prometheus/client_golang/prometheus +github.com/prometheus/client_golang/prometheus/internal +github.com/prometheus/client_golang/prometheus/promhttp +# github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 +github.com/prometheus/client_model/go +# github.com/prometheus/common v0.7.0 +github.com/prometheus/common/expfmt +github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg +github.com/prometheus/common/model +# github.com/prometheus/procfs v0.0.11 +github.com/prometheus/procfs +github.com/prometheus/procfs/internal/fs +github.com/prometheus/procfs/internal/util +# github.com/prometheus/prometheus v2.5.0+incompatible +github.com/prometheus/prometheus/prompb +# github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 +github.com/rcrowley/go-metrics +# github.com/reviewdog/errorformat v0.0.0-20200109134752-8983be9bc7dd +github.com/reviewdog/errorformat +github.com/reviewdog/errorformat/fmts +# github.com/reviewdog/reviewdog v0.9.17 +github.com/reviewdog/reviewdog +github.com/reviewdog/reviewdog/cienv +github.com/reviewdog/reviewdog/cmd/reviewdog +github.com/reviewdog/reviewdog/commands +github.com/reviewdog/reviewdog/diff +github.com/reviewdog/reviewdog/doghouse +github.com/reviewdog/reviewdog/doghouse/client +github.com/reviewdog/reviewdog/doghouse/server +github.com/reviewdog/reviewdog/doghouse/server/storage +github.com/reviewdog/reviewdog/project +github.com/reviewdog/reviewdog/service/github +github.com/reviewdog/reviewdog/service/github/githubutils +github.com/reviewdog/reviewdog/service/gitlab +github.com/reviewdog/reviewdog/service/serviceutil +# github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e +github.com/samuel/go-parser/parser +# github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 +github.com/samuel/go-thrift/parser +# github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b +github.com/sanathkr/go-yaml +# github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522 +github.com/sanathkr/yaml +# github.com/santhosh-tekuri/jsonschema v1.2.4 +github.com/santhosh-tekuri/jsonschema +github.com/santhosh-tekuri/jsonschema/decoders +github.com/santhosh-tekuri/jsonschema/formats +github.com/santhosh-tekuri/jsonschema/loader +github.com/santhosh-tekuri/jsonschema/mediatypes +# github.com/shirou/gopsutil v2.19.11+incompatible +github.com/shirou/gopsutil/disk +github.com/shirou/gopsutil/internal/common +github.com/shirou/gopsutil/net +# github.com/sirupsen/logrus v1.4.2 +github.com/sirupsen/logrus +# github.com/spf13/cobra v0.0.3 +github.com/spf13/cobra +# github.com/spf13/pflag v1.0.5 +github.com/spf13/pflag +# github.com/stretchr/objx v0.2.0 +github.com/stretchr/objx +# github.com/stretchr/testify v1.5.1 +github.com/stretchr/testify/assert +github.com/stretchr/testify/mock +github.com/stretchr/testify/require +github.com/stretchr/testify/suite +# github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b +github.com/tsg/go-daemon +# github.com/tsg/gopacket v0.0.0-20190320122513-dd3d0e41124a +github.com/tsg/gopacket +github.com/tsg/gopacket/afpacket +github.com/tsg/gopacket/layers +github.com/tsg/gopacket/pcap +# github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 +github.com/urso/diag +github.com/urso/diag/ctxfmt +# github.com/urso/go-bin v0.0.0-20180220135811-781c575c9f0e +github.com/urso/go-bin +# github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a +github.com/urso/magetools/clitool +github.com/urso/magetools/ctrl +github.com/urso/magetools/fs +github.com/urso/magetools/gotool +github.com/urso/magetools/mgenv +# github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec +github.com/urso/sderr +# github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 +github.com/vmware/govmomi +github.com/vmware/govmomi/find +github.com/vmware/govmomi/list +github.com/vmware/govmomi/nfc +github.com/vmware/govmomi/object +github.com/vmware/govmomi/property +github.com/vmware/govmomi/session +github.com/vmware/govmomi/simulator +github.com/vmware/govmomi/simulator/esx +github.com/vmware/govmomi/simulator/vpx +github.com/vmware/govmomi/task +github.com/vmware/govmomi/view +github.com/vmware/govmomi/vim25 +github.com/vmware/govmomi/vim25/debug +github.com/vmware/govmomi/vim25/methods +github.com/vmware/govmomi/vim25/mo +github.com/vmware/govmomi/vim25/progress +github.com/vmware/govmomi/vim25/soap +github.com/vmware/govmomi/vim25/types +github.com/vmware/govmomi/vim25/xml +# github.com/xanzy/go-gitlab v0.22.3 +github.com/xanzy/go-gitlab +# github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c +github.com/xdg/scram +# github.com/xdg/stringprep v1.0.0 +github.com/xdg/stringprep +# github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 +github.com/yuin/gopher-lua +github.com/yuin/gopher-lua/ast +github.com/yuin/gopher-lua/parse +github.com/yuin/gopher-lua/pm +# go.elastic.co/apm v1.7.2 +go.elastic.co/apm +go.elastic.co/apm/apmconfig +go.elastic.co/apm/apmtest +go.elastic.co/apm/internal/apmcontext +go.elastic.co/apm/internal/apmhostutil +go.elastic.co/apm/internal/apmhttputil +go.elastic.co/apm/internal/apmlog +go.elastic.co/apm/internal/apmschema +go.elastic.co/apm/internal/apmstrings +go.elastic.co/apm/internal/apmversion +go.elastic.co/apm/internal/configutil +go.elastic.co/apm/internal/iochan +go.elastic.co/apm/internal/pkgerrorsutil +go.elastic.co/apm/internal/ringbuffer +go.elastic.co/apm/internal/wildcard +go.elastic.co/apm/model +go.elastic.co/apm/stacktrace +go.elastic.co/apm/transport +go.elastic.co/apm/transport/transporttest +# go.elastic.co/apm/module/apmelasticsearch v1.7.2 +go.elastic.co/apm/module/apmelasticsearch +# go.elastic.co/apm/module/apmhttp v1.7.2 +go.elastic.co/apm/module/apmhttp +# go.elastic.co/ecszap v0.2.0 +go.elastic.co/ecszap +go.elastic.co/ecszap/internal +# go.elastic.co/fastjson v1.0.0 +go.elastic.co/fastjson +# go.opencensus.io v0.22.2 +go.opencensus.io +go.opencensus.io/internal +go.opencensus.io/internal/tagencoding +go.opencensus.io/metric/metricdata +go.opencensus.io/metric/metricproducer +go.opencensus.io/plugin/ocgrpc +go.opencensus.io/plugin/ochttp +go.opencensus.io/plugin/ochttp/propagation/b3 +go.opencensus.io/resource +go.opencensus.io/stats +go.opencensus.io/stats/internal +go.opencensus.io/stats/view +go.opencensus.io/tag +go.opencensus.io/trace +go.opencensus.io/trace/internal +go.opencensus.io/trace/propagation +go.opencensus.io/trace/tracestate +# go.uber.org/atomic v1.5.0 +go.uber.org/atomic +# go.uber.org/multierr v1.3.0 +go.uber.org/multierr +# go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee +go.uber.org/tools/update-license +# go.uber.org/zap v1.14.0 +go.uber.org/zap +go.uber.org/zap/buffer +go.uber.org/zap/internal/bufferpool +go.uber.org/zap/internal/color +go.uber.org/zap/internal/exit +go.uber.org/zap/zapcore +go.uber.org/zap/zaptest/observer +# golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 +golang.org/x/crypto/blake2b +golang.org/x/crypto/blowfish +golang.org/x/crypto/cast5 +golang.org/x/crypto/chacha20 +golang.org/x/crypto/curve25519 +golang.org/x/crypto/ed25519 +golang.org/x/crypto/ed25519/internal/edwards25519 +golang.org/x/crypto/internal/subtle +golang.org/x/crypto/md4 +golang.org/x/crypto/openpgp +golang.org/x/crypto/openpgp/armor +golang.org/x/crypto/openpgp/elgamal +golang.org/x/crypto/openpgp/errors +golang.org/x/crypto/openpgp/packet +golang.org/x/crypto/openpgp/s2k +golang.org/x/crypto/pbkdf2 +golang.org/x/crypto/pkcs12 +golang.org/x/crypto/pkcs12/internal/rc2 +golang.org/x/crypto/poly1305 +golang.org/x/crypto/sha3 +golang.org/x/crypto/ssh +golang.org/x/crypto/ssh/internal/bcrypt_pbkdf +golang.org/x/crypto/ssh/terminal +# golang.org/x/exp v0.0.0-20191227195350-da58074b4299 +golang.org/x/exp/apidiff +golang.org/x/exp/cmd/apidiff +# golang.org/x/lint v0.0.0-20200130185559-910be7a94367 +golang.org/x/lint +golang.org/x/lint/golint +# golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee +golang.org/x/mod/module +golang.org/x/mod/semver +# golang.org/x/net v0.0.0-20200202094626-16171245cfb2 +golang.org/x/net/bpf +golang.org/x/net/context +golang.org/x/net/context/ctxhttp +golang.org/x/net/http/httpguts +golang.org/x/net/http/httpproxy +golang.org/x/net/http2 +golang.org/x/net/http2/hpack +golang.org/x/net/icmp +golang.org/x/net/idna +golang.org/x/net/internal/iana +golang.org/x/net/internal/socket +golang.org/x/net/internal/socks +golang.org/x/net/internal/timeseries +golang.org/x/net/ipv4 +golang.org/x/net/ipv6 +golang.org/x/net/netutil +golang.org/x/net/proxy +golang.org/x/net/publicsuffix +golang.org/x/net/trace +golang.org/x/net/websocket +# golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d +golang.org/x/oauth2 +golang.org/x/oauth2/clientcredentials +golang.org/x/oauth2/endpoints +golang.org/x/oauth2/google +golang.org/x/oauth2/internal +golang.org/x/oauth2/jws +golang.org/x/oauth2/jwt +# golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e +golang.org/x/sync/errgroup +golang.org/x/sync/semaphore +# golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e +golang.org/x/sys/cpu +golang.org/x/sys/unix +golang.org/x/sys/windows +golang.org/x/sys/windows/registry +golang.org/x/sys/windows/svc +golang.org/x/sys/windows/svc/debug +golang.org/x/sys/windows/svc/eventlog +# golang.org/x/text v0.3.2 +golang.org/x/text/cases +golang.org/x/text/collate +golang.org/x/text/encoding +golang.org/x/text/encoding/charmap +golang.org/x/text/encoding/htmlindex +golang.org/x/text/encoding/internal +golang.org/x/text/encoding/internal/identifier +golang.org/x/text/encoding/japanese +golang.org/x/text/encoding/korean +golang.org/x/text/encoding/simplifiedchinese +golang.org/x/text/encoding/traditionalchinese +golang.org/x/text/encoding/unicode +golang.org/x/text/internal +golang.org/x/text/internal/colltab +golang.org/x/text/internal/language +golang.org/x/text/internal/language/compact +golang.org/x/text/internal/tag +golang.org/x/text/internal/utf8internal +golang.org/x/text/language +golang.org/x/text/runes +golang.org/x/text/secure/bidirule +golang.org/x/text/transform +golang.org/x/text/unicode/bidi +golang.org/x/text/unicode/norm +# golang.org/x/time v0.0.0-20191024005414-555d28b269f0 +golang.org/x/time/rate +# golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2 +golang.org/x/tools/cmd/goimports +golang.org/x/tools/cmd/stringer +golang.org/x/tools/go/analysis +golang.org/x/tools/go/analysis/passes/inspect +golang.org/x/tools/go/ast/astutil +golang.org/x/tools/go/ast/inspector +golang.org/x/tools/go/buildutil +golang.org/x/tools/go/gcexportdata +golang.org/x/tools/go/internal/gcimporter +golang.org/x/tools/go/internal/packagesdriver +golang.org/x/tools/go/packages +golang.org/x/tools/go/types/objectpath +golang.org/x/tools/go/types/typeutil +golang.org/x/tools/go/vcs +golang.org/x/tools/internal/fastwalk +golang.org/x/tools/internal/gopathwalk +golang.org/x/tools/internal/imports +golang.org/x/tools/internal/packagesinternal +# golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 +golang.org/x/xerrors +golang.org/x/xerrors/internal +# google.golang.org/api v0.15.0 +google.golang.org/api/cloudfunctions/v1 +google.golang.org/api/compute/v1 +google.golang.org/api/googleapi +google.golang.org/api/googleapi/transport +google.golang.org/api/internal +google.golang.org/api/internal/gensupport +google.golang.org/api/internal/third_party/uritemplates +google.golang.org/api/iterator +google.golang.org/api/option +google.golang.org/api/storage/v1 +google.golang.org/api/support/bundler +google.golang.org/api/transport +google.golang.org/api/transport/grpc +google.golang.org/api/transport/http +google.golang.org/api/transport/http/internal/propagation +# google.golang.org/appengine v1.6.5 +google.golang.org/appengine +google.golang.org/appengine/cloudsql +google.golang.org/appengine/internal +google.golang.org/appengine/internal/app_identity +google.golang.org/appengine/internal/base +google.golang.org/appengine/internal/datastore +google.golang.org/appengine/internal/log +google.golang.org/appengine/internal/modules +google.golang.org/appengine/internal/remote_api +google.golang.org/appengine/internal/socket +google.golang.org/appengine/internal/urlfetch +google.golang.org/appengine/socket +google.golang.org/appengine/urlfetch +# google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb +google.golang.org/genproto/googleapis/api +google.golang.org/genproto/googleapis/api/annotations +google.golang.org/genproto/googleapis/api/distribution +google.golang.org/genproto/googleapis/api/httpbody +google.golang.org/genproto/googleapis/api/label +google.golang.org/genproto/googleapis/api/metric +google.golang.org/genproto/googleapis/api/monitoredres +google.golang.org/genproto/googleapis/datastore/v1 +google.golang.org/genproto/googleapis/iam/v1 +google.golang.org/genproto/googleapis/monitoring/v3 +google.golang.org/genproto/googleapis/pubsub/v1 +google.golang.org/genproto/googleapis/rpc/code +google.golang.org/genproto/googleapis/rpc/status +google.golang.org/genproto/googleapis/type/calendarperiod +google.golang.org/genproto/googleapis/type/expr +google.golang.org/genproto/googleapis/type/latlng +google.golang.org/genproto/protobuf/field_mask +# google.golang.org/grpc v1.29.1 +google.golang.org/grpc +google.golang.org/grpc/attributes +google.golang.org/grpc/backoff +google.golang.org/grpc/balancer +google.golang.org/grpc/balancer/base +google.golang.org/grpc/balancer/grpclb +google.golang.org/grpc/balancer/grpclb/grpc_lb_v1 +google.golang.org/grpc/balancer/roundrobin +google.golang.org/grpc/binarylog/grpc_binarylog_v1 +google.golang.org/grpc/codes +google.golang.org/grpc/connectivity +google.golang.org/grpc/credentials +google.golang.org/grpc/credentials/alts +google.golang.org/grpc/credentials/alts/internal +google.golang.org/grpc/credentials/alts/internal/authinfo +google.golang.org/grpc/credentials/alts/internal/conn +google.golang.org/grpc/credentials/alts/internal/handshaker +google.golang.org/grpc/credentials/alts/internal/handshaker/service +google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp +google.golang.org/grpc/credentials/google +google.golang.org/grpc/credentials/internal +google.golang.org/grpc/credentials/oauth +google.golang.org/grpc/encoding +google.golang.org/grpc/encoding/proto +google.golang.org/grpc/grpclog +google.golang.org/grpc/internal +google.golang.org/grpc/internal/backoff +google.golang.org/grpc/internal/balancerload +google.golang.org/grpc/internal/binarylog +google.golang.org/grpc/internal/buffer +google.golang.org/grpc/internal/channelz +google.golang.org/grpc/internal/envconfig +google.golang.org/grpc/internal/grpclog +google.golang.org/grpc/internal/grpcrand +google.golang.org/grpc/internal/grpcsync +google.golang.org/grpc/internal/grpcutil +google.golang.org/grpc/internal/resolver/dns +google.golang.org/grpc/internal/resolver/passthrough +google.golang.org/grpc/internal/status +google.golang.org/grpc/internal/syscall +google.golang.org/grpc/internal/transport +google.golang.org/grpc/keepalive +google.golang.org/grpc/metadata +google.golang.org/grpc/naming +google.golang.org/grpc/peer +google.golang.org/grpc/resolver +google.golang.org/grpc/serviceconfig +google.golang.org/grpc/stats +google.golang.org/grpc/status +google.golang.org/grpc/tap +# google.golang.org/protobuf v1.23.0 +google.golang.org/protobuf/cmd/protoc-gen-go/internal_gengo +google.golang.org/protobuf/compiler/protogen +google.golang.org/protobuf/encoding/protojson +google.golang.org/protobuf/encoding/prototext +google.golang.org/protobuf/encoding/protowire +google.golang.org/protobuf/internal/descfmt +google.golang.org/protobuf/internal/descopts +google.golang.org/protobuf/internal/detectknown +google.golang.org/protobuf/internal/detrand +google.golang.org/protobuf/internal/encoding/defval +google.golang.org/protobuf/internal/encoding/json +google.golang.org/protobuf/internal/encoding/messageset +google.golang.org/protobuf/internal/encoding/tag +google.golang.org/protobuf/internal/encoding/text +google.golang.org/protobuf/internal/errors +google.golang.org/protobuf/internal/fieldnum +google.golang.org/protobuf/internal/fieldsort +google.golang.org/protobuf/internal/filedesc +google.golang.org/protobuf/internal/filetype +google.golang.org/protobuf/internal/flags +google.golang.org/protobuf/internal/genname +google.golang.org/protobuf/internal/impl +google.golang.org/protobuf/internal/mapsort +google.golang.org/protobuf/internal/pragma +google.golang.org/protobuf/internal/set +google.golang.org/protobuf/internal/strs +google.golang.org/protobuf/internal/version +google.golang.org/protobuf/proto +google.golang.org/protobuf/reflect/protodesc +google.golang.org/protobuf/reflect/protoreflect +google.golang.org/protobuf/reflect/protoregistry +google.golang.org/protobuf/runtime/protoiface +google.golang.org/protobuf/runtime/protoimpl +google.golang.org/protobuf/types/descriptorpb +google.golang.org/protobuf/types/known/anypb +google.golang.org/protobuf/types/known/durationpb +google.golang.org/protobuf/types/known/emptypb +google.golang.org/protobuf/types/known/structpb +google.golang.org/protobuf/types/known/timestamppb +google.golang.org/protobuf/types/known/wrapperspb +google.golang.org/protobuf/types/pluginpb +# gopkg.in/inf.v0 v0.9.1 +gopkg.in/inf.v0 +# gopkg.in/jcmturner/aescts.v1 v1.0.1 +gopkg.in/jcmturner/aescts.v1 +# gopkg.in/jcmturner/dnsutils.v1 v1.0.1 +gopkg.in/jcmturner/dnsutils.v1 +# gopkg.in/jcmturner/goidentity.v3 v3.0.0 +gopkg.in/jcmturner/goidentity.v3 +# gopkg.in/jcmturner/gokrb5.v7 v7.5.0 +gopkg.in/jcmturner/gokrb5.v7/asn1tools +gopkg.in/jcmturner/gokrb5.v7/client +gopkg.in/jcmturner/gokrb5.v7/config +gopkg.in/jcmturner/gokrb5.v7/credentials +gopkg.in/jcmturner/gokrb5.v7/crypto +gopkg.in/jcmturner/gokrb5.v7/crypto/common +gopkg.in/jcmturner/gokrb5.v7/crypto/etype +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3961 +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3962 +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc4757 +gopkg.in/jcmturner/gokrb5.v7/crypto/rfc8009 +gopkg.in/jcmturner/gokrb5.v7/gssapi +gopkg.in/jcmturner/gokrb5.v7/iana +gopkg.in/jcmturner/gokrb5.v7/iana/addrtype +gopkg.in/jcmturner/gokrb5.v7/iana/adtype +gopkg.in/jcmturner/gokrb5.v7/iana/asnAppTag +gopkg.in/jcmturner/gokrb5.v7/iana/chksumtype +gopkg.in/jcmturner/gokrb5.v7/iana/errorcode +gopkg.in/jcmturner/gokrb5.v7/iana/etypeID +gopkg.in/jcmturner/gokrb5.v7/iana/flags +gopkg.in/jcmturner/gokrb5.v7/iana/keyusage +gopkg.in/jcmturner/gokrb5.v7/iana/msgtype +gopkg.in/jcmturner/gokrb5.v7/iana/nametype +gopkg.in/jcmturner/gokrb5.v7/iana/patype +gopkg.in/jcmturner/gokrb5.v7/kadmin +gopkg.in/jcmturner/gokrb5.v7/keytab +gopkg.in/jcmturner/gokrb5.v7/krberror +gopkg.in/jcmturner/gokrb5.v7/messages +gopkg.in/jcmturner/gokrb5.v7/pac +gopkg.in/jcmturner/gokrb5.v7/service +gopkg.in/jcmturner/gokrb5.v7/spnego +gopkg.in/jcmturner/gokrb5.v7/types +# gopkg.in/jcmturner/rpc.v1 v1.1.0 +gopkg.in/jcmturner/rpc.v1/mstypes +gopkg.in/jcmturner/rpc.v1/ndr +# gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 +gopkg.in/mgo.v2 +gopkg.in/mgo.v2/bson +gopkg.in/mgo.v2/internal/json +gopkg.in/mgo.v2/internal/sasl +gopkg.in/mgo.v2/internal/scram +# gopkg.in/yaml.v2 v2.3.0 +gopkg.in/yaml.v2 +# honnef.co/go/tools v0.0.1-2019.2.3 +honnef.co/go/tools/arg +honnef.co/go/tools/cmd/staticcheck +honnef.co/go/tools/config +honnef.co/go/tools/deprecated +honnef.co/go/tools/facts +honnef.co/go/tools/functions +honnef.co/go/tools/go/types/typeutil +honnef.co/go/tools/internal/cache +honnef.co/go/tools/internal/passes/buildssa +honnef.co/go/tools/internal/renameio +honnef.co/go/tools/internal/sharedcheck +honnef.co/go/tools/lint +honnef.co/go/tools/lint/lintdsl +honnef.co/go/tools/lint/lintutil +honnef.co/go/tools/lint/lintutil/format +honnef.co/go/tools/loader +honnef.co/go/tools/printf +honnef.co/go/tools/simple +honnef.co/go/tools/ssa +honnef.co/go/tools/ssautil +honnef.co/go/tools/staticcheck +honnef.co/go/tools/staticcheck/vrp +honnef.co/go/tools/stylecheck +honnef.co/go/tools/unused +honnef.co/go/tools/version +# howett.net/plist v0.0.0-20181124034731-591f970eefbb +howett.net/plist +# k8s.io/api v0.18.3 +k8s.io/api/admissionregistration/v1 +k8s.io/api/admissionregistration/v1beta1 +k8s.io/api/apps/v1 +k8s.io/api/apps/v1beta1 +k8s.io/api/apps/v1beta2 +k8s.io/api/auditregistration/v1alpha1 +k8s.io/api/authentication/v1 +k8s.io/api/authentication/v1beta1 +k8s.io/api/authorization/v1 +k8s.io/api/authorization/v1beta1 +k8s.io/api/autoscaling/v1 +k8s.io/api/autoscaling/v2beta1 +k8s.io/api/autoscaling/v2beta2 +k8s.io/api/batch/v1 +k8s.io/api/batch/v1beta1 +k8s.io/api/batch/v2alpha1 +k8s.io/api/certificates/v1beta1 +k8s.io/api/coordination/v1 +k8s.io/api/coordination/v1beta1 +k8s.io/api/core/v1 +k8s.io/api/discovery/v1alpha1 +k8s.io/api/discovery/v1beta1 +k8s.io/api/events/v1beta1 +k8s.io/api/extensions/v1beta1 +k8s.io/api/flowcontrol/v1alpha1 +k8s.io/api/networking/v1 +k8s.io/api/networking/v1beta1 +k8s.io/api/node/v1alpha1 +k8s.io/api/node/v1beta1 +k8s.io/api/policy/v1beta1 +k8s.io/api/rbac/v1 +k8s.io/api/rbac/v1alpha1 +k8s.io/api/rbac/v1beta1 +k8s.io/api/scheduling/v1 +k8s.io/api/scheduling/v1alpha1 +k8s.io/api/scheduling/v1beta1 +k8s.io/api/settings/v1alpha1 +k8s.io/api/storage/v1 +k8s.io/api/storage/v1alpha1 +k8s.io/api/storage/v1beta1 +# k8s.io/apimachinery v0.18.3 +k8s.io/apimachinery/pkg/api/errors +k8s.io/apimachinery/pkg/api/meta +k8s.io/apimachinery/pkg/api/resource +k8s.io/apimachinery/pkg/apis/meta/internalversion +k8s.io/apimachinery/pkg/apis/meta/v1 +k8s.io/apimachinery/pkg/apis/meta/v1/unstructured +k8s.io/apimachinery/pkg/apis/meta/v1beta1 +k8s.io/apimachinery/pkg/conversion +k8s.io/apimachinery/pkg/conversion/queryparams +k8s.io/apimachinery/pkg/fields +k8s.io/apimachinery/pkg/labels +k8s.io/apimachinery/pkg/runtime +k8s.io/apimachinery/pkg/runtime/schema +k8s.io/apimachinery/pkg/runtime/serializer +k8s.io/apimachinery/pkg/runtime/serializer/json +k8s.io/apimachinery/pkg/runtime/serializer/protobuf +k8s.io/apimachinery/pkg/runtime/serializer/recognizer +k8s.io/apimachinery/pkg/runtime/serializer/streaming +k8s.io/apimachinery/pkg/runtime/serializer/versioning +k8s.io/apimachinery/pkg/selection +k8s.io/apimachinery/pkg/types +k8s.io/apimachinery/pkg/util/cache +k8s.io/apimachinery/pkg/util/clock +k8s.io/apimachinery/pkg/util/diff +k8s.io/apimachinery/pkg/util/errors +k8s.io/apimachinery/pkg/util/framer +k8s.io/apimachinery/pkg/util/httpstream +k8s.io/apimachinery/pkg/util/httpstream/spdy +k8s.io/apimachinery/pkg/util/intstr +k8s.io/apimachinery/pkg/util/json +k8s.io/apimachinery/pkg/util/mergepatch +k8s.io/apimachinery/pkg/util/naming +k8s.io/apimachinery/pkg/util/net +k8s.io/apimachinery/pkg/util/runtime +k8s.io/apimachinery/pkg/util/sets +k8s.io/apimachinery/pkg/util/strategicpatch +k8s.io/apimachinery/pkg/util/validation +k8s.io/apimachinery/pkg/util/validation/field +k8s.io/apimachinery/pkg/util/wait +k8s.io/apimachinery/pkg/util/yaml +k8s.io/apimachinery/pkg/version +k8s.io/apimachinery/pkg/watch +k8s.io/apimachinery/third_party/forked/golang/json +k8s.io/apimachinery/third_party/forked/golang/netutil +k8s.io/apimachinery/third_party/forked/golang/reflect +# k8s.io/client-go v0.18.3 +k8s.io/client-go/discovery +k8s.io/client-go/discovery/fake +k8s.io/client-go/kubernetes +k8s.io/client-go/kubernetes/fake +k8s.io/client-go/kubernetes/scheme +k8s.io/client-go/kubernetes/typed/admissionregistration/v1 +k8s.io/client-go/kubernetes/typed/admissionregistration/v1/fake +k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 +k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1/fake +k8s.io/client-go/kubernetes/typed/apps/v1 +k8s.io/client-go/kubernetes/typed/apps/v1/fake +k8s.io/client-go/kubernetes/typed/apps/v1beta1 +k8s.io/client-go/kubernetes/typed/apps/v1beta1/fake +k8s.io/client-go/kubernetes/typed/apps/v1beta2 +k8s.io/client-go/kubernetes/typed/apps/v1beta2/fake +k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1 +k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/authentication/v1 +k8s.io/client-go/kubernetes/typed/authentication/v1/fake +k8s.io/client-go/kubernetes/typed/authentication/v1beta1 +k8s.io/client-go/kubernetes/typed/authentication/v1beta1/fake +k8s.io/client-go/kubernetes/typed/authorization/v1 +k8s.io/client-go/kubernetes/typed/authorization/v1/fake +k8s.io/client-go/kubernetes/typed/authorization/v1beta1 +k8s.io/client-go/kubernetes/typed/authorization/v1beta1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v1 +k8s.io/client-go/kubernetes/typed/autoscaling/v1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1 +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1/fake +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2 +k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2/fake +k8s.io/client-go/kubernetes/typed/batch/v1 +k8s.io/client-go/kubernetes/typed/batch/v1/fake +k8s.io/client-go/kubernetes/typed/batch/v1beta1 +k8s.io/client-go/kubernetes/typed/batch/v1beta1/fake +k8s.io/client-go/kubernetes/typed/batch/v2alpha1 +k8s.io/client-go/kubernetes/typed/batch/v2alpha1/fake +k8s.io/client-go/kubernetes/typed/certificates/v1beta1 +k8s.io/client-go/kubernetes/typed/certificates/v1beta1/fake +k8s.io/client-go/kubernetes/typed/coordination/v1 +k8s.io/client-go/kubernetes/typed/coordination/v1/fake +k8s.io/client-go/kubernetes/typed/coordination/v1beta1 +k8s.io/client-go/kubernetes/typed/coordination/v1beta1/fake +k8s.io/client-go/kubernetes/typed/core/v1 +k8s.io/client-go/kubernetes/typed/core/v1/fake +k8s.io/client-go/kubernetes/typed/discovery/v1alpha1 +k8s.io/client-go/kubernetes/typed/discovery/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/discovery/v1beta1 +k8s.io/client-go/kubernetes/typed/discovery/v1beta1/fake +k8s.io/client-go/kubernetes/typed/events/v1beta1 +k8s.io/client-go/kubernetes/typed/events/v1beta1/fake +k8s.io/client-go/kubernetes/typed/extensions/v1beta1 +k8s.io/client-go/kubernetes/typed/extensions/v1beta1/fake +k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1 +k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/networking/v1 +k8s.io/client-go/kubernetes/typed/networking/v1/fake +k8s.io/client-go/kubernetes/typed/networking/v1beta1 +k8s.io/client-go/kubernetes/typed/networking/v1beta1/fake +k8s.io/client-go/kubernetes/typed/node/v1alpha1 +k8s.io/client-go/kubernetes/typed/node/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/node/v1beta1 +k8s.io/client-go/kubernetes/typed/node/v1beta1/fake +k8s.io/client-go/kubernetes/typed/policy/v1beta1 +k8s.io/client-go/kubernetes/typed/policy/v1beta1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1 +k8s.io/client-go/kubernetes/typed/rbac/v1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1alpha1 +k8s.io/client-go/kubernetes/typed/rbac/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/rbac/v1beta1 +k8s.io/client-go/kubernetes/typed/rbac/v1beta1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1 +k8s.io/client-go/kubernetes/typed/scheduling/v1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1 +k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/scheduling/v1beta1 +k8s.io/client-go/kubernetes/typed/scheduling/v1beta1/fake +k8s.io/client-go/kubernetes/typed/settings/v1alpha1 +k8s.io/client-go/kubernetes/typed/settings/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/storage/v1 +k8s.io/client-go/kubernetes/typed/storage/v1/fake +k8s.io/client-go/kubernetes/typed/storage/v1alpha1 +k8s.io/client-go/kubernetes/typed/storage/v1alpha1/fake +k8s.io/client-go/kubernetes/typed/storage/v1beta1 +k8s.io/client-go/kubernetes/typed/storage/v1beta1/fake +k8s.io/client-go/pkg/apis/clientauthentication +k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 +k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 +k8s.io/client-go/pkg/version +k8s.io/client-go/plugin/pkg/client/auth/exec +k8s.io/client-go/rest +k8s.io/client-go/rest/watch +k8s.io/client-go/testing +k8s.io/client-go/tools/auth +k8s.io/client-go/tools/cache +k8s.io/client-go/tools/clientcmd +k8s.io/client-go/tools/clientcmd/api +k8s.io/client-go/tools/clientcmd/api/latest +k8s.io/client-go/tools/clientcmd/api/v1 +k8s.io/client-go/tools/metrics +k8s.io/client-go/tools/pager +k8s.io/client-go/tools/portforward +k8s.io/client-go/tools/reference +k8s.io/client-go/tools/watch +k8s.io/client-go/transport +k8s.io/client-go/transport/spdy +k8s.io/client-go/util/cert +k8s.io/client-go/util/connrotation +k8s.io/client-go/util/flowcontrol +k8s.io/client-go/util/homedir +k8s.io/client-go/util/keyutil +k8s.io/client-go/util/workqueue +# k8s.io/klog v1.0.0 +k8s.io/klog +# k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 +k8s.io/kube-openapi/pkg/util/proto +# k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89 +k8s.io/utils/buffer +k8s.io/utils/integer +k8s.io/utils/trace +# sigs.k8s.io/structured-merge-diff/v3 v3.0.0 +sigs.k8s.io/structured-merge-diff/v3/value +# sigs.k8s.io/yaml v1.2.0 +sigs.k8s.io/yaml diff --git a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl index 03dcb39c7308..81dbde654195 100644 --- a/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl @@ -124,6 +124,75 @@ inputs: # # Default is false # exponential: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl index 6fd2e4f4e247..59ea36c70bc4 100644 --- a/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl @@ -124,6 +124,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl index 962484e11f20..212d6944b6ef 100644 --- a/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl +++ b/x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl @@ -124,6 +124,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/_meta/elastic-agent.yml b/x-pack/elastic-agent/_meta/elastic-agent.yml index b7db83d12bda..e8d47c9ea75e 100644 --- a/x-pack/elastic-agent/_meta/elastic-agent.yml +++ b/x-pack/elastic-agent/_meta/elastic-agent.yml @@ -119,6 +119,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml index 47a22e9d1b4a..5673f4e724e1 100644 --- a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml +++ b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml @@ -30,9 +30,80 @@ settings.monitoring: # enables metrics monitoring metrics: true + +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, {{.BeatName | title}} periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/{{.BeatName}} + + # The name of the files where the logs are written to. + #name: {{.BeatName}} + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false + inputs: - type: logfile diff --git a/x-pack/elastic-agent/elastic-agent.docker.yml b/x-pack/elastic-agent/elastic-agent.docker.yml index 962484e11f20..8c51ecd61206 100644 --- a/x-pack/elastic-agent/elastic-agent.docker.yml +++ b/x-pack/elastic-agent/elastic-agent.docker.yml @@ -124,6 +124,75 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, Elastic-Agent periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/elastic-agent + + # The name of the files where the logs are written to. + #name: elastic-agent + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/elastic-agent.reference.yml b/x-pack/elastic-agent/elastic-agent.reference.yml index 5530d0b455ba..d5a5966dfb48 100644 --- a/x-pack/elastic-agent/elastic-agent.reference.yml +++ b/x-pack/elastic-agent/elastic-agent.reference.yml @@ -130,7 +130,76 @@ inputs: # # enables metrics monitoring # metrics: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, Elastic-Agent periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/elastic-agent + + # The name of the files where the logs are written to. + #name: elastic-agent + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/elastic-agent.yml b/x-pack/elastic-agent/elastic-agent.yml index a24a07a982dc..d1c7c6220be5 100644 --- a/x-pack/elastic-agent/elastic-agent.yml +++ b/x-pack/elastic-agent/elastic-agent.yml @@ -130,7 +130,76 @@ inputs: # # Default is false # exponential: false +# Logging + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + # Sets log level. The default log level is info. # Available log levels are: error, warning, info, debug -#logging.level: trace +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +logging.to_stderr: true + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, Elastic-Agent periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +#logging.to_files: true +#logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/elastic-agent + + # The name of the files where the logs are written to. + #name: elastic-agent + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false diff --git a/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go b/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go index 998b90cdfe3d..6aa8eda8caae 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_dispatcher.go @@ -33,7 +33,7 @@ type actionDispatcher struct { func newActionDispatcher(ctx context.Context, log *logger.Logger, def actionHandler) (*actionDispatcher, error) { var err error if log == nil { - log, err = logger.New() + log, err = logger.New("action_dispatcher") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go index 02e62efd1a4e..4205deda8b64 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go @@ -19,7 +19,7 @@ import ( ) func TestActionStore(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("action_store") withFile := func(fn func(t *testing.T, file string)) func(*testing.T) { return func(t *testing.T) { dir, err := ioutil.TempDir("", "action-store") diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go index 4c061abd8357..747e5ba5cdca 100644 --- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go @@ -43,7 +43,7 @@ func (m *mockStore) Save(in io.Reader) error { } func TestEnroll(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("tst") t.Run("fail to save is propagated", withTLSServer( func(t *testing.T) *http.ServeMux { diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go index 64d3f51f3e34..624824e14ec1 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go @@ -23,7 +23,7 @@ func TestAcker(t *testing.T) { Events []fleetapi.AckEvent `json:"events"` } - log, _ := logger.New() + log, _ := logger.New("fleet_acker") client := newTestingClient() agentInfo := &testAgentInfo{} acker, err := newActionAcker(log, agentInfo, client) diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go index 4bc48b8fd98e..5e1fbe9418e4 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go @@ -111,7 +111,7 @@ func withGateway(agentInfo agentInfo, settings *fleetGatewaySettings, fn withGat client := newTestingClient() dispatcher := newTestingDispatcher() - log, _ := logger.New() + log, _ := logger.New("fleet_gateway") rep := getReporter(agentInfo, log, t) ctx, cancel := context.WithCancel(context.Background()) @@ -243,7 +243,7 @@ func TestFleetGateway(t *testing.T) { dispatcher := newTestingDispatcher() ctx, cancel := context.WithCancel(context.Background()) - log, _ := logger.New() + log, _ := logger.New("tst") gateway, err := newFleetGatewayWithScheduler( ctx, log, @@ -328,7 +328,7 @@ func TestFleetGateway(t *testing.T) { dispatcher := newTestingDispatcher() ctx, cancel := context.WithCancel(context.Background()) - log, _ := logger.New() + log, _ := logger.New("tst") gateway, err := newFleetGatewayWithScheduler( ctx, log, diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go index 400052f66f4d..b95c259e7c78 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go @@ -29,7 +29,7 @@ func (m *mockEmitter) Emitter(policy *config.Config) error { } func TestPolicyChange(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") ack := newNoopAcker() t.Run("Receive a config change and successfully emits a raw configuration", func(t *testing.T) { @@ -68,7 +68,7 @@ func TestPolicyChange(t *testing.T) { } func TestPolicyAcked(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") t.Run("Config change should not ACK on error", func(t *testing.T) { tacker := &testAcker{} diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go index 5c7ee622a751..8c48384e2596 100644 --- a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go @@ -7,11 +7,7 @@ package application import ( "fmt" - "github.com/urso/ecslog" - "github.com/urso/ecslog/backend" - "github.com/urso/ecslog/backend/appender" - "github.com/urso/ecslog/backend/layout" - + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" @@ -202,9 +198,5 @@ func (r *inmemRouter) Dispatch(id string, grpProg map[routingKey][]program.Progr } func newErrorLogger() (*logger.Logger, error) { - backend, err := appender.Console(backend.Error, layout.Text(true)) - if err != nil { - return nil, err - } - return ecslog.New(backend), nil + return logger.NewWithLogpLevel("", logp.ErrorLevel) } diff --git a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go index cd941e64f836..aa48e9bd9499 100644 --- a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go @@ -24,7 +24,7 @@ func TestLazyAcker(t *testing.T) { Events []fleetapi.AckEvent `json:"events"` } - log, _ := logger.New() + log, _ := logger.New("") client := newTestingClient() agentInfo := &testAgentInfo{} acker, err := newActionAcker(log, agentInfo, client) diff --git a/x-pack/elastic-agent/pkg/agent/application/local_mode.go b/x-pack/elastic-agent/pkg/agent/application/local_mode.go index dc7a81e198af..b6c8a462018a 100644 --- a/x-pack/elastic-agent/pkg/agent/application/local_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/local_mode.go @@ -58,7 +58,7 @@ func newLocal( ) (*Local, error) { var err error if log == nil { - log, err = logger.NewFromConfig(rawConfig) + log, err = logger.NewFromConfig("", rawConfig) if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go index b5303d0dc605..8113a58903d8 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go @@ -25,7 +25,7 @@ func TestManagedModeRouting(t *testing.T) { return m, nil } - log, _ := logger.New() + log, _ := logger.New("") router, _ := newRouter(log, streamFn) emit := emitter(log, router, &configModifiers{Decorators: []decoratorFunc{injectMonitoring}, Filters: []filterFunc{filters.ConstraintFilter}}) diff --git a/x-pack/elastic-agent/pkg/agent/application/router.go b/x-pack/elastic-agent/pkg/agent/application/router.go index b48156d7d48c..16f529bfa2d5 100644 --- a/x-pack/elastic-agent/pkg/agent/application/router.go +++ b/x-pack/elastic-agent/pkg/agent/application/router.go @@ -34,7 +34,7 @@ type router struct { func newRouter(log *logger.Logger, factory streamFunc) (*router, error) { var err error if log == nil { - log, err = logger.New() + log, err = logger.New("router") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index abc5efb3b90a..8d01592c8fce 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -67,7 +67,7 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args } } - logger, err := logger.NewFromConfig(config) + logger, err := logger.NewFromConfig("", config) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index cb0fd62923aa..4a53d2fc6ad9 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -43,7 +43,7 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { errors.M(errors.MetaKeyPath, pathConfigFile)) } - logger, err := logger.NewFromConfig(config) + logger, err := logger.NewFromConfig("", config) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index 1f3232844056..b746d6cd0e20 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -96,7 +96,7 @@ func getLogger() *logger.Logger { "level": "error", }, }) - l, _ := logger.NewFromConfig(cfg) + l, _ := logger.NewFromConfig("", cfg) return l } diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index b92785a11b35..c9c4f9eb9d04 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -430,7 +430,7 @@ func TestConfiguration(t *testing.T) { }, } - l, _ := logger.New() + l, _ := logger.New("") for name, test := range testcases { t.Run(name, func(t *testing.T) { singleConfig, err := ioutil.ReadFile(filepath.Join("testdata", name+".yml")) diff --git a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go index df6335f0668d..545832c08950 100644 --- a/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go +++ b/x-pack/elastic-agent/pkg/agent/stateresolver/stateresolver_test.go @@ -25,7 +25,7 @@ func TestStateResolverAcking(t *testing.T) { } t.Run("when we ACK the should state", func(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") r, err := NewStateResolver(log) require.NoError(t, err) @@ -44,7 +44,7 @@ func TestStateResolverAcking(t *testing.T) { }) t.Run("when we don't ACK the should state", func(t *testing.T) { - log, _ := logger.New() + log, _ := logger.New("") r, err := NewStateResolver(log) require.NoError(t, err) diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go index 5c7e2aa9527e..cf60bd5e4dd0 100644 --- a/x-pack/elastic-agent/pkg/core/logger/logger.go +++ b/x-pack/elastic-agent/pkg/core/logger/logger.go @@ -6,90 +6,103 @@ package logger import ( "fmt" + "path/filepath" - "github.com/urso/ecslog" - "github.com/urso/ecslog/backend" - "github.com/urso/ecslog/backend/appender" - "github.com/urso/ecslog/backend/layout" + "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/logp/configure" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" ) +const agentName = "elastic-agent" + // Logger alias ecslog.Logger with Logger. -type Logger = ecslog.Logger +type Logger = logp.Logger -// Config is a configuration of logging. -type Config struct { - Level loggingLevel `config:"level"` -} +// Config is a logging config. +type Config = logp.Config -// DefaultLoggingConfig creates a default logging configuration. -func DefaultLoggingConfig() *Config { - return &Config{ - Level: loggingLevel(backend.Trace), +// New returns a configured ECS Logger +func New(name string) (*Logger, error) { + dc, err := defaultConfig() + if err != nil { + return nil, err } + return new(name, dc) } -// New returns a configured ECS Logger -func New() (*Logger, error) { - return new(backend.Trace) -} +// NewWithLogpLevel returns a configured logp Logger with specified level. +func NewWithLogpLevel(name string, level logp.Level) (*Logger, error) { + cfg := struct { + Level string `config:"level"` + }{Level: level.String()} -func createJSONBackend(lvl backend.Level) (backend.Backend, error) { - return appender.Console(lvl, layout.Text(true)) + commonCfg, err := common.NewConfigFrom(cfg) + if err != nil { + return nil, err + } + + return new(name, commonCfg) } //NewFromConfig takes the user configuration and generate the right logger. // TODO: Finish implementation, need support on the library that we use. -func NewFromConfig(cfg *config.Config) (*Logger, error) { - wrappedConfig := &struct { - Logging *Config `config:"logging"` - }{ - Logging: DefaultLoggingConfig(), +func NewFromConfig(name string, cfg *config.Config) (*Logger, error) { + defaultCfg, err := defaultConfig() + if err != nil { + return nil, err } + wrappedConfig := &struct { + Logging *common.Config `config:"logging"` + }{Logging: defaultCfg} if err := cfg.Unpack(&wrappedConfig); err != nil { return nil, err } - return new(backend.Level(wrappedConfig.Logging.Level)) + return new(name, wrappedConfig.Logging) } -func new(lvl backend.Level) (*Logger, error) { - backend, err := createJSONBackend(lvl) - if err != nil { - return nil, err +func new(name string, cfg *common.Config) (*Logger, error) { + if err := configure.Logging("", cfg); err != nil { + return nil, fmt.Errorf("error initializing logging: %v", err) } - return ecslog.New(backend), nil -} - -type loggingLevel backend.Level -var loggingLevelMap = map[string]loggingLevel{ - "trace": loggingLevel(backend.Trace), - "debug": loggingLevel(backend.Debug), - "info": loggingLevel(backend.Info), - "error": loggingLevel(backend.Error), + return logp.NewLogger(name), nil } -func (m *loggingLevel) Unpack(v string) error { - mgt, ok := loggingLevelMap[v] - if !ok { - return fmt.Errorf( - "unknown logging level mode, received '%s' and valid values are 'trace', 'debug', 'info' or 'error'", - v, - ) +func defaultConfig() (*common.Config, error) { + cfg := DefaultLoggingConfig() + + // work around custom types and common config + // when custom type is transformed to common.Config + // value is determined based on reflect value which is incorrect + // enum vs human readable form + yamlCfg, err := yaml.Marshal(cfg) + if err != nil { + return nil, err } - *m = mgt - return nil -} -func (m *loggingLevel) String() string { - for s, v := range loggingLevelMap { - if v == *m { - return s - } + commonLogp, err := common.NewConfigFrom(string(yamlCfg)) + if err != nil { + return nil, errors.New(err, errors.TypeConfig) } - return "unknown" + return commonLogp, nil +} + +// DefaultLoggingConfig returns default configuration for agent logging. +func DefaultLoggingConfig() *Config { + cfg := logp.DefaultConfig(logp.DefaultEnvironment) + cfg.Beat = agentName + cfg.ECSEnabled = true + cfg.Level = logp.DebugLevel + cfg.Files.Path = filepath.Join(paths.Home(), "data", "logs") + cfg.Files.Name = agentName + + return &cfg } diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/start.go b/x-pack/elastic-agent/pkg/core/plugin/process/start.go index ee45c4a8abed..9ecaf7d5039b 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/start.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/start.go @@ -136,8 +136,6 @@ func injectLogLevel(logLevel string, args []string) []string { var level string // Translate to level beat understands switch logLevel { - case "trace": - level = "debug" case "info": level = "info" case "debug": diff --git a/x-pack/elastic-agent/pkg/core/server/server_test.go b/x-pack/elastic-agent/pkg/core/server/server_test.go index feef37c9bba2..9bb82f303aed 100644 --- a/x-pack/elastic-agent/pkg/core/server/server_test.go +++ b/x-pack/elastic-agent/pkg/core/server/server_test.go @@ -578,7 +578,7 @@ func newErrorLogger(t *testing.T) *logger.Logger { }, }) require.NoError(t, err) - log, err := logger.NewFromConfig(cfg) + log, err := logger.NewFromConfig("", cfg) require.NoError(t, err) return log } diff --git a/x-pack/elastic-agent/pkg/filewatcher/watcher.go b/x-pack/elastic-agent/pkg/filewatcher/watcher.go index 32d96436b880..03619a041354 100644 --- a/x-pack/elastic-agent/pkg/filewatcher/watcher.go +++ b/x-pack/elastic-agent/pkg/filewatcher/watcher.go @@ -50,7 +50,7 @@ type Watch struct { func New(log *logger.Logger, f Comparer) (*Watch, error) { var err error if log == nil { - log, err = logger.New() + log, err = logger.New("watcher") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/fleetapi/client.go b/x-pack/elastic-agent/pkg/fleetapi/client.go index bfdc131ce908..c78b0d8ee133 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/client.go +++ b/x-pack/elastic-agent/pkg/fleetapi/client.go @@ -47,7 +47,7 @@ func init() { return nil, err } - l, err := logger.New() + l, err := logger.New("fleet_client") if err != nil { return nil, errors.New(err, "could not create the logger for debugging HTTP request") } diff --git a/x-pack/elastic-agent/pkg/fleetapi/helper_test.go b/x-pack/elastic-agent/pkg/fleetapi/helper_test.go index e9b2baed1788..85f66e747ca1 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/helper_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/helper_test.go @@ -45,7 +45,7 @@ func withServerWithAuthClient( ) func(t *testing.T) { return withServer(m, func(t *testing.T, host string) { - log, _ := logger.New() + log, _ := logger.New("") cfg := &kibana.Config{ Host: host, } diff --git a/x-pack/elastic-agent/pkg/kibana/client.go b/x-pack/elastic-agent/pkg/kibana/client.go index 33883d2c5977..5a20d1b077af 100644 --- a/x-pack/elastic-agent/pkg/kibana/client.go +++ b/x-pack/elastic-agent/pkg/kibana/client.go @@ -95,7 +95,7 @@ func NewConfigFromURL(kURL string) (*Config, error) { func NewWithRawConfig(log *logger.Logger, config *config.Config, wrapper wrapperFunc) (*Client, error) { l := log if l == nil { - log, err := logger.New() + log, err := logger.New("kibana_client") if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/kibana/client_test.go b/x-pack/elastic-agent/pkg/kibana/client_test.go index c860d62d88d2..818b9a5ea2f0 100644 --- a/x-pack/elastic-agent/pkg/kibana/client_test.go +++ b/x-pack/elastic-agent/pkg/kibana/client_test.go @@ -34,7 +34,7 @@ func addCatchAll(mux *http.ServeMux, t *testing.T) *http.ServeMux { } func TestPortDefaults(t *testing.T) { - l, err := logger.New() + l, err := logger.New("") require.NoError(t, err) testCases := []struct { @@ -69,7 +69,7 @@ func TestPortDefaults(t *testing.T) { // - Prefix. func TestHTTPClient(t *testing.T) { ctx := context.Background() - l, err := logger.New() + l, err := logger.New("") require.NoError(t, err) t.Run("Guard against double slashes on path", withServer( diff --git a/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go b/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go index d3a2d902a63f..8d25852efe7e 100644 --- a/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go +++ b/x-pack/elastic-agent/pkg/reporter/fleet/reporter_test.go @@ -209,7 +209,7 @@ func getEvents(count int) []reporter.Event { } func newTestReporter(frequency time.Duration, threshold int) *Reporter { - log, _ := logger.New() + log, _ := logger.New("") r := &Reporter{ info: &testInfo{}, queue: make([]fleetapi.SerializableEvent, 0), From 8d25201e7527b7b07fde0cb482b61f8609e9e756 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 1 Jul 2020 16:50:49 +0200 Subject: [PATCH 06/35] Change link in readme for new snapshots URL (#19357) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 63501084508c..1b383c288de5 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ your dev environment to build Beats from the source. ## Snapshots -For testing purposes, we generate snapshot builds that you can find [here](https://beats-ci.elastic.co/job/elastic+beats+master+multijob-package-linux/lastSuccessfulBuild/gcsObjects/). Please be aware that these are built on top of master and are not meant for production. +For testing purposes, we generate snapshot builds that you can find [here](https://beats-ci.elastic.co/job/Beats/job/packaging/job/master/lastSuccessfulBuild/gcsObjects/). Please be aware that these are built on top of master and are not meant for production. ## CI From 3ff02cbba4184957cf63cad3e3bf5e23d17bd0f2 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 1 Jul 2020 16:59:47 +0200 Subject: [PATCH 07/35] Allow the Docker image to be run with a random user id (#12905) (#18873) Prepare docker images to be run with arbitrary user ids. Following common practices and recommendations, files that need to be read by Beats have now read permissions for the group and belong to the root group. Also, the user included in the docker image is added to the root group so it can read these files when run on docker with default user and privileges. Some changes are also added to Kubernetes reference manifests to help running beats with arbitrary user ids, though this is not completely supported and it requires additional setup. Co-authored-by: Michael Morello --- CHANGELOG.next.asciidoc | 1 + deploy/kubernetes/auditbeat-kubernetes.yaml | 5 +++-- deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml | 5 +++-- deploy/kubernetes/filebeat-kubernetes.yaml | 3 ++- deploy/kubernetes/filebeat/filebeat-daemonset.yaml | 3 ++- deploy/kubernetes/metricbeat-kubernetes.yaml | 9 +++++---- deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml | 5 +++-- deploy/kubernetes/metricbeat/metricbeat-deployment.yaml | 4 ++-- dev-tools/packaging/templates/docker/Dockerfile.tmpl | 4 ++-- 9 files changed, 23 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b8f05a254c1a..1e9a4eb011b9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -311,6 +311,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958] - Add backoff configuration options for the Kafka output. {issue}16777[16777] {pull}17808[17808] - Add TLS support to Kerberos authentication in Elasticsearch. {pull}18607[18607] +- Change ownership of files in docker images so they can be used in secured environments. {pull}12905[12905] - Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817] - Add support for multiple sets of hints on autodiscover {pull}18883[18883] diff --git a/deploy/kubernetes/auditbeat-kubernetes.yaml b/deploy/kubernetes/auditbeat-kubernetes.yaml index bd6c4736c8b8..24492c50653f 100644 --- a/deploy/kubernetes/auditbeat-kubernetes.yaml +++ b/deploy/kubernetes/auditbeat-kubernetes.yaml @@ -196,14 +196,15 @@ spec: path: /etc - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-daemonset-modules - name: data hostPath: + # When auditbeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/auditbeat-data type: DirectoryOrCreate - name: run-containerd diff --git a/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml b/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml index 21ffb167107a..39eaf726eefc 100644 --- a/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml +++ b/deploy/kubernetes/auditbeat/auditbeat-daemonset.yaml @@ -109,14 +109,15 @@ spec: path: /etc - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: auditbeat-daemonset-modules - name: data hostPath: + # When auditbeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/auditbeat-data type: DirectoryOrCreate - name: run-containerd diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml index 49e4ff12f0bd..7399635e42d6 100644 --- a/deploy/kubernetes/filebeat-kubernetes.yaml +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -112,7 +112,7 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: filebeat-config - name: varlibdockercontainers hostPath: @@ -123,6 +123,7 @@ spec: # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/filebeat-data type: DirectoryOrCreate --- diff --git a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml index 20c742d518d9..b6df8f31fdbd 100644 --- a/deploy/kubernetes/filebeat/filebeat-daemonset.yaml +++ b/deploy/kubernetes/filebeat/filebeat-daemonset.yaml @@ -68,7 +68,7 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: filebeat-config - name: varlibdockercontainers hostPath: @@ -79,5 +79,6 @@ spec: # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart - name: data hostPath: + # When filebeat runs as non-root user, this directory needs to be writable by group (g+w). path: /var/lib/filebeat-data type: DirectoryOrCreate diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 216bbc24f9fb..9690b9cb84f2 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -177,14 +177,15 @@ spec: path: /var/run/docker.sock - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-modules - name: data hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) path: /var/lib/metricbeat-data type: DirectoryOrCreate --- @@ -302,11 +303,11 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-modules --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml index 96f841c45198..0197fe136b6d 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml @@ -84,13 +84,14 @@ spec: path: /var/run/docker.sock - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-daemonset-modules - name: data hostPath: + # When metricbeat runs as non-root user, this directory needs to be writable by group (g+w) path: /var/lib/metricbeat-data type: DirectoryOrCreate diff --git a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml index 8b0c5351ed07..0e11187cac38 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-deployment.yaml @@ -61,9 +61,9 @@ spec: volumes: - name: config configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-config - name: modules configMap: - defaultMode: 0600 + defaultMode: 0640 name: metricbeat-deployment-modules diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 1123bb14f7be..9080b7c534d2 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -30,7 +30,7 @@ RUN chmod 755 /usr/local/bin/docker-entrypoint RUN groupadd --gid 1000 {{ .BeatName }} RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ - chown -R root:{{ .BeatName }} {{ $beatHome }} && \ + chown -R root:root {{ $beatHome }} && \ find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ chmod 0750 {{ $beatBinary }} && \ @@ -43,7 +43,7 @@ RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs {{- if ne .user "root" }} -RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }} +RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} {{- end }} USER {{ .user }} From a9f55c42d3cb49e49dbb5e774b2dd8da2e3fb890 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 1 Jul 2020 17:17:01 +0200 Subject: [PATCH 08/35] Add unit tests for cursor API (#19529) Stateful inputs can access the current state via the Cursor type. This change adds unit tests to ensure that the cursor correctly interacts with the ephemeral and persistent stores. --- filebeat/input/v2/input-cursor/cursor_test.go | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 filebeat/input/v2/input-cursor/cursor_test.go diff --git a/filebeat/input/v2/input-cursor/cursor_test.go b/filebeat/input/v2/input-cursor/cursor_test.go new file mode 100644 index 000000000000..ed1f41208696 --- /dev/null +++ b/filebeat/input/v2/input-cursor/cursor_test.go @@ -0,0 +1,124 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCursor_IsNew(t *testing.T) { + t.Run("true if key is not in store", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.True(t, cursor.IsNew()) + }) + + t.Run("true if key is in store but without cursor value", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: nil}, + })) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.True(t, cursor.IsNew()) + }) + + t.Run("false if key with cursor value is in persistent store", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + cursor := makeCursor(store, store.Get("test::key")) + require.False(t, cursor.IsNew()) + }) + + t.Run("false if key with cursor value is in memory store only", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: nil}, + })) + defer store.Release() + + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + cursor := makeCursor(store, res) + require.False(t, cursor.IsNew()) + }) +} + +func TestCursor_Unpack(t *testing.T) { + t.Run("nothing to unpack if key is new", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "", st) + }) + + t.Run("unpack fails if types are not compatible", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + var st struct{ A uint } + cursor := makeCursor(store, store.Get("test::key")) + require.Error(t, cursor.Unpack(&st)) + }) + + t.Run("unpack from state in persistent store", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "test", st) + }) + + t.Run("unpack from in memory state if updates are pending", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, map[string]state{ + "test::key": {Cursor: "test"}, + })) + defer store.Release() + + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + defer op.done(1) + + var st string + cursor := makeCursor(store, store.Get("test::key")) + + require.NoError(t, cursor.Unpack(&st)) + require.Equal(t, "test-state-update", st) + }) +} From a31e76d5d8779792fedf019ab6a70271aae047c3 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 1 Jul 2020 09:23:54 -0600 Subject: [PATCH 09/35] change #19423 changelog from bug to add (#19556) --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1e9a4eb011b9..19ddf5ed5fec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -259,7 +259,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix incorrect usage of hints builder when exposed port is a substring of the hint {pull}19052[19052] - Remove dedot for tag values in aws module. {issue}19112[19112] {pull}19221[19221] - Stop counterCache only when already started {pull}19103[19103] -- Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] - Fix bug incorrect parsing of float numbers as integers in Couchbase module {issue}18949[18949] {pull}19055[19055] *Packetbeat* @@ -518,6 +517,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update Couchbase to version 6.5 {issue}18595[18595] {pull}19055[19055] - Add dashboards for googlecloud load balancing metricset. {pull}18369[18369] - Add support for v1 consumer API in Cloud Foundry module, use it by default. {pull}19268[19268] +- Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] *Packetbeat* From 4815ba1839512faf5f3c280d5bd610c0db09ced0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 1 Jul 2020 18:05:10 +0200 Subject: [PATCH 10/35] Remove outdated references to vendor folder (#19557) --- Makefile | 6 +++--- dev-tools/mage/dashboard.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 10f7fd476cbb..0e21f801f104 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PYTHON_ENV?=$(BUILD_DIR)/python-env PYTHON_EXE?=python3 PYTHON_ENV_EXE=${PYTHON_ENV}/bin/$(notdir ${PYTHON_EXE}) VENV_PARAMS?= -FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*" +FIND=find . -type f -not -path "*/build/*" -not -path "*/.git/*" GOLINT=golint GOLINT_REPO=golang.org/x/lint/golint REVIEWDOG=reviewdog @@ -96,8 +96,8 @@ clean: mage .PHONY: check check: python-env @$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;) - @$(FIND) -name *.py -name *.py -not -path "*/build/*" -not -path "*/vendor/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) - @$(FIND) -name *.py -not -path "*/build/*" -not -path "*/vendor/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) + @$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) + @$(FIND) -name *.py -not -path "*/build/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false) @# Validate that all updates were committed @$(MAKE) update @$(MAKE) check-headers diff --git a/dev-tools/mage/dashboard.go b/dev-tools/mage/dashboard.go index 562a3bf8f799..764a9e286952 100644 --- a/dev-tools/mage/dashboard.go +++ b/dev-tools/mage/dashboard.go @@ -44,7 +44,7 @@ func ExportDashboard() error { return err } - dashboardCmd := sh.RunCmd("go", "run", "-mod", "vendor", filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go")) + dashboardCmd := sh.RunCmd("go", "run", filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go")) // TODO: This is currently hardcoded for KB 7, we need to figure out what we do for KB 8 if applicable file := CWD("module", module, "_meta/kibana/7/dashboard", id+".json") From f7b2a9bf1bd7703d96617f3fb50d0ec6e197cdab Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 1 Jul 2020 12:22:39 -0400 Subject: [PATCH 11/35] Update Jenkinsfile to check go.mod instead of vendor/ --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 02231608a012..6aec0cf79f69 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1104,7 +1104,7 @@ def isChanged(patterns){ def isChangedOSSCode(patterns) { def allPatterns = [ "^Jenkinsfile", - "^vendor/.*", + "^go.mod", "^libbeat/.*", "^testing/.*", "^dev-tools/.*", @@ -1117,7 +1117,7 @@ def isChangedOSSCode(patterns) { def isChangedXPackCode(patterns) { def allPatterns = [ "^Jenkinsfile", - "^vendor/.*", + "^go.mod", "^libbeat/.*", "^dev-tools/.*", "^testing/.*", From a63eb660df2cefc817f457058f1d3a50282f7c0d Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 1 Jul 2020 18:39:52 +0200 Subject: [PATCH 12/35] ci: trigger packages builds only manually or with a comment (#19559) * feat: trigger build only manually or with a comment * Update .ci/packaging.groovy --- .ci/packaging.groovy | 184 +++++++++++++++++++++---------------------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 25ec166fd83d..4c1fa26ebdb2 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -3,7 +3,7 @@ @Library('apm@current') _ pipeline { - agent { label 'ubuntu && immutable' } + agent none environment { BASE_DIR = 'src/github.com/elastic/beats' JOB_GCS_BUCKET = 'beats-ci-artifacts' @@ -12,6 +12,7 @@ pipeline { DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_REGISTRY = 'docker.elastic.co' SNAPSHOT = "true" + PIPELINE_LOG_LEVEL = "INFO" } options { timeout(time: 3, unit: 'HOURS') @@ -32,106 +33,105 @@ pipeline { booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.') } stages { - stage('Checkout') { + stage('Filter build') { + agent { label 'ubuntu && immutable' } when { beforeAgent true - not { - triggeredBy 'SCMTrigger' + expression { + return isCommentTrigger() || isUserTrigger() } } - options { skipDefaultCheckout() } - steps { - deleteDir() - gitCheckout(basedir: "${BASE_DIR}") - setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") - } - } - stage('Build Packages'){ - when { - beforeAgent true - not { - triggeredBy 'SCMTrigger' - } - } - matrix { - axes { - axis { - name 'BEATS_FOLDER' - values ( - 'auditbeat', - 'filebeat', - 'heartbeat', - 'journalbeat', - 'metricbeat', - 'packetbeat', - 'winlogbeat', - 'x-pack/auditbeat', - 'x-pack/elastic-agent', - 'x-pack/dockerlogbeat', - 'x-pack/filebeat', - 'x-pack/functionbeat', - // 'x-pack/heartbeat', - // 'x-pack/journalbeat', - 'x-pack/metricbeat', - // 'x-pack/packetbeat', - 'x-pack/winlogbeat' - ) + stages { + stage('Checkout') { + options { skipDefaultCheckout() } + steps { + deleteDir() + gitCheckout(basedir: "${BASE_DIR}") + setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim()) + stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}") } } - stages { - stage('Package Linux'){ - agent { label 'ubuntu && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.linux - } - } - environment { - HOME = "${env.WORKSPACE}" - PLATFORMS = [ - '+all', - 'linux/amd64', - 'linux/386', - 'linux/arm64', - 'linux/armv7', - 'linux/ppc64le', - 'linux/mips64', - 'linux/s390x', - 'windows/amd64', - 'windows/386', - (params.macos ? '' : 'darwin/amd64'), - ].join(' ') - } - steps { - withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") { - release() - pushCIDockerImages() + stage('Build Packages'){ + matrix { + axes { + axis { + name 'BEATS_FOLDER' + values ( + 'auditbeat', + 'filebeat', + 'heartbeat', + 'journalbeat', + 'metricbeat', + 'packetbeat', + 'winlogbeat', + 'x-pack/auditbeat', + 'x-pack/elastic-agent', + 'x-pack/dockerlogbeat', + 'x-pack/filebeat', + 'x-pack/functionbeat', + // 'x-pack/heartbeat', + // 'x-pack/journalbeat', + 'x-pack/metricbeat', + // 'x-pack/packetbeat', + 'x-pack/winlogbeat' + ) } } - } - stage('Package Mac OS'){ - agent { label 'macosx-10.12' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.macos + stages { + stage('Package Linux'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.linux + } + } + environment { + HOME = "${env.WORKSPACE}" + PLATFORMS = [ + '+all', + 'linux/amd64', + 'linux/386', + 'linux/arm64', + 'linux/armv7', + 'linux/ppc64le', + 'linux/mips64', + 'linux/s390x', + 'windows/amd64', + 'windows/386', + (params.macos ? '' : 'darwin/amd64'), + ].join(' ') + } + steps { + withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") { + release() + pushCIDockerImages() + } + } } - } - environment { - HOME = "${env.WORKSPACE}" - PLATFORMS = [ - '+all', - 'darwin/amd64', - ].join(' ') - } - steps { - withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") { - withMacOSEnv(){ - release() + stage('Package Mac OS'){ + agent { label 'macosx-10.12' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macos + } + } + environment { + HOME = "${env.WORKSPACE}" + PLATFORMS = [ + '+all', + 'darwin/amd64', + ].join(' ') + } + steps { + withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") { + withMacOSEnv(){ + release() + } + } } } } From cf46a2a12fbe879562ab56ee1f9c33d6bb4ac40d Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Wed, 1 Jul 2020 14:37:35 -0400 Subject: [PATCH 13/35] Update Beats to the current Sarama fork --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 1f1b68965d41..0301ab2b2b25 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2034,11 +2034,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/sarama -Version: v1.24.1-elastic.0.20200519143807-cbc80333a91e +Version: v1.19.1-0.20200629123429-0e7b69039eec Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.24.1-elastic.0.20200519143807-cbc80333a91e/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/sarama@v1.19.1-0.20200629123429-0e7b69039eec/LICENSE: Copyright (c) 2013 Shopify diff --git a/go.mod b/go.mod index f5cb0cde6fdb..5a22835bec5b 100644 --- a/go.mod +++ b/go.mod @@ -179,7 +179,7 @@ require ( replace ( github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible - github.com/Shopify/sarama => github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e + github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec github.com/cucumber/godog => github.com/cucumber/godog v0.8.1 github.com/docker/docker => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f diff --git a/go.sum b/go.sum index 214b18ae45d9..cc55ea372c3b 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gosigar v0.10.5 h1:GzPQ+78RaAb4J63unidA/JavQRKrB6s8IOzN6Ib59jo= github.com/elastic/gosigar v0.10.5/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs= -github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e h1:2jm3380rkaGcosRpvtgIQrl7F5Cb99aFJYis7Y5hoJw= -github.com/elastic/sarama v1.24.1-elastic.0.20200519143807-cbc80333a91e/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE= +github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec h1:rAHd7DeHIHjSzvnkl197GKh9TCWGKg/z2BBbbGOEiWI= +github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= From 07639fe0a0bb6f12671c5c3f9b5039dc9d092e09 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 1 Jul 2020 15:15:27 -0600 Subject: [PATCH 14/35] Add awscloudwatch filebeat input (#19025) * Add awscloudwatch filebeat input * Use log group ARN instead of log group name and region name * add api_sleep, log_group_name and region_name config --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 42 +++ .../docs/inputs/input-awscloudwatch.asciidoc | 119 +++++++ x-pack/filebeat/include/list.go | 1 + .../input/awscloudwatch/_meta/fields.yml | 20 ++ x-pack/filebeat/input/awscloudwatch/config.go | 57 ++++ x-pack/filebeat/input/awscloudwatch/fields.go | 23 ++ x-pack/filebeat/input/awscloudwatch/input.go | 316 ++++++++++++++++++ .../input/awscloudwatch/input_test.go | 156 +++++++++ 9 files changed, 735 insertions(+) create mode 100644 x-pack/filebeat/docs/inputs/input-awscloudwatch.asciidoc create mode 100644 x-pack/filebeat/input/awscloudwatch/_meta/fields.yml create mode 100644 x-pack/filebeat/input/awscloudwatch/config.go create mode 100644 x-pack/filebeat/input/awscloudwatch/fields.go create mode 100644 x-pack/filebeat/input/awscloudwatch/input.go create mode 100644 x-pack/filebeat/input/awscloudwatch/input_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 19ddf5ed5fec..b4b98b5dc002 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -418,6 +418,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Explicitly set ECS version in all Filebeat modules. {pull}19198[19198] - Add new mode to multiline reader to aggregate constant number of lines {pull}18352[18352] - Add automatic retries and exponential backoff to httpjson input. {pull}18956[18956] +- Add awscloudwatch input. {pull}19025[19025] - Changed the panw module to pass through (rather than drop) message types other than threat and traffic. {issue}16815[16815] {pull}19375[19375] - Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346] - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 8562e3da0e2e..71c438adde9a 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2040,6 +2041,47 @@ type: keyword The type of traffic: IPv4, IPv6, or EFA. +type: keyword + +-- + +[[exported-fields-awscloudwatch]] +== awscloudwatch fields + +Fields from AWS CloudWatch logs. + + + +[float] +=== awscloudwatch + +Fields from AWS CloudWatch logs. + + + +*`awscloudwatch.log_group`*:: ++ +-- +The name of the log group to which this event belongs. + +type: keyword + +-- + +*`awscloudwatch.log_stream`*:: ++ +-- +The name of the log stream to which this event belongs. + +type: keyword + +-- + +*`awscloudwatch.ingestion_time`*:: ++ +-- +The time the event was ingested in AWS CloudWatch. + type: keyword -- diff --git a/x-pack/filebeat/docs/inputs/input-awscloudwatch.asciidoc b/x-pack/filebeat/docs/inputs/input-awscloudwatch.asciidoc new file mode 100644 index 000000000000..76f80963d5a6 --- /dev/null +++ b/x-pack/filebeat/docs/inputs/input-awscloudwatch.asciidoc @@ -0,0 +1,119 @@ +[role="xpack"] + +:libbeat-xpack-dir: ../../../../x-pack/libbeat + +:type: awscloudwatch + +[id="{beatname_lc}-input-{type}"] +=== awscloudwatch input + +++++ +awscloudwatch +++++ + +beta[] + +`awscloudwatch` input can be used to retrieve all logs from all log streams in a +specific log group. `filterLogEvents` AWS API is used to list log events from +the specified log group. Amazon CloudWatch Logs can be used to store log files +from Amazon Elastic Compute Cloud(EC2), AWS CloudTrail, Route53, and other sources. + +A log group is a group of log streams that share the same retention, monitoring, +and access control settings. You can define log groups and specify which streams +to put into each group. There is no limit on the number of log streams that can +belong to one log group. + +A log stream is a sequence of log events that share the same source. Each +separate source of logs in CloudWatch Logs makes up a separate log stream. + +["source","yaml",subs="attributes"] +---- +{beatname_lc}.inputs: +- type: awscloudwatch + log_group_arn: arn:aws:logs:us-east-1:428152502467:log-group:test:* + scan_frequency: 1m + credential_profile_name: elastic-beats + start_position: beginning +---- + +The `awscloudwatch` input supports the following configuration options plus the +<<{beatname_lc}-input-{type}-common-options>> described later. + +[float] +==== `log_group_arn` +ARN of the log group to collect logs from. + +==== `log_group_name` +Name of the log group to collect logs from. Note: region_name is required when +log_group_name is given. + +==== `region_name` +Region that the specified log group belongs to. + +[float] +==== `log_streams` +A list of strings of log streams names that Filebeat collect log events from. + +[float] +==== `log_stream_prefix` +A string to filter the results to include only log events from log streams +that have names starting with this prefix. + +[float] +==== `start_position` +`start_position` allows user to specify if this input should read log files from +the `beginning` or from the `end`. + +* `beginning`: reads from the beginning of the log group (default). +* `end`: read only new messages from current time minus `scan_frequency` going forward + +For example, with `scan_frequency` equals to `30s` and current timestamp is +`2020-06-24 12:00:00`: + +* with `start_position = beginning`: +** first iteration: startTime=0, endTime=2020-06-24 12:00:00 +** second iteration: startTime=2020-06-24 12:00:00, endTime=2020-06-24 12:00:30 + +* with `start_position = end`: +** first iteration: startTime=2020-06-24 11:59:30, endTime=2020-06-24 12:00:00 +** second iteration: startTime=2020-06-24 12:00:00, endTime=2020-06-24 12:00:30 + +[float] +==== `scan_frequency` +This config parameter sets how often Filebeat checks for new log events from the +specified log group. Default `scan_frequency` is 1 minute, which means Filebeat +will sleep for 1 minute before querying for new logs again. + +[float] +==== `api_timeout` +The maximum duration of AWS API can take. If it exceeds the timeout, AWS API +will be interrupted. The default AWS API timeout for a message is 120 seconds. +The minimum is 0 seconds. The maximum is half of the visibility timeout value. + +[float] +==== `api_sleep` +This is used to sleep between AWS `FilterLogEvents` API calls inside the same +collection period. `FilterLogEvents` API has a quota of 5 transactions per +second (TPS)/account/Region. By default, `api_sleep` is 200 ms. This value should +only be adjusted when there are multiple Filebeats or multiple Filebeat inputs +collecting logs from the same region and AWS account. + +[float] +==== `aws credentials` +In order to make AWS API calls, `awscloudwatch` input requires AWS credentials. +Please see <> for more details. + +[float] +=== AWS Permissions +Specific AWS permissions are required for IAM user to access awscloudwatch: +---- +logs:FilterLogEvents +---- + +[id="{beatname_lc}-input-{type}-common-options"] +include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] + +[id="aws-credentials-config"] +include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[] + +:type!: diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index a5719d18fc73..da50998f5343 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -8,6 +8,7 @@ package include import ( // Import packages that need to register themselves. + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/cloudfoundry" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/googlepubsub" diff --git a/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml b/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml new file mode 100644 index 000000000000..ea5fdd558747 --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml @@ -0,0 +1,20 @@ +- key: awscloudwatch + title: "awscloudwatch" + description: > + Fields from AWS CloudWatch logs. + fields: + - name: awscloudwatch + type: group + default_field: false + description: > + Fields from AWS CloudWatch logs. + fields: + - name: log_group + type: keyword + description: The name of the log group to which this event belongs. + - name: log_stream + type: keyword + description: The name of the log stream to which this event belongs. + - name: ingestion_time + type: keyword + description: The time the event was ingested in AWS CloudWatch. diff --git a/x-pack/filebeat/input/awscloudwatch/config.go b/x-pack/filebeat/input/awscloudwatch/config.go new file mode 100644 index 000000000000..d916ea065da1 --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/config.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awscloudwatch + +import ( + "errors" + "time" + + "github.com/elastic/beats/v7/filebeat/harvester" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" +) + +type config struct { + harvester.ForwarderConfig `config:",inline"` + LogGroupARN string `config:"log_group_arn"` + LogGroupName string `config:"log_group_name"` + RegionName string `config:"region_name"` + LogStreams []string `config:"log_streams"` + LogStreamPrefix string `config:"log_stream_prefix"` + StartPosition string `config:"start_position" default:"beginning"` + ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"` + APITimeout time.Duration `config:"api_timeout" validate:"min=0,nonzero"` + APISleep time.Duration `config:"api_sleep" validate:"min=0,nonzero"` + AwsConfig awscommon.ConfigAWS `config:",inline"` +} + +func defaultConfig() config { + return config{ + ForwarderConfig: harvester.ForwarderConfig{ + Type: "awscloudwatch", + }, + StartPosition: "beginning", + ScanFrequency: 10 * time.Second, + APITimeout: 120 * time.Second, + APISleep: 200 * time.Millisecond, // FilterLogEvents has a limit of 5 transactions per second (TPS)/account/Region: 1s / 5 = 200 ms + } +} + +func (c *config) Validate() error { + if c.StartPosition != "beginning" && c.StartPosition != "end" { + return errors.New("start_position config parameter can only be " + + "either 'beginning' or 'end'") + } + + if c.LogGroupARN == "" && c.LogGroupName == "" { + return errors.New("log_group_arn and log_group_name config parameter" + + "cannot be both empty") + } + + if c.LogGroupName != "" && c.RegionName == "" { + return errors.New("region_name is required when log_group_name " + + "config parameter is given") + } + return nil +} diff --git a/x-pack/filebeat/input/awscloudwatch/fields.go b/x-pack/filebeat/input/awscloudwatch/fields.go new file mode 100644 index 000000000000..61fa1b5d745f --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package awscloudwatch + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "awscloudwatch", asset.ModuleFieldsPri, AssetAwscloudwatch); err != nil { + panic(err) + } +} + +// AssetAwscloudwatch returns asset data. +// This is the base64 encoded gzipped contents of input/awscloudwatch. +func AssetAwscloudwatch() string { + return "eJys0cFqwzAMBuB7nuKn9/YBfBiMwV5ggx6DFyuxqWMFW5nJ2w87ZU3WyzaqoyL9+oKPuNCioHPqPM8ma+lsA4gTTwqHXf/QAIZSF90kjoPCUwMAr468Segjj3g+v+GlLJzLAjwP6dQAfR1RdfyIoEe6P1lKlokUhsjzdO0Y6vXspa0JCr32ib4/3VF+ySm1JW1Znod2C7ixLrRkjmbT3wneLdUIcA+xVILWP4EwsnWdhViXQJ8UBB/kOdw0+/tJIunxEYA16W8CFwZKJbIVN9J/FGWvEtZLWadrKBm48ONVTs1XAAAA//8tpMbE" +} diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go new file mode 100644 index 000000000000..8c19ca4a24ff --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/input.go @@ -0,0 +1,316 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awscloudwatch + +import ( + "context" + "strings" + "sync" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/aws/awserr" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/filebeat/channel" + "github.com/elastic/beats/v7/filebeat/input" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/logp" + awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" +) + +const inputName = "awscloudwatch" + +func init() { + err := input.Register(inputName, NewInput) + if err != nil { + panic(errors.Wrapf(err, "failed to register %v input", inputName)) + } +} + +// awsCloudWatchInput is a input for AWS CloudWatch logs +type awsCloudWatchInput struct { + config config + awsConfig awssdk.Config + + logger *logp.Logger + outlet channel.Outleter // Output of received awscloudwatch logs. + inputCtx *channelContext + + workerOnce sync.Once // Guarantees that the worker goroutine is only started once. + workerWg sync.WaitGroup // Waits on awscloudwatch worker goroutine. + stopOnce sync.Once + close chan struct{} + + prevEndTime int64 // track previous endTime for each iteration. +} + +// channelContext implements context.Context by wrapping a channel +type channelContext struct { + done <-chan struct{} +} + +func (c *channelContext) Deadline() (time.Time, bool) { return time.Time{}, false } +func (c *channelContext) Done() <-chan struct{} { return c.done } +func (c *channelContext) Err() error { + select { + case <-c.done: + return context.Canceled + default: + return nil + } +} +func (c *channelContext) Value(key interface{}) interface{} { return nil } + +// NewInput creates a new awscloudwatch input +func NewInput(cfg *common.Config, connector channel.Connector, context input.Context) (input.Input, error) { + cfgwarn.Beta("awsclouwatch input type is used") + logger := logp.NewLogger(inputName) + + // Extract and validate the input's configuration. + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, errors.Wrap(err, "failed unpacking config") + } + logger.Debug("awscloudwatch input config = ", config) + + if config.LogGroupARN != "" { + logGroupName, regionName, err := parseARN(config.LogGroupARN) + if err != nil { + return nil, errors.Wrap(err, "parse log group ARN failed") + } + + config.LogGroupName = logGroupName + config.RegionName = regionName + } + + awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) + if err != nil { + return nil, errors.Wrap(err, "getAWSCredentials failed") + } + awsConfig.Region = config.RegionName + + closeChannel := make(chan struct{}) + in := &awsCloudWatchInput{ + config: config, + awsConfig: awsConfig, + logger: logger, + close: closeChannel, + inputCtx: &channelContext{closeChannel}, + prevEndTime: int64(0), + } + + // Build outlet for events. + in.outlet, err = connector.Connect(cfg) + if err != nil { + return nil, err + } + + in.logger.Info("Initialized AWS CloudWatch input.") + return in, nil +} + +// Run runs the input +func (in *awsCloudWatchInput) Run() { + in.workerOnce.Do(func() { + in.workerWg.Add(1) + go func() { + in.logger.Infof("awscloudwatch input worker for log group: '%v' has started", in.config.LogGroupName) + defer in.logger.Infof("awscloudwatch input worker for log group '%v' has stopped.", in.config.LogGroupName) + defer in.workerWg.Done() + in.run() + }() + }) +} + +func (in *awsCloudWatchInput) run() { + cwConfig := awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "cloudwatchlogs", in.config.RegionName, in.awsConfig) + svc := cloudwatchlogs.New(cwConfig) + for in.inputCtx.Err() == nil { + err := in.getLogEventsFromCloudWatch(svc) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + continue + } + in.logger.Error("getLogEventsFromCloudWatch failed: ", err) + continue + } + + in.logger.Debugf("sleeping for %v before checking new logs", in.config.ScanFrequency) + time.Sleep(in.config.ScanFrequency) + in.logger.Debug("done sleeping") + } +} + +func parseARN(logGroupARN string) (string, string, error) { + arnParsed, err := arn.Parse(logGroupARN) + if err != nil { + return "", "", errors.Errorf("error Parse arn %s: %v", logGroupARN, err) + } + + if strings.Contains(arnParsed.Resource, ":") { + resourceARNSplit := strings.Split(arnParsed.Resource, ":") + if len(resourceARNSplit) >= 2 && resourceARNSplit[0] == "log-group" { + return resourceARNSplit[1], arnParsed.Region, nil + } + } + return "", "", errors.Errorf("cannot get log group name from log group ARN: %s", logGroupARN) +} + +// getLogEventsFromCloudWatch uses FilterLogEvents API to collect logs from CloudWatch +func (in *awsCloudWatchInput) getLogEventsFromCloudWatch(svc cloudwatchlogsiface.ClientAPI) error { + ctx, cancelFn := context.WithTimeout(in.inputCtx, in.config.APITimeout) + defer cancelFn() + + init := true + nextToken := "" + currentTime := time.Now() + startTime, endTime := getStartPosition(in.config.StartPosition, currentTime, in.prevEndTime, in.config.ScanFrequency) + in.logger.Debugf("start_position = %s, startTime = %v, endTime = %v", in.config.StartPosition, time.Unix(startTime/1000, 0), time.Unix(endTime/1000, 0)) + + // overwrite prevEndTime using new endTime + in.prevEndTime = endTime + + for nextToken != "" || init { + // construct FilterLogEventsInput + filterLogEventsInput := in.constructFilterLogEventsInput(startTime, endTime, nextToken) + + // make API request + req := svc.FilterLogEventsRequest(filterLogEventsInput) + resp, err := req.Send(ctx) + if err != nil { + in.logger.Error("failed FilterLogEventsRequest", err) + return err + } + + // get token for next API call, if resp.NextToken is nil, nextToken set to "" + nextToken = "" + if resp.NextToken != nil { + nextToken = *resp.NextToken + } + + logEvents := resp.Events + in.logger.Debugf("Processing #%v events", len(logEvents)) + + err = in.processLogEvents(logEvents) + if err != nil { + err = errors.Wrap(err, "processLogEvents failed") + in.logger.Error(err) + cancelFn() + } + + init = false + + // This sleep is to avoid hitting the FilterLogEvents API limit(5 transactions per second (TPS)/account/Region). + in.logger.Debugf("sleeping for %v before making FilterLogEvents API call again", in.config.APISleep) + time.Sleep(in.config.APISleep) + in.logger.Debug("done sleeping") + } + return nil +} + +func (in *awsCloudWatchInput) constructFilterLogEventsInput(startTime int64, endTime int64, nextToken string) *cloudwatchlogs.FilterLogEventsInput { + filterLogEventsInput := &cloudwatchlogs.FilterLogEventsInput{ + LogGroupName: awssdk.String(in.config.LogGroupName), + StartTime: awssdk.Int64(startTime), + EndTime: awssdk.Int64(endTime), + } + + if len(in.config.LogStreams) > 0 { + filterLogEventsInput.LogStreamNames = in.config.LogStreams + } + + if in.config.LogStreamPrefix != "" { + filterLogEventsInput.LogStreamNamePrefix = awssdk.String(in.config.LogStreamPrefix) + } + + if nextToken != "" { + filterLogEventsInput.NextToken = awssdk.String(nextToken) + } + return filterLogEventsInput +} + +func getStartPosition(startPosition string, currentTime time.Time, prevEndTime int64, scanFrequency time.Duration) (startTime int64, endTime int64) { + switch startPosition { + case "beginning": + if prevEndTime != int64(0) { + return prevEndTime, currentTime.UnixNano() / int64(time.Millisecond) + } + return 0, currentTime.UnixNano() / int64(time.Millisecond) + case "end": + if prevEndTime != int64(0) { + return prevEndTime, currentTime.UnixNano() / int64(time.Millisecond) + } + return currentTime.Add(-scanFrequency).UnixNano() / int64(time.Millisecond), currentTime.UnixNano() / int64(time.Millisecond) + } + return +} + +func (in *awsCloudWatchInput) processLogEvents(logEvents []cloudwatchlogs.FilteredLogEvent) error { + for _, logEvent := range logEvents { + event := createEvent(logEvent, in.config.LogGroupName, in.config.RegionName) + err := in.forwardEvent(event) + if err != nil { + err = errors.Wrap(err, "forwardEvent failed") + in.logger.Error(err) + return err + } + } + return nil +} + +func createEvent(logEvent cloudwatchlogs.FilteredLogEvent, logGroup string, regionName string) beat.Event { + event := beat.Event{ + Timestamp: time.Unix(*logEvent.Timestamp/1000, 0).UTC(), + Fields: common.MapStr{ + "message": *logEvent.Message, + "log.file.path": logGroup + "/" + *logEvent.LogStreamName, + "event": common.MapStr{ + "id": *logEvent.EventId, + "ingested": time.Now(), + }, + "awscloudwatch": common.MapStr{ + "log_group": logGroup, + "log_stream": *logEvent.LogStreamName, + "ingestion_time": time.Unix(*logEvent.IngestionTime/1000, 0), + }, + "cloud": common.MapStr{ + "provider": "aws", + "region": regionName, + }, + }, + } + event.SetID(*logEvent.EventId) + + return event +} + +func (in *awsCloudWatchInput) forwardEvent(event beat.Event) error { + ok := in.outlet.OnEvent(event) + if !ok { + return errors.New("OnEvent returned false. Stopping input worker") + } + return nil +} + +// Stop stops the awscloudwatch input +func (in *awsCloudWatchInput) Stop() { + in.stopOnce.Do(func() { + defer in.outlet.Close() + close(in.close) + in.logger.Info("Stopping awscloudwatch input") + }) +} + +// Wait is an alias for Stop. +func (in *awsCloudWatchInput) Wait() { + in.Stop() + in.workerWg.Wait() +} diff --git a/x-pack/filebeat/input/awscloudwatch/input_test.go b/x-pack/filebeat/input/awscloudwatch/input_test.go new file mode 100644 index 000000000000..6694ad260674 --- /dev/null +++ b/x-pack/filebeat/input/awscloudwatch/input_test.go @@ -0,0 +1,156 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awscloudwatch + +import ( + "net/http" + "testing" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/cloudwatchlogsiface" + + "github.com/elastic/beats/v7/libbeat/common" + + "github.com/stretchr/testify/assert" +) + +func TestGetStartPosition(t *testing.T) { + currentTime := time.Date(2020, time.June, 1, 0, 0, 0, 0, time.UTC) + cases := []struct { + title string + startPosition string + prevEndTime int64 + scanFrequency time.Duration + expectedStartTime int64 + expectedEndTime int64 + }{ + { + "startPosition=beginning", + "beginning", + int64(0), + 30 * time.Second, + int64(0), + int64(1590969600000), + }, + { + "startPosition=end", + "end", + int64(0), + 30 * time.Second, + int64(1590969570000), + int64(1590969600000), + }, + { + "startPosition=typo", + "typo", + int64(0), + 30 * time.Second, + int64(0), + int64(0), + }, + { + "startPosition=beginning with prevEndTime", + "beginning", + int64(1590000000000), + 30 * time.Second, + int64(1590000000000), + int64(1590969600000), + }, + { + "startPosition=end with prevEndTime", + "end", + int64(1590000000000), + 30 * time.Second, + int64(1590000000000), + int64(1590969600000), + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + startTime, endTime := getStartPosition(c.startPosition, currentTime, c.prevEndTime, c.scanFrequency) + assert.Equal(t, c.expectedStartTime, startTime) + assert.Equal(t, c.expectedEndTime, endTime) + }) + } +} + +// MockCloudwatchlogsClient struct is used for unit tests. +type MockCloudwatchlogsClient struct { + cloudwatchlogsiface.ClientAPI +} + +var ( + mockSvc = &MockCloudwatchlogsClient{} +) + +func (m *MockCloudwatchlogsClient) FilterLogEventsRequest(input *cloudwatchlogs.FilterLogEventsInput) cloudwatchlogs.FilterLogEventsRequest { + events := []cloudwatchlogs.FilteredLogEvent{ + { + EventId: awssdk.String("id-1"), + IngestionTime: awssdk.Int64(1590000000000), + LogStreamName: awssdk.String("logStreamName1"), + Message: awssdk.String("test-message-1"), + Timestamp: awssdk.Int64(1590000000000), + }, + { + EventId: awssdk.String("id-2"), + IngestionTime: awssdk.Int64(1600000000000), + LogStreamName: awssdk.String("logStreamName1"), + Message: awssdk.String("test-message-2"), + Timestamp: awssdk.Int64(1600000000000), + }, + } + + httpReq, _ := http.NewRequest("", "", nil) + return cloudwatchlogs.FilterLogEventsRequest{ + Request: &awssdk.Request{ + Data: &cloudwatchlogs.FilterLogEventsOutput{ + Events: events, + NextToken: awssdk.String(""), + }, + HTTPRequest: httpReq, + }, + } +} + +func TestCreateEvent(t *testing.T) { + logEvent := cloudwatchlogs.FilteredLogEvent{ + EventId: awssdk.String("id-1"), + IngestionTime: awssdk.Int64(1590000000000), + LogStreamName: awssdk.String("logStreamName1"), + Message: awssdk.String("test-message-1"), + Timestamp: awssdk.Int64(1600000000000), + } + + expectedEventFields := common.MapStr{ + "message": "test-message-1", + "event": common.MapStr{ + "id": *logEvent.EventId, + }, + "log.file.path": "logGroup1" + "/" + *logEvent.LogStreamName, + "awscloudwatch": common.MapStr{ + "log_group": "logGroup1", + "log_stream": *logEvent.LogStreamName, + "ingestion_time": time.Unix(*logEvent.IngestionTime/1000, 0), + }, + "cloud": common.MapStr{ + "provider": "aws", + "region": "us-east-1", + }, + } + event := createEvent(logEvent, "logGroup1", "us-east-1") + event.Fields.Delete("event.ingested") + assert.Equal(t, expectedEventFields, event.Fields) +} + +func TestParseARN(t *testing.T) { + logGroup, regionName, err := parseARN("arn:aws:logs:us-east-1:428152502467:log-group:test:*") + assert.Equal(t, "test", logGroup) + assert.Equal(t, "us-east-1", regionName) + assert.NoError(t, err) +} From 340587620ef9c71330579ab7f0d1f9a62fc0e4f8 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 1 Jul 2020 23:27:26 +0200 Subject: [PATCH 15/35] Cursor input old state collection (#19528) The change provides the collection and deletion support for old states. All entries added to the store have a TTL configured. A key-value pair will be removed from the store once the most recent update timestamp + TTL < now. Pairs with pending updates will not be deleted yet. The collector currently only implements the logic for the clean_inactive setting. In comparison to the old registar, is the TTL not reset on startup. Old entries are still subject to collection, even if they've not been claimed. The TTL will be updated by the InputManager if an input with a different TTL is started for an already seen source. --- NOTICE.txt | 4 +- filebeat/input/v2/input-cursor/clean.go | 82 +++++++++- filebeat/input/v2/input-cursor/clean_test.go | 162 +++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 5 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 filebeat/input/v2/input-cursor/clean_test.go diff --git a/NOTICE.txt b/NOTICE.txt index 0301ab2b2b25..8a2746cb5647 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -5936,11 +5936,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-concert -Version: v0.0.2 +Version: v0.0.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.0.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-concert@v0.0.3/LICENSE: Apache License Version 2.0, January 2004 diff --git a/filebeat/input/v2/input-cursor/clean.go b/filebeat/input/v2/input-cursor/clean.go index e833a9eead0e..e4daaaf6e7dc 100644 --- a/filebeat/input/v2/input-cursor/clean.go +++ b/filebeat/input/v2/input-cursor/clean.go @@ -20,8 +20,10 @@ package cursor import ( "time" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/timed" "github.com/elastic/go-concert/unison" + + "github.com/elastic/beats/v7/libbeat/logp" ) // cleaner removes finished entries from the registry file. @@ -41,5 +43,81 @@ type cleaner struct { // for a long time, and the life time has been exhausted, then the resource will be removed immediately // once the last event has been ACKed. func (c *cleaner) run(canceler unison.Canceler, store *store, interval time.Duration) { - panic("TODO: implement me") + started := time.Now() + timed.Periodic(canceler, interval, func() { + gcStore(c.log, started, store) + }) +} + +// gcStore looks for resources to remove and deletes these. `gcStore` receives +// the start timestamp of the cleaner as reference. If we have entries without +// updates in the registry, that are older than `started`, we will use `started +// + ttl` to decide if an entry will be removed. This way old entries are not +// removed immediately on startup if the Beat is down for a longer period of +// time. +func gcStore(log *logp.Logger, started time.Time, store *store) { + log.Debugf("Start store cleanup") + defer log.Debugf("Done store cleanup") + + states := store.ephemeralStore + states.mu.Lock() + defer states.mu.Unlock() + + keys := gcFind(states.table, started, time.Now()) + if len(keys) == 0 { + log.Debug("No entries to remove were found") + return + } + + if err := gcClean(store, keys); err != nil { + log.Errorf("Failed to remove all entries from the registry: %+v", err) + } +} + +// gcFind searches the store of resources that can be removed. A set of keys to delete is returned. +func gcFind(table map[string]*resource, started, now time.Time) map[string]struct{} { + keys := map[string]struct{}{} + for key, resource := range table { + clean := checkCleanResource(started, now, resource) + if !clean { + // do not clean the resource if it is still live or not serialized to the persistent store yet. + continue + } + keys[key] = struct{}{} + } + + return keys +} + +// gcClean removes key value pairs in the removeSet from the store. +// If deletion in the persistent store fails the entry is kept in memory and +// eventually cleaned up later. +func gcClean(store *store, removeSet map[string]struct{}) error { + for key := range removeSet { + if err := store.persistentStore.Remove(key); err != nil { + return err + } + delete(store.ephemeralStore.table, key) + } + return nil +} + +// checkCleanResource returns true for a key-value pair is assumed to be old, +// if is not in use and there are no more pending updates that still need to be +// written to the persistent store anymore. +func checkCleanResource(started, now time.Time, resource *resource) bool { + if !resource.Finished() { + return false + } + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + ttl := resource.internalState.TTL + reference := resource.internalState.Updated + if started.After(reference) { + reference = started + } + + return reference.Add(ttl).Before(now) && resource.stored } diff --git a/filebeat/input/v2/input-cursor/clean_test.go b/filebeat/input/v2/input-cursor/clean_test.go new file mode 100644 index 000000000000..452120f03586 --- /dev/null +++ b/filebeat/input/v2/input-cursor/clean_test.go @@ -0,0 +1,162 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestGCStore(t *testing.T) { + t.Run("empty store", func(t *testing.T) { + started := time.Now() + + backend := createSampleStore(t, nil) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("state is still alive", func(t *testing.T) { + started := time.Now() + const ttl = 60 * time.Second + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl / 2), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + checkEqualStoreState(t, initState, backend.snapshot()) + }) + + t.Run("old state can be removed", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("old state is not removed if cleanup is not active long enough", func(t *testing.T) { + const ttl = 60 * time.Minute + started := time.Now() + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-2 * ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + + checkEqualStoreState(t, initState, backend.snapshot()) + }) + + t.Run("old state but resource is accessed", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + // access resource and check it is not gc'ed + res := store.Get("test::key") + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, initState, backend.snapshot()) + + // release resource and check it gets gc'ed + res.Release() + want := map[string]state{} + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, want, backend.snapshot()) + }) + + t.Run("old state but resource has pending updates", func(t *testing.T) { + const ttl = 60 * time.Second + started := time.Now().Add(-5 * ttl) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: ttl, + Updated: started.Add(-ttl), + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + // create pending update operation + res := store.Get("test::key") + op, err := createUpdateOp(store, res, "test-state-update") + require.NoError(t, err) + res.Release() + + // cleanup fails + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, initState, backend.snapshot()) + + // cancel operation (no more pending operations) and try to gc again + op.done(1) + gcStore(logp.NewLogger("test"), started, store) + want := map[string]state{} + checkEqualStoreState(t, want, backend.snapshot()) + }) +} diff --git a/go.mod b/go.mod index 5a22835bec5b..aa84ec00f940 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 github.com/elastic/ecs v1.5.0 github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07 - github.com/elastic/go-concert v0.0.2 + github.com/elastic/go-concert v0.0.3 github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 github.com/elastic/go-licenser v0.3.1 github.com/elastic/go-lookslike v0.3.0 diff --git a/go.sum b/go.sum index cc55ea372c3b..ef0fde2efdbf 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,8 @@ github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07 h1 github.com/elastic/elastic-agent-client/v7 v7.0.0-20200601155656-d6a9eb4f6d07/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdboCbArMF/nSCDUXgQuWTeoMmE/z8607X+k7ng= -github.com/elastic/go-concert v0.0.2 h1:hJb9h99LS/lyjf7pE1wQ+eiNw+0CXVLCJR42yx+AvOQ= -github.com/elastic/go-concert v0.0.2/go.mod h1:9MtFarjXroUgmm0m6HY3NSe1XiKhdktiNRRj9hWvIaM= +github.com/elastic/go-concert v0.0.3 h1:f0F4WOi8tBOFIgwA7YbHRQ+Ok8vR+/qFrG7vYvbpX5Q= +github.com/elastic/go-concert v0.0.3/go.mod h1:9MtFarjXroUgmm0m6HY3NSe1XiKhdktiNRRj9hWvIaM= github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8 h1:Jcnojiuok7Ea5hitJK9VWmBigganE2MMETOH0VZasEA= github.com/elastic/go-libaudit/v2 v2.0.0-20200515221334-92371bef3fb8/go.mod h1:j2CZcVcluWDGhQTnq1SOPy1NKEIa74FtQ39Nnz87Jxk= github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrEzObU= From 6edb1d38bb1eb7463e3eb739be901b92954af158 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 1 Jul 2020 23:29:46 +0200 Subject: [PATCH 16/35] Add publisher implementation for stateful inputs (#19530) Add the publisher implementation and unit tests for statefull inputs. The tests check that the publisher correctly updates the ephemeral store and prepare the update operation for the persistent store, that will be applied to the store after it has been ACKed. --- filebeat/input/v2/input-cursor/publish.go | 53 +++++- .../input/v2/input-cursor/publish_test.go | 158 ++++++++++++++++++ 2 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 filebeat/input/v2/input-cursor/publish_test.go diff --git a/filebeat/input/v2/input-cursor/publish.go b/filebeat/input/v2/input-cursor/publish.go index b5d33a7fce4d..64096951132a 100644 --- a/filebeat/input/v2/input-cursor/publish.go +++ b/filebeat/input/v2/input-cursor/publish.go @@ -23,6 +23,7 @@ import ( input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" + "github.com/elastic/beats/v7/libbeat/statestore" ) // Publisher is used to publish an event and update the cursor in a single call to Publish. @@ -64,12 +65,25 @@ type updateOp struct { // The ACK ordering in the publisher pipeline guarantees that update operations // will be ACKed and executed in the correct order. func (c *cursorPublisher) Publish(event beat.Event, cursorUpdate interface{}) error { - panic("TODO: implement me") + if cursorUpdate == nil { + return c.forward(event) + } + + op, err := createUpdateOp(c.cursor.store, c.cursor.resource, cursorUpdate) + if err != nil { + return err + } + + event.Private = op + return c.forward(event) } -// Execute updates the persistent store with the scheduled changes and releases the resource. -func (op *updateOp) Execute(numEvents uint) { - panic("TODO: implement me") +func (c *cursorPublisher) forward(event beat.Event) error { + c.client.Publish(event) + if c.canceler == nil { + return nil + } + return c.canceler.Err() } func createUpdateOp(store *store, resource *resource, updates interface{}) (*updateOp, error) { @@ -106,3 +120,34 @@ func (op *updateOp) done(n uint) { op.resource = nil *op = updateOp{} } + +// Execute updates the persistent store with the scheduled changes and releases the resource. +func (op *updateOp) Execute(n uint) { + resource := op.resource + defer op.done(n) + + resource.stateMutex.Lock() + defer resource.stateMutex.Unlock() + + resource.activeCursorOperations -= n + if resource.activeCursorOperations == 0 { + resource.cursor = resource.pendingCursor + resource.pendingCursor = nil + } else { + typeconv.Convert(&resource.cursor, op.delta) + } + + if resource.internalState.Updated.Before(op.timestamp) { + resource.internalState.Updated = op.timestamp + } + + err := op.store.persistentStore.Set(resource.key, resource.inSyncStateSnapshot()) + if err != nil { + if !statestore.IsClosed(err) { + op.store.log.Errorf("Failed to update state in the registry for '%v'", resource.key) + } + } else { + resource.internalInSync = true + resource.stored = true + } +} diff --git a/filebeat/input/v2/input-cursor/publish_test.go b/filebeat/input/v2/input-cursor/publish_test.go new file mode 100644 index 000000000000..28c274baf94c --- /dev/null +++ b/filebeat/input/v2/input-cursor/publish_test.go @@ -0,0 +1,158 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" +) + +func TestPublish(t *testing.T) { + t.Run("event with cursor state creates update operation", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + var actual beat.Event + client := &pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { actual = event }, + } + publisher := cursorPublisher{nil, client, &cursor} + publisher.Publish(beat.Event{}, "test") + + require.NotNil(t, actual.Private) + }) + + t.Run("event without cursor creates no update operation", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + var actual beat.Event + client := &pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { actual = event }, + } + publisher := cursorPublisher{nil, client, &cursor} + publisher.Publish(beat.Event{}, nil) + require.Nil(t, actual.Private) + }) + + t.Run("publish returns error if context has been cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + cancel() + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + cursor := makeCursor(store, store.Get("test::key")) + + publisher := cursorPublisher{ctx, &pubtest.FakeClient{}, &cursor} + err := publisher.Publish(beat.Event{}, nil) + require.Equal(t, context.Canceled, err) + }) +} + +func TestOp_Execute(t *testing.T) { + t.Run("applying final op marks the key as finished", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create op and release resource. The 'resource' must still be active + op := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state") + res.Release() + require.False(t, res.Finished()) + + // this was the last op, the resource should become inactive + op.Execute(1) + require.True(t, res.Finished()) + + // validate state: + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + want := "test-updated-cursor-state" + assert.Equal(t, want, inSyncCursor) + assert.Equal(t, want, inMemCursor) + }) + + t.Run("acking multiple ops applies the latest update and marks key as finished", func(t *testing.T) { + // when acking N events, intermediate updates are dropped in favor of the latest update operation. + // This test checks that the resource is correctly marked as finished. + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create update operations and release resource. The 'resource' must still be active + mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-dropped") + op := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-final") + res.Release() + require.False(t, res.Finished()) + + // this was the last op, the resource should become inactive + op.Execute(2) + require.True(t, res.Finished()) + + // validate state: + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + want := "test-updated-cursor-state-final" + assert.Equal(t, want, inSyncCursor) + assert.Equal(t, want, inMemCursor) + }) + + t.Run("ACK only subset of pending ops will only update up to ACKed state", func(t *testing.T) { + // when acking N events, intermediate updates are dropped in favor of the latest update operation. + // This test checks that the resource is correctly marked as finished. + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + res := store.Get("test::key") + + // create update operations and release resource. The 'resource' must still be active + op1 := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-intermediate") + op2 := mustCreateUpdateOp(t, store, res, "test-updated-cursor-state-final") + res.Release() + require.False(t, res.Finished()) + + defer op2.done(1) // cleanup after test + + // this was the intermediate op, the resource should still be active + op1.Execute(1) + require.False(t, res.Finished()) + + // validate state (in memory state is always up to data to most recent update): + inSyncCursor := storeInSyncSnapshot(store)["test::key"].Cursor + inMemCursor := storeMemorySnapshot(store)["test::key"].Cursor + assert.Equal(t, "test-updated-cursor-state-intermediate", inSyncCursor) + assert.Equal(t, "test-updated-cursor-state-final", inMemCursor) + }) +} + +func mustCreateUpdateOp(t *testing.T, store *store, resource *resource, updates interface{}) *updateOp { + op, err := createUpdateOp(store, resource, updates) + if err != nil { + t.Fatalf("Failed to create update op: %v", err) + } + return op +} From 1360ff32b782ad0239b91940a742dfa34374b63d Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 1 Jul 2020 16:12:44 -0600 Subject: [PATCH 17/35] [Filebeat] Add default_fields: false to fields.yml in aws module (#19568) * Add default_fields: false to fields.yml in aws module --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/module/aws/elb/_meta/fields.yml | 1 + x-pack/filebeat/module/aws/fields.go | 2 +- x-pack/filebeat/module/aws/s3access/_meta/fields.yml | 1 + x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b4b98b5dc002..9d8db101f4e5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -198,6 +198,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Cisco ASA dissect pattern for 313008 & 313009 messages. {pull}19149[19149] - Fix date and timestamp formats for fortigate module {pull}19316[19316] - Fix memory leak in tcp and unix input sources. {pull}19459[19459] +- Add missing `default_field: false` to aws filesets fields.yml. {pull}19568[19568] *Heartbeat* diff --git a/x-pack/filebeat/module/aws/elb/_meta/fields.yml b/x-pack/filebeat/module/aws/elb/_meta/fields.yml index 9ddfb1239013..9499f8bbb0ed 100644 --- a/x-pack/filebeat/module/aws/elb/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/elb/_meta/fields.yml @@ -1,6 +1,7 @@ - name: elb type: group release: ga + default_field: false description: > Fields for AWS ELB logs. fields: diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index aee1d393f7be..f1ad3e120c03 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXN9z27aTf89fsdOXrzMj6aZp5+bGN70ZxXGuurqpz1LauycWAlcSzhDAAqAV5a+/WQCkSBGUbEtqvnpIbJFcfHaxv7H0EB5xew1sY98AOOEkXsP4j+kbAIMSmcVrmKNjbwBytNyIwgmtruE/3gAA/KrzUiIstIEVU7kUaglSLy0sjF4TmdEbgIVAmdtr/8AQFFtjtRx93LbAa1gaXRbxm8Q69PnoydSU/TqjeLW5RHMZLnWZO8OErC+lVqTPPrfVJ8cFK6XL/BLXsGDSYutyEmwTsDYe7w1hmRGWFvQU/CYL+ITKZU9orNCqdUfFySNuN9rke9cOAKPPbIVNRJE+6AW4FRLAsDChXzM3SkIrLZpM5KiccNsktH0hd4ENk8iI8iQSBpS4JihcK8eEspCjY0JaYHNdOo+XVgO96NCajH+FCiC4FXOwZjn6Rwz+VaJ1A2Aqh81K8BVwg/5eJi1s0GCHXGkxH8FkAQ7XhTbMbDvP+HsGfoUKt13pjYWV3tC3HZodAnpOXGI+2rs1pSTN3SAZdC4e1pHudiRuCDsSJewZ69nyhnWbfU19OZKuYlRQxmv2VSt4QKtLwxE+sTXC1fjh09sKYGGE4qJgcm/POZNyX6wN1JyjtdkjbjORwncu/GEdIgSTDwHhhlmvOOA0WLFUTQ3tB2zRktFmZBj4xfVCTlnhcwFPFk0sHqgX50a4VcMMLPLSpFQC2ipO5lYbhme9MPpJ5GhBqOBryA3tLDvymKRbi44bZA5z72rdSltsLpl4tM+UmsJdL1jGSrciKpyoJ+8+rhXPFTRE7XhiskQQFpyh/6P4tXbeKYI23qn5nzfEai+xpGeKItptKJNWexm2eA3by9Jip8+vH8eQ45Pg+O+g3QrNRlgchOjYVdimXP1ekdbmzPWBDzI9cMNLBEpkvJN3Yo2wWWGwrq7udiUmrC27jrjNT2WE/l5zkKE+O3wJR+ewR3i9TfbSi+Ht+eGs+hyyRTge3qrPc8wQXiBniNoTY0wMLIeVZgC25KuDJJmFB63dgIz4s0UzIIN+0LLHaJoCqINaOjpdWhBCOTSKSYpZURrNvKoZwZbYryfQ1r3jbKdziUtzO374VHEZNeCKca5LFbbO+1+/d0ZLfHuQXEo8RxTpGVIJYL6NKsTFA2d6o+zltKHiV6gn/Yh5Nk95tHMlZrRUtetUsVk0FOH6CgcydmCp/AKqHPX25h2MS6dhypkvjmMteCuZdYLDe2TKOiYf0wUWGqNNxnW+7/meX/il66smd36ROtGIccWgK42yPjLQ9UP41mgtW54T4uQwmFBeNYjUDqkfaqSVFcywNTo0+/t2qkh3hAckTKa2g2gLFAUtxdYQo/sy+x1SW2hlMYsh+dxAK/p1yKeUkHF6xlaq/ojAV0wt0cJVSK4H3WK4oMzKO8EcJVKSFYi8TfPF8lzQdSYz31rIWavPcipr45o8ZXysUUKERkbtl5R2tFkuKk2HUKV15NyjqA6rVMcHn7pDoQTwgEOpvRBoW4oTUpNowHMUatntKzApMYclKjTM+eeFDaR7jNk3mhJp1kmW3MZfdREaW1IBzBt7ZZBrk/eoUSFO7oYdxTm+n9QtMWat5mJXFfnrGzsuxA2TskPJczAjPg/Ies0UW3rrC7ZwTjuA91pLZKpHjTYrpHqtIW1hYd8QoYEw3NVnAizPtJLp7t/JW7HDKizogvSEdoQA+6WHtPTuQq8/9Rlc2o++pkM5Bims9x817djUwRyE2on2pS28y/XN6mbZ+OFTN2N5VlZ5DhjjmDnuaohKgpRHJsrnszY3j4jGO6faUky1k6EC9oci19dsY4fR7w49smsKNkN61P/eo4FcFIKMvVfApxjMAxYGKcEIvovtZOxt3yBH8eT9q7CHjDnyFTxSFpvsl/Oxdf5Jyw1AKC7LnHLkDaF2RiyXaEJYSDvZ0NQJOlTKPqZWzGAeeTqr2P/z8+RDI3rNt83zFKehVOKvEuW2Uqnm9TRD8XDLC4dKEcrSQ04VvbgNeaTTkIvFAg39Es7q2p+oAjYtkqeCZ6jyQotzi2Rvh3+/v4FqIbKmcMoSc5jYHvJllWe7G4PoeaeBKd9hbBYtdfFVFVrTH9K8cq2slphJvRTpfOE1ASCe7NkCuVgITiBvwkJ3tE7czZc6/+P58WHUXeRpH7xLlG+9GX2gdJmKgOM8HOKjyctaz4XEnjytzck8ZCuntkW7SUOLG0r8vZYFZHAIWcWE15nM6YPwz9P4//xw19mBwwJesKy0Rw4lLibadSmdgAXjjkrH3cEBeapNT0cE4vkp5KUhL59ktXVuv2Gu1U39luf2fxCWF53bH+qKJE7rjhhuG0RFfNQRHPJ3315itzfvwhSIUA3gzxWcKDKW5wbt63OPZFUdGtjoIFLfZaA+HqFJSFPOny/NJXudsO7ev0ivOlnmyYJptj2lZjnMmWSKY08n7aQOQRJAc7agBcAL6ekd3NGX7+OXPVmNY2aJLvPbM+oWUidCbJwHhIWCHuymno629ahkxG6JcyIurz2RcifdR0p/FPL+2rgw2mmu9/PRE0FVVNN7erVyrgBtwPGip1VYN2uNpqpaqGXmxBpHFnkS6UJq9kKH6vVOOybDsbBQYJFrRVmdUBz3pBeagaEvKOxOxqVyQoJopeYMDC5pS6h4mTP+iKqnlRUv/hOx2WCDrkSA4ISUrS+sY8bZ2B+lou1IK/sbc1i3vJt7V9daLS7be+nFI0VvybwTXeBrnY5ZUncatC/jKhpSc6cUrIWUIjI7iNwG+LpAhS2GuNR2/wy+9pzSZuTH7Io94mX5qKZPZndTqJckQXO9LvwZwh5foBNaumIW5ogK0Do2l8Ku+lirzE/s10onerjJ/X4WUSnRTtNDv+WYB64QFtq8vguc9sLauKqHdSo6ctmj+jTEOuZKe9q5ZBJyIAxEuGueV82rXQMND1A8JPt12oRe7J+90P9M94uszLgoVucO1NPpHQS6oQoSiozgX/zX9Sb0pDaE6TKRmpavo/WLcfGVtqgyjsZdNOMK6wCtIxZ+QA1ixzNMGDSU4LXwLRrBzizcQBNUuZ6juTAvQnG99uFV2oxJPLczoeJpiSaeKemF9+B+nUZEnW9TKXxlx4ErfzgeOe4PR/RTnqWaXCfyQbA98ZC/9yAwjOMpndHk0n5KV7k6Yvz5P8Px+qsazmi14ST/E1bI8r6qa00lNOaZKSWlU0InpvtOTtsD1d0m+9yplPUEjIfQHoIRC0p46Z54uefUNiRJ+AV52Z2lPRF47JFXxMOMZyvU7WZJrhbabJjJB7AQXzAfVpFh0BqAHY1Gb0cwccCZqt47AItPaJgM4umxQ4O5MMhdVpoze5PPD3fRQ3uJx3X8ATWvGm+1CA6M64wMMnvuN0nCDFGgXI0S1dsR8S2YkE1kdXT7IZydXrzTMv3BHx+gqSbwX9J2mZf8EV2WOqU81S8wpZXgTIah7t3RqF9rbwoxwOhJ1vy1y/WGAv3gC/ZnkKO10W4vmVD9s01r7TDrScc7Xz/H9ouCmRAtn93eg26r4W/b1Ho9PzjFYOjVs1Tt8fvqjOxig0cd0GOwzvflWyea9ela+kAT2e48Lw21ns04r3h3syC++ZXDKozrQ45cMkr9mYXpb+P7UX3nAB5up7PRz7PZfbZGt9L5qJo98ENPA/jj9v10Mrs9dIs28H48u/l59OH27nZ2O/rt/X/d3szSrD/imePzd4+4/a45uraLwhQcUFFVk3uQ3w2/q9zwTlS5xjD95qjoZv4FpHpq8bCmlUacl5eHQHj4+WHS4ohkXzuWzilHExrVdVko587Yq1DlGo3gAUez4NyNqxyYCDzDvG66FqrN8NYH2hudY3OflY4RWHNeGtPbDdk6tJntmzd7tcRieVN32fw6vkQfAH6pxjq8SHet4Sc0lO822fiKRve4kfn/UVJlxde0ZE9pTBHROuT6dajy8hrY2xfxT/pW2UUk2eqA7YdaoWAhxXLlGtMiPq35h4UCjS0oLXzq0VBXGpUxo0uV/23wmWsosC0oWDeS860uzbGB6AUac+4I3SpyvG4+xHViHXa8UvXvP7PlKfObz4T22aIZjmmlg0VinGo4e+lajcJOPlQNxTryPDfYRBKT/FjIWekzZzbEwJchW38dinz4zr+BUWsjfnGo8l3CBZMPPS04sVTMlaZvpuXUVlFFvpLTAKZi+btHSz/8OOi+V9PMGFtO4tV5ZehPZrYUnRcvT+0zIifmppoqCAt3bIsGrqbTu7dVU3Q3no9L7UT9Oi+p/zTFGl3o6TW0BlJOm2hPx414Xl2/Q9uegAl/AWBcutXP3lbDCxbte4IV2wH8d4lmOw2pN933F/1e5eJXhcEh6QbmlOK9ff3WeqsKi5658q/m+yq1jK1K+vHIbJ6T9jLWNDNMWX/8ERRtWr1idzW7m76tvVlD02Lncv+orzExuZB68/wORWey5rk9it/vb4CWelFv4iJCJCQfCcmdXtpqCf/HL7a6pN2OL2H5maE4JB1ezKrkKyy8qx8Is5JbYMBL6/S674keXTnD4HQ6s/YjtvXAdHU+WW1BX7PdoVlcoke86xModBttHndreWxhgtZPoRi2WAgeT7S1yQ93Xi/ScK2GklNvyER8Axjf3Nzez/xrzrf9xbLUy0PF3KuRSr1ckieNpVwUbrW9A/jtlwF8+u3DeDb2ofaXyT393Lft1jF10V2vlvCi/UdXsq/QikGVm9W0hfWtRe/1trrsmQx6dJk1nOV5OmC8pldXMAr/Q4lPKOFKG7EUism3VW+ze6ge2elHmFv3tyDMqRhUIXQ3YFbu4iDOp4JfUGP8jDzZYf3Xkc7qPWw5V3h+t7vDHxa4JAuOF9lCsuWZPctcuDWzj7FYqwOHllJvyOPMbu7BL3sN736a/u+nwff/Rv8Nxze/DL7/6ePk0+DHnx6mszTky41YBqldw+T+6ccB/fuvvoa7/Tgevfn/AAAA//8RgkyD" + return "eJzMXN9z27aTf89fsdOXrzMj6aZp5+bGN70ZxXGuurqpz1LauycWAlcSzhDAAqAV5a+/WQCkSBGUbEtqvnpIbJFcfHaxv7H0EB5xew1sY98AOOEkXsP4j+kbAIMSmcVrmKNjbwBytNyIwgmtruE/3gAA/KrzUiIstIEVU7kUaglSLy0sjF4TmdEbgIVAmdtr/8AQFFtjtRx93LbAa1gaXRbxm8Q69PnoydSU/TqjeLW5RHMZLnWZO8OErC+lVqTPPrfVJ8cFK6XL/BLXsGDSYutyEmwTsDYe7w1hmRGWFvQU/CYL+ITKZU9orNCqdUfFySNuN9rke9cOAKPPbIVNRJE+6AW4FRLAsDChXzM3SkIrLZpM5KiccNsktH0hd4ENk8iI8iQSBpS4JihcK8eEspCjY0JaYHNdOo+XVgO96NCajH+FCiC4FXOwZjn6Rwz+VaJ1A2Aqh81K8BVwg/5eJi1s0GCHXGkxH8FkAQ7XhTbMbDvP+HsGfoUKt13pjYWV3tC3HZodAnpOXGI+2rs1pSTN3SAZdC4e1pHudiRuCDsSJewZ69nyhnWbfU19OZKuYlRQxmv2VSt4QKtLwxE+sTXC1fjh09sKYGGE4qJgcm/POZNyX6wN1JyjtdkjbjORwncu/GEdIgSTDwHhhlmvOOA0WLFUTQ3tB2zRktFmZBj4xfVCTlnhcwFPFk0sHqgX50a4VcMMLPLSpFQC2ipO5lYbhme9MPpJ5GhBqOBryA3tLDvymKRbi44bZA5z72rdSltsLpl4tM+UmsJdL1jGSrciKpyoJ+8+rhXPFTRE7XhiskQQFpyh/6P4tXbeKYI23qn5nzfEai+xpGeKItptKJNWexm2eA3by9Jip8+vH8eQ45Pg+O+g3QrNRlgchOjYVdimXP1ekdbmzPWBDzI9cMNLBEpkvJN3Yo2wWWGwrq7udiUmrC27jrjNT2WE/l5zkKE+O3wJR+ewR3i9TfbSi+Ht+eGs+hyyRTge3qrPc8wQXiBniNoTY0wMLIeVZgC25KuDJJmFB63dgIz4s0UzIIN+0LLHaJoCqINaOjpdWhBCOTSKSYpZURrNvKoZwZbYryfQ1r3jbKdziUtzO374VHEZNeCKca5LFbbO+1+/d0ZLfHuQXEo8RxTpGVIJYL6NKsTFA2d6o+zltKHiV6gn/Yh5Nk95tHMlZrRUtetUsVk0FOH6CgcydmCp/AKqHPX25h2MS6dhypkvjmMteCuZdYLDe2TKOiYf0wUWGqNNxnW+7/meX/il66smd36ROtGIccWgK42yPjLQ9UP41mgtW54T4uQwmFBeNYjUDqkfaqSVFcywNTo0+/t2qkh3hAckTKa2g2gLFAUtxdYQo/sy+x1SW2hlMYsh+dxAK/p1yKeUkHF6xlaq/ojAV0wt0cJVSK4H3WK4oMzKO8EcJVKSFYi8TfPF8lzQdSYz31rIWavPcipr45o8ZXysUUKERkbtl5R2tFkuKk2HUKV15NyjqA6rVMcHn7pDoQTwgEOpvRBoW4oTUpNowHMUatntKzApMYclKjTM+eeFDaR7jNk3mhJp1kmW3MZfdREaW1IBzBt7ZZBrk/eoUSFO7oYdxTm+n9QtMWat5mJXFfnrGzsuxA2TskPJczAjPg/Ies0UW3rrC7ZwTjuA91pLZKpHjTYrpHqtIW1hYd8QoYEw3NVnAizPtJLp7t/JW7HDKizogvSEdoQA+6WHtPTuQq8/9Rlc2o++pkM5Bims9x817djUwRyE2on2pS28y/XN6mbZ+OFTN2N5VlZ5DhjjmDnuaohKgpRHJsrnszY3j4jGO6faUky1k6EC9oci19dsY4fR7w49smsKNkN61P/eo4FcFIKMvVfApxjMAxYGKcEIvovtZOxt3yBH8eT9q7CHjDnyFTxSFpvsl/Oxdf5Jyw1AKC7LnHLkDaF2RiyXaEJYSDvZ0NQJOlTKPqZWzGAeeTqr2P/z8+RDI3rNt83zFKehVOKvEuW2Uqnm9TRD8XDLC4dKEcrSQ04VvbgNeaTTkIvFAg39Es7q2p+oAjYtkqeCZ6jyQotzi2Rvh3+/v4FqIbKmcMoSc5jYHvJllWe7G4PoeaeBKd9hbBYtdfFVFVrTH9K8cq2slphJvRTpfOE1ASCe7NkCuVgITiBvwkJ3tE7czZc6/+P58WHUXeRpH7xLlG+9GX2gdJmKgOM8HOKjyctaz4XEnjytzck8ZCuntkW7SUOLG0r8vZYFZHAIWcWE15nM6YPwz9P4//xw19mBwwJesKy0Rw4lLibadSmdgAXjjkrH3cEBeapNT0cE4vkp5KUhL59ktXVuv2Gu1U39luf2fxCWF53bH+qKJE7rjhhuG0RFfNQRHPJ3315itzfvwhSIUA3gzxWcKDKW5wbt63OPZFUdGtjoIFLfZaA+HqFJSFPOny/N5WVkeff+RWrXSUJPlluzKyo1y2HOJFMcexptJzUQkgCaowctAF5IT+/gjr58H7/sSXocM0t0md+9UbfOOhFi47ggLBTUZDcUdbTrRxUldiugE3F57YmUO9UAUnakkPeXzoXRTnO9n66eCKqimt7Tq5VzBWgDjhc9ncS6l2s0Fd1CLTMn1jiyyJNIF1KzF/pbr3faMRlOjYUCi1wrSvqE4rgnvdArDG1DYXcyLpUTEkQrc2dgcElbQrXNnPFHVD2drnjxn4jNBht0JQIEJ6RsfWEdM87G9inVdEc63d+Yw7oj3ty7uhRrcdneSy8eKXor6p3oAl/rdEiTutO/fRlX0ZCaO6VgLaQUkdlB5DbA1wUqbDHEpbb7R/S155Q2Iz9mV+wRL8tHNZwyu5tCvSQJmut14Y8Y9vgCndDSFbMwR1SA1rG5FHbVx1plfmK/lDrRw03u95OMSol2mh7aMcc8cIWw0Ob1TeK0F9bGVS2uU9GRyx7VhyXWMVfa044tk5ADYSDCXfO8al7tGmh4gOIh2a/TJrRq/+yF/me6nWRlxkWxOnegnk7vINANRZJQZAT/4r+uN6EntSFMl4nUtHwdrV+Mi6+0RZVxNO6iGVdYB2gdsfDzaxAbomEAoaEEr4Vv0Qh2ZuEGmqDK9RzNhXkRiuu1D6/SZkziuZ0J1VZLNPHISS+8B/frNCLqfJtK4Ss7Dlz5s/PIcX84op/yLNUDO5EPgu2Jh/y9B4FhHE9pnCaX9kO8ytUR48//GY7XX9VwRqsNJ/mfsEKW91Vda6qwMc9MKSmdEjox/Hdy2h6o7jbZ506lrAdkPIT2jIxYUMJL98TLPYe6IUnCL8jL7qjticBjC70iHkZAW6FuN2pytdBmw0w+gIX4gvmwigyD1nzsaDR6O4KJA85U9VoCWHxCw2QQT48dGsyFQe6y0pzZm3x+uIse2ks8ruPPr3nVl6tFcGCaZ2SQ2XO/aBJGjALlatKo3o6Ib8GEbCKro9sP4Wj1Wzdipj/4wwc01fz+S7oy85I/ostSZ5ynug2mtBKcyTASvjtY9WvtzTAGGD25nL92udZRoB9cxf4EczRGUoYlE6p/MmqtHWY92Xrn6+e4hqJgJgTTZzcHoduJ+Ns2tV7Pj10xGHr1LFV7eL86YbvY2FIH9Bis81391nlofTaXPg5FtjsNTEOtJzvOK97dJInvjeWwCsP+kCOXjCoDZmH62/h+VN85gIfb6Wz082x2n63RrXQ+qiYX/MjUAP64fT+dzG4P3aINvB/Pbn4efbi9u53djn57/1+3N7M064945vD93SNuv2sOvu2CNMUOVFT05B7kd8PvKi+9E1WuMczOOarJmX99qZ55PKxppRHn5eUhEB5+fpi0OCLZ146lc0bShEZlXxaqvTO2MlS5RiN4wNGsR3fDLgfmCc8w7ZsulWozvPVx+Ebn2NxnpWOA1pyXxvQ2S7YObWb7ptVeLbFY/dRNOL+Or+AHgF+qoRAv0l3n+AkNpcNNNr6i0T1uZP5/lHNZ8TUt2VP6VkS0Drl+HSrMvAb2tk38k76TdhFJthpk+6FWKFhIsVy5xqyJT2v+YaFAYwvKGp96NNSVRmXM6FLlfxt85hoKbAsK1o3cfatLc2yceoHGnDtCt2ogr5sPcZ1Yph0vZP3b02x5yvTnM6F9tmiGY1rpYA0ZZyLOXtlWg7STD1W/sY48zw02kcQkPxZyVvrMmQ0x8GXI1l+HIh++8+9v1NqIXxyqfJdwweRDT4dOLBVzpembiDm1k1SRr+Q0gKlY/u7R0g8/Drpv5TQzxpaTeHVeGdqXmS1F57XNU9uQyIm5qaYKwsId26KBq+n07m3VM90N9+NSO1G/DEzqP02xRhd6WhGtcZbT5uHTcSMeZ9dv4LbnZ8LfDxiXbvWzt9Xwekb7nmDFdgD/XaLZTkPqTff9Rb9XufhVYXBIuoE5pXhvX7+13qrComduDFTTgZVaxk4m/Xhkss9JexlrmhmmrD8dCYo2rV7Qu5rdTd/W3qyhabGxuX8S2Ji3XEi9eX4D41JzOb/f3wAheVHr4iIyJiQfCcmdXtpqCf+XNba6JGWIb3h5xuMEdnjrqxK/sPCufiAMYm6BAS+t0+u+J3pU6QxT2enE28/v1tPY1elmtQV9rXqHZnGJDvOujaDQbbR53K3lsYXxXD/DYthiIXg8D9cmP9y3PS/MvYnn1Os3Ed8Axjc3t/cz/w71bX8tLfXyUK33aqRSL5fkaGOlF4Vbbe8AfvtlAJ9++zCejX0k/mVyTz/3bbt1TF1016slvGj/0ZXsK7RiUKVuNW1hfefRO8WtLnvmih5dZg1neZ6OJ69p5RWMsoOhxCeUcKWNWArF5Nuq9dk9ko/s9CPMrftbEOZUK6oQ2RswK3dxEOdTwS+oMX4An+yw/tNLZ/UetpwrPL/b3eEPC1ySBceLbCHZ8syeZS7cmtnHWMvVgUNLqTfkcWY39+CXvYZ3P03/99Pg+3+j/4bjm18G3//0cfJp8ONPD9NZGvLlBjSD1K5hcv/044D+/Vdf4t1+HI/e/H8AAAD//6QmZ2g=" } diff --git a/x-pack/filebeat/module/aws/s3access/_meta/fields.yml b/x-pack/filebeat/module/aws/s3access/_meta/fields.yml index 7451a258831d..0bb6d8f83268 100644 --- a/x-pack/filebeat/module/aws/s3access/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/s3access/_meta/fields.yml @@ -1,6 +1,7 @@ - name: s3access type: group release: ga + default_field: false description: > Fields for AWS S3 server access logs. fields: diff --git a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml index 1fbd4b37562f..226860698481 100644 --- a/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/vpcflow/_meta/fields.yml @@ -1,6 +1,7 @@ - name: vpcflow type: group release: beta + default_field: false description: > Fields for AWS VPC flow logs. fields: From f96ef57a39f816865bf54fdb2796764664a8c6fb Mon Sep 17 00:00:00 2001 From: martin Date: Thu, 2 Jul 2020 10:34:04 +0800 Subject: [PATCH 18/35] [MetricBeat] set tags correctly if the dimension value is ARN (#19433) * fix setting tags when dimension value is arn format in aws cloudwatch module --- CHANGELOG.next.asciidoc | 1 + .../module/aws/cloudwatch/cloudwatch.go | 7 +++++++ .../module/aws/cloudwatch/cloudwatch_test.go | 16 ++++++++++++++++ x-pack/metricbeat/module/aws/utils.go | 6 +++--- x-pack/metricbeat/module/aws/utils_test.go | 2 +- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9d8db101f4e5..331f348f9927 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -260,6 +260,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix incorrect usage of hints builder when exposed port is a substring of the hint {pull}19052[19052] - Remove dedot for tag values in aws module. {issue}19112[19112] {pull}19221[19221] - Stop counterCache only when already started {pull}19103[19103] +- Set tags correctly if the dimension value is ARN {issue}19111[19111] {pull}19433[19433] - Fix bug incorrect parsing of float numbers as integers in Couchbase module {issue}18949[18949] {pull}19055[19055] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 2cc447723bed..aac756190f2b 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -615,6 +615,13 @@ func insertTags(events map[string]mb.Event, identifier string, resourceTagMap ma subIdentifiers := strings.Split(identifier, dimensionSeparator) for _, v := range subIdentifiers { tags := resourceTagMap[v] + // some metric dimension values are arn format, eg: AWS/DDOS namespace metric + if len(tags) == 0 && strings.HasPrefix(v, "arn:") { + resourceID, err := aws.FindIdentifierFromARN(v) + if err == nil { + tags = resourceTagMap[resourceID] + } + } if len(tags) != 0 { // By default, replace dot "." using underscore "_" for tag keys. // Note: tag values are not dedotted. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index e6b5cdebe5a0..93a0c42492cc 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1407,10 +1407,14 @@ func TestInsertTags(t *testing.T) { tagValue1 := "engineering" tagKey2 := "owner" tagValue2 := "foo" + identifierContainsArn := "arn:aws:ec2:ap-northeast-1:111111111111:eip-allocation/eipalloc-0123456789abcdef,SYNFlood" + tagKey3 := "env" + tagValue3 := "dev" events := map[string]mb.Event{} events[identifier1] = aws.InitEvent(regionName, accountName, accountID) events[identifier2] = aws.InitEvent(regionName, accountName, accountID) + events[identifierContainsArn] = aws.InitEvent(regionName, accountName, accountID) resourceTagMap := map[string][]resourcegroupstaggingapi.Tag{} resourceTagMap["test-s3-1"] = []resourcegroupstaggingapi.Tag{ @@ -1425,6 +1429,12 @@ func TestInsertTags(t *testing.T) { Value: awssdk.String(tagValue2), }, } + resourceTagMap["eipalloc-0123456789abcdef"] = []resourcegroupstaggingapi.Tag{ + { + Key: awssdk.String(tagKey3), + Value: awssdk.String(tagValue3), + }, + } cases := []struct { title string @@ -1444,6 +1454,12 @@ func TestInsertTags(t *testing.T) { "aws.tags.owner", tagValue2, }, + { + "test identifier with arn value", + identifierContainsArn, + "aws.tags.env", + tagValue3, + }, } for _, c := range cases { diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index f9e0f58b16c8..4c92cb140c64 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -188,9 +188,9 @@ func GetResourcesTags(svc resourcegroupstaggingapiiface.ClientAPI, resourceTypeF } for _, resourceTag := range output.ResourceTagMappingList { - identifier, err := findIdentifierFromARN(*resourceTag.ResourceARN) + identifier, err := FindIdentifierFromARN(*resourceTag.ResourceARN) if err != nil { - err = errors.Wrap(err, "error findIdentifierFromARN") + err = errors.Wrap(err, "error FindIdentifierFromARN") return nil, err } resourceTagMap[identifier] = resourceTag.Tags @@ -199,7 +199,7 @@ func GetResourcesTags(svc resourcegroupstaggingapiiface.ClientAPI, resourceTypeF return resourceTagMap, nil } -func findIdentifierFromARN(resourceARN string) (string, error) { +func FindIdentifierFromARN(resourceARN string) (string, error) { arnParsed, err := arn.Parse(resourceARN) if err != nil { err = errors.Wrap(err, "error Parse arn") diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index c33d61c641f0..4270d52b6ef0 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -377,7 +377,7 @@ func TestFindIdentifierFromARN(t *testing.T) { } for _, c := range cases { - identifier, err := findIdentifierFromARN(c.resourceARN) + identifier, err := FindIdentifierFromARN(c.resourceARN) assert.NoError(t, err) assert.Equal(t, c.expectedIdentifier, identifier) } From 8711c7eb8488c2e0b180dd8f1c2f0b5155346c57 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 2 Jul 2020 11:22:26 +0200 Subject: [PATCH 19/35] ci: some jjbb improvements (#19588) * ci: some jjbb improvements * ci: fix build ITs docker images job --- .ci/jobs/apm-beats-update.yml | 8 +++----- .ci/jobs/beats.yml | 2 +- .ci/jobs/build-it-docker-images.yml | 3 +-- .ci/jobs/defaults.yml | 3 +-- .ci/jobs/packaging.yml | 8 +++++--- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml index f612aa5a6659..fd38576b3877 100644 --- a/.ci/jobs/apm-beats-update.yml +++ b/.ci/jobs/apm-beats-update.yml @@ -5,7 +5,6 @@ view: Beats concurrent: true project-type: multibranch - periodic-folder-trigger: 1w prune-dead-branches: true number-to-keep: 10 days-to-keep: 30 @@ -19,20 +18,19 @@ discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' discover-tags: true + head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)' disable-pr-notifications: true notification-context: 'apm-beats-update' - property-strategies: - all-branches: - - suppress-scm-triggering: true repo: 'beats' repo-owner: 'elastic' credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken ssh-checkout: credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba build-strategies: + - skip-initial-build: true - tags: ignore-tags-older-than: -1 - ignore-tags-newer-than: -1 + ignore-tags-newer-than: 30 - named-branches: - exact-name: name: 'master' diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml index 076a8a81bbe9..5e0bee6b03f1 100644 --- a/.ci/jobs/beats.yml +++ b/.ci/jobs/beats.yml @@ -6,7 +6,6 @@ view: Beats concurrent: true project-type: multibranch - periodic-folder-trigger: 1w prune-dead-branches: true number-to-keep: 10 days-to-keep: 30 @@ -19,6 +18,7 @@ discover-pr-forks-strategy: 'merge-current' discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' + head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)' discover-tags: true notification-context: "beats-ci" property-strategies: diff --git a/.ci/jobs/build-it-docker-images.yml b/.ci/jobs/build-it-docker-images.yml index 562af8d11016..ac51519e1838 100644 --- a/.ci/jobs/build-it-docker-images.yml +++ b/.ci/jobs/build-it-docker-images.yml @@ -15,8 +15,7 @@ scm: - git: url: git@github.com:elastic/beats.git - refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* - wipe-workspace: 'True' + wipe-workspace: true name: origin shallow-clone: true credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml index f93e0cf52db5..40d416f28fcf 100644 --- a/.ci/jobs/defaults.yml +++ b/.ci/jobs/defaults.yml @@ -1,4 +1,4 @@ - + --- ##### GLOBAL METADATA @@ -16,5 +16,4 @@ publishers: - email: recipients: infra-root+build@elastic.co - periodic-folder-trigger: 1w prune-dead-branches: true diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml index 848ca742aed3..5aa8967e2f77 100644 --- a/.ci/jobs/packaging.yml +++ b/.ci/jobs/packaging.yml @@ -14,7 +14,8 @@ discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' discover-tags: true - disable-pr-notifications: false + head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)' + disable-pr-notifications: true notification-context: 'beats-packaging' repo: 'beats' repo-owner: 'elastic' @@ -22,9 +23,10 @@ ssh-checkout: credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba build-strategies: + - skip-initial-build: true - tags: - ignore-tags-older-than: 30 - ignore-tags-newer-than: -1 + ignore-tags-older-than: -1 + ignore-tags-newer-than: 30 - named-branches: - exact-name: name: 'master' From d0e4f81ea9c17e1ae381f8de1a16d30ef6f32454 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 2 Jul 2020 11:22:55 +0200 Subject: [PATCH 20/35] ci: enable upstream triggering on the packaging job (#19589) --- .ci/packaging.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 4c1fa26ebdb2..81d67809272e 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -38,7 +38,7 @@ pipeline { when { beforeAgent true expression { - return isCommentTrigger() || isUserTrigger() + return isCommentTrigger() || isUserTrigger() || isUpstreamTrigger() } } stages { From 5e3fd6dc74ec6028f49018906f99db9b84cfd928 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 2 Jul 2020 12:00:12 +0200 Subject: [PATCH 21/35] ci: apm-server-update trigered only on upstream, comments, and manual triggered (#19590) --- .ci/apm-beats-update.groovy | 98 +++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/.ci/apm-beats-update.groovy b/.ci/apm-beats-update.groovy index e455c0102b34..376ceb4190fb 100644 --- a/.ci/apm-beats-update.groovy +++ b/.ci/apm-beats-update.groovy @@ -2,7 +2,7 @@ @Library('apm@current') _ pipeline { - agent { label 'linux && immutable' } + agent none environment { REPO = 'apm-server' BASE_DIR = "src/github.com/elastic/${env.REPO}" @@ -27,63 +27,67 @@ pipeline { } triggers { issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*') + upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }") } stages { - /** - Checkout the code and stash it, to use it on other stages. - */ - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - deleteDir() - gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false) - script { - dir("${BEATS_DIR}"){ - env.GO_VERSION = readFile(".go-version").trim() - def regexps =[ - "^devtools/mage.*", - "^libbeat/scripts/Makefile", - ] - env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps) - // Skip all the stages except docs for PR's with asciidoc changes only - env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true) - } - } - } - } - /** - updates beats updates the framework part and go parts of beats. - Then build and test. - Finally archive the results. - */ - stage('Update Beats') { - options { skipDefaultCheckout() } + stage('Filter build') { + agent { label 'ubuntu && immutable' } when { beforeAgent true - anyOf { - branch 'master' - branch "\\d+\\.\\d+" - branch "v\\d?" - tag "v\\d+\\.\\d+\\.\\d+*" - allOf { - expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() } - changeRequest() + expression { + return isCommentTrigger() || isUserTrigger() + } + } + /** + Checkout the code and stash it, to use it on other stages. + */ + stage('Checkout') { + steps { + deleteDir() + gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false) + script { + dir("${BEATS_DIR}"){ + env.GO_VERSION = readFile(".go-version").trim() + def regexps =[ + "^devtools/mage.*", + "^libbeat/scripts/Makefile", + ] + env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps) + // Skip all the stages except docs for PR's with asciidoc changes only + env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true) + } } - } } - steps { - withGithubNotify(context: 'Check Apm Server Beats Update') { - beatsUpdate() + /** + updates beats updates the framework part and go parts of beats. + Then build and test. + Finally archive the results. + */ + stage('Update Beats') { + options { skipDefaultCheckout() } + when { + beforeAgent true + anyOf { + branch 'master' + branch "\\d+\\.\\d+" + branch "v\\d?" + tag "v\\d+\\.\\d+\\.\\d+*" + allOf { + expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() } + changeRequest() + } + + } + } + steps { + withGithubNotify(context: 'Check Apm Server Beats Update') { + beatsUpdate() + } } } } } - // post { - // cleanup { - // notifyBuildResult() - // } - // } } def beatsUpdate() { From 794ed81680a96c8ee0c59009cc961bfbfa7acd4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 2 Jul 2020 12:12:53 +0200 Subject: [PATCH 22/35] Add descriptions to HAProxy fields in Metricbeat (#19561) --- metricbeat/docs/fields.asciidoc | 48 +++++++++++++++---- metricbeat/module/haproxy/fields.go | 2 +- .../module/haproxy/info/_meta/fields.yml | 48 +++++++++++++++---- 3 files changed, 77 insertions(+), 21 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index fb06fcfb57dc..2e6f35dfece9 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19417,6 +19417,7 @@ type: long *`haproxy.info.compress.bps.in`*:: + -- +Incoming compressed data in bits per second. type: long @@ -19426,6 +19427,7 @@ type: long *`haproxy.info.compress.bps.out`*:: + -- +Outgoing compressed data in bits per second. type: long @@ -19435,6 +19437,7 @@ type: long *`haproxy.info.compress.bps.rate_limit`*:: + -- +Rate limit of compressed data in bits per second. type: long @@ -19456,6 +19459,7 @@ type: long *`haproxy.info.connection.rate.value`*:: + -- +Number of connections in the last second. type: long @@ -19465,6 +19469,7 @@ type: long *`haproxy.info.connection.rate.limit`*:: + -- +Rate limit of connections. type: long @@ -19474,6 +19479,7 @@ type: long *`haproxy.info.connection.rate.max`*:: + -- +Maximum rate of connections. type: long @@ -19552,6 +19558,7 @@ type: long *`haproxy.info.requests.total`*:: + -- +Total number of requests. type: long @@ -19561,6 +19568,7 @@ type: long *`haproxy.info.sockets.max`*:: + -- +Maximum number of sockets. type: long @@ -19570,6 +19578,7 @@ type: long *`haproxy.info.requests.max`*:: + -- +Maximum number of requests. type: long @@ -19585,6 +19594,7 @@ type: long *`haproxy.info.pipes.used`*:: + -- +Number of used pipes during kernel-based tcp splicing. type: integer @@ -19594,6 +19604,7 @@ type: integer *`haproxy.info.pipes.free`*:: + -- +Number of free pipes. type: integer @@ -19603,6 +19614,7 @@ type: integer *`haproxy.info.pipes.max`*:: + -- +Maximum number of used pipes. type: integer @@ -19618,6 +19630,7 @@ None *`haproxy.info.session.rate.value`*:: + -- +Rate of session per seconds. type: integer @@ -19627,6 +19640,7 @@ type: integer *`haproxy.info.session.rate.limit`*:: + -- +Rate limit of sessions. type: integer @@ -19636,6 +19650,7 @@ type: integer *`haproxy.info.session.rate.max`*:: + -- +Maximum rate of sessions. type: integer @@ -19651,7 +19666,8 @@ None *`haproxy.info.ssl.rate.value`*:: + -- -None +Rate of SSL requests. + type: integer @@ -19660,7 +19676,8 @@ type: integer *`haproxy.info.ssl.rate.limit`*:: + -- -None +Rate limit of SSL requests. + type: integer @@ -19669,7 +19686,8 @@ type: integer *`haproxy.info.ssl.rate.max`*:: + -- -None +Maximum rate of SSL requests. + type: integer @@ -19684,7 +19702,8 @@ None *`haproxy.info.ssl.frontend.key_rate.value`*:: + -- -None +Key rate of SSL frontend. + type: integer @@ -19693,7 +19712,8 @@ type: integer *`haproxy.info.ssl.frontend.key_rate.max`*:: + -- -None +Maximum key rate of SSL frontend. + type: integer @@ -19702,7 +19722,8 @@ type: integer *`haproxy.info.ssl.frontend.session_reuse.pct`*:: + -- -None +Rate of reuse of SSL frontend sessions. + type: scaled_float @@ -19719,7 +19740,8 @@ None *`haproxy.info.ssl.backend.key_rate.value`*:: + -- -None +Key rate of SSL backend sessions. + type: integer @@ -19728,7 +19750,8 @@ type: integer *`haproxy.info.ssl.backend.key_rate.max`*:: + -- -MaxConnRate +Maximum key rate of SSL backend sessions. + type: integer @@ -19737,7 +19760,8 @@ type: integer *`haproxy.info.ssl.cached_lookups`*:: + -- -None +Number of SSL cache lookups. + type: long @@ -19746,7 +19770,8 @@ type: long *`haproxy.info.ssl.cache_misses`*:: + -- -None +Number of SSL cache misses. + type: long @@ -19761,6 +19786,7 @@ type: long *`haproxy.info.zlib_mem_usage.value`*:: + -- +Memory usage of zlib. type: integer @@ -19770,6 +19796,7 @@ type: integer *`haproxy.info.zlib_mem_usage.max`*:: + -- +Maximum memory usage of zlib. type: integer @@ -19779,6 +19806,7 @@ type: integer *`haproxy.info.idle.pct`*:: + -- +Percentage of idle time. type: scaled_float diff --git a/metricbeat/module/haproxy/fields.go b/metricbeat/module/haproxy/fields.go index 378af3d10ebb..2cf687fbb9d0 100644 --- a/metricbeat/module/haproxy/fields.go +++ b/metricbeat/module/haproxy/fields.go @@ -32,5 +32,5 @@ func init() { // AssetHaproxy returns asset data. // This is the base64 encoded gzipped contents of module/haproxy. func AssetHaproxy() string { - return "eJzsXW9z2zaTf59PseM3lZ+T1aR1m5nMtDON8/SSqeN4bOeeFzc3KkSuRJxJgAVAy+qnv8EfUhQFkpBF2pm5h28SS+TubxfA/sOCOoN73LyDhOSCP25eASiqUnwHJx9/u9afnLwCiFFGguaKcvYOfn0FAOC+hc88LlJ8BSATLtQ84mxJV+9gSVKpPxWYIpH4DlZE34NKUbaS7+C/T6RMT6ZwkiiVn/zPK4AlxTSW7wzxM2AkwzoofalNrgkJXuTuEw+uOrYMlaCRnLkv6hzqXChb8upDH5sOVvr6T2QoSGroiIzoW4AseKEqILngEUqJFRR97aqmvJog60ArMjvflohTzlaNLzpA6+uqyBYogC99ALsQzFmRDYTh2lIEZrD0sFeJQBIPL76j2yc8jb2cSUpJE1NOVFKpa7b/ZEZXglhgShT4NJ19+tCDWBRs/leBe/QP1ZiXuFQ8z+keieNHoyQM/8sXfWOibxkcAEnTEN4FM0jJIsX5KDhqDELwpFQqbYqGB1JR7kEQC57nGM9TvhoehCMOmngPjkUhN/Ocp+kY01MTB0e8B8eS0BTjuUDJ00JTHl4rlgXUWPTZUCLvj4XhXw65ohnOJEYDCXlRCIFMOcJAGUiMOOu10xlmXGxmGXmcLTYq3Fta7/0OfA/1QP1MHmlWZEAyXjClx8WCgEKSlYFuiMJEJQjffcYsI4/zz++/gweSFggRZw8oFMaguL3z1COjf6q3StiMYMATXewJUpLlhdr7rotwEPE6A8UVSb13dIxQefWMRnnV1qzRvtSziReqZf40IQqimo5zaIS/PaDQE8Ti44XKC2X4hg5/jm32/qjhJ5GiDz7hOwUPELrmZA0LK0DHgJSIIs4YRgqbgdSwoCourbj8o8B5cy4PMQhpyiMyjsi39G+0kS9u+Rg5AsaikCMPQ4b6f9IwgqXg2dNwWt84KlLnft280e5Xm3CNrxQidA4VKc2omrOBnGfpkVgFlefIYElTlNrTGY2W+Uk3sohnuUAZbmZCogZ/pltnu8ibHLu5hnPu4g47hYGBzX8nN7/LHY2d9jJzM+fG5Now35T75/d4s6jFiT/nNDIx3nMO7eij2mSYkcex2VUTySYFIxj1Mt2o2fIAH9MWxx6L5k7TPRCLlOlsfP3c3l4+Ade4enoaJv+0PRZR6XkPxzQunsOwJETE8+EAeZ2CwL8KlEp6J8cwNUoe3aOSnpEehHwlwEj0c5ofkNx3zoU+P9kZ01OmcIXiCE9ZheQC293xcGy6Ju5xXKqZhVI+NZp5QggzawsihtacYdYWQIzCbJyx8g+aDE/Tv4UBe8bherbBCrERnClk7eYoJGx/Yqx+j5t55/CFiBsuspd1XzQ9NGNnyuYCC4mzPOpOHWREUozny5STthvLQnqOIvLHouFAq2oAie7/PScGYKzjwQvO2I0vMa6yLBIlZh+P3xcdZZiAGLCbxTyjnn6GpzHoNv1/p3QxzzCbmx2ZY2OqAx3Ccznv8cMeGqc+E9FrGrpNQguCbafBDr2DG3RuFVESIp6mtrZvKsoD9Ob4YwxFVOEP2u9xs+aiacN6UrhbQw8mX6+n8OHLv66mcPXl8v0UPv/26epuClzY/00eKDmdzWa+jcM6vDXSVeIfvsOLzDb5tiRhsuSiNNPy1CCTKB5Q7NxgP/Lub9ZhxnzNFM2ObVzZBVoShcl2L/l0Br/XcE9BJVS6bVkqTZW8BQtUZfR1wlMsSUyBcWU+lkVWbrFUnMtHnBoCKu+cIVNzLbhXF/5l3bezXtI1RGDy+pcy5prCm18qQX74xcI0Y/njLzaZ/r5sBukbwrJX7NvrmoLJazMQSyqkAsqkIizCKbwBO0P1xJgCYTFIDpz1CaqVRCOc678GXPeWquEBk99vvlzd/fPqg8VdDdb73y7+KD+tho0LIGxjH9wuueBxo+zZ2ibem21vynoQ8UI9M6T2JoGqxk6kmkcJYS3BxJPWZa0HzlonkJRFaCyGZghfr89+1U5Aj7H+9+zXr9egBGGSapr97ZNcqWd04OVVlo5LACUVsmpaRFgnyECmfC0VEfupA5WuccBMc8ZLS73cPqPvocze1btybUQwSAGyvKyv2e67arsvgVQCElnxnQJSlaAwSmC43qPlUjNppDWqEXgWU5kTFSWUrazzcs7E+S4TpYDAnAtlHNgeVa3uJr76ENQQ9s0ooY1OPGux8Ycr79OH0mWazu/vHSS6tKwoW+nhRUYWaS+4iPN7OqBFvjD03IxzKB0+N4eNyXXfWO7lX8449xkVTuL5gqSERZSt5iRdcUFV4u+1fpIMl5zEUHGAikOvKg/evg1ucxlrZ+iiyIqUmCYj1mzwCdwEEagEHbmDpDQPW2Sg+K65cDBCdiNphjPy4AN1LOKyVc3hhLInM5PANdDKR715/cP5dmckQMldO/QDNnWZ5pxdqNbLBmIccDesfn2kqwSl2pqVLdxZSydgEx1RCrNcjbbH6lMloFRkkVKZZNqzOwhhi6qQ+LxYDcsQbDROD+8W+Yb7XrU8O/OfPBCamgMFOuSwQxEGfIymkuZ1qXkAZ8fi3qkxjjuetkTaOp1hlGE1cjnWXdO6DjShyldnHROgZtl3Qsn6ieGjihjZOJ77xnk2xwEWGJFCokvaCkHVRs/cCEVniGGvf5j4/e7i2obuVNbJEch0mI8xqCg/c5rStJW2uKJIcdZJ9uPdXQ/dRKktYR3DEpHTJukeFc8Xm/l2pc71s0fV8w9Xu1ZfTTuV0ddIDhKj2gh7WRkcjGABzAm/+Bk6vyyjbWznq5zuZZ8u+PYS5suKWEl8TVXCC7UNgImUdMWCol+niHHbqKo4bVftAfBQCL53WGMIaG5eOQYzuOXbNDTnUlLtN81Uk0AE9ng2bT6QiHQDCkVGmT3QXHXhRylFpqawwCUXtjRVTtyEaDuDzBy0aaUtkMQWapNo6yP2a5NytHdJV7dFKZc7PfmtDzwQQXkhYUG2s7oJqt3GlmLrFM1azvaCCez4vLKC80ypZR1onblJNBk3FSi72syS9lLtqTH11JP8JNfUwiBrsjFqDz8HpLOj0dbTVoPVpFAJUYAs4gVTKDRm5uawEhvKVqC4l1SVLW9T+kCzaXePvETN14ooEwuTNC2JbQcpTwtpNgHqCZtRFzDuX0BESh5Rc7xG22AgkBOhaFSkpNrIm8giSoDIekkLEvKgFcD8CnDHvHq2/WCkjv3t9VKN+s1rO7dMeFZNsBzLnadGjQJTkmtz1qxV+AQYvg+/ee0fK2oT48V759sgCoyQPgRZaqoXe4T52McOa3y6KmctGZTMOZPhLS3BKdSz2FcLvgpd7mq77yxKixib7iUmivitoiBMLlFIIAtuDlMvNnU3NHFvk5lp0zlzttjdeupfVzaQMv7RWc/+8OkfsBZUlRIBZ7VYwvWjw2TN2XcKFgjWocTNXTAN8bSF/JLQtBAIJM9T43qWNFVabMVdNNaYEAELcfyycTXSYXXjcov+7uI6xGf0ZPsH9z7VrxuHPCzj38u8vTT7s3Gd/JldvxNu0EGUYHRv6ionIcc3lPK5yec89vbmcWxn5Ox6OTwmbHnz+AgRj0Ormj+8CMgfDgP544uA/PEwkOcvAvL8MJA/vQjInw4DaRzOC8C0jk4DlTDJBVc84qn1Y8GvI0mQxHvgB4hFBBqverBJO6iW3/q+gH4+wbwgcC8IQgYbwgccfK8tsKNVKhfWRDDKvG9X8g72oWeggkd7vFJmbYvCFTXLFpsXP5/Jas1fwZC6j/4cUfI173EshEk8bTnasHrBrG73/WwH6OhJBYVh6gUHHMsIKRmUQh9eLeiUo39rdxA5anMqLTd6Ga59UgUCH/BUzCElj3DQXrNpIvlgoznMCRTvqQTo6SULVJI7oeA2HMw0TJCkKrGSzuAL05lOr2P8evWHxfwrFOye8XVb4f7T1afyRsqooiSlf++/6a+C9+Xij3/e3Oi7XfptQpqWuy/Pv/zhaBv0kBOpF5Q2gGSDAs6nwDgUuR5384kEhVLpRNy1JbZSvvvy9c5QtpTenJ33bGlcnl98uYLGI7WSbi74IsVsalJlfCRZ7m1f2b1OLuqNMctCYnwCExXlIKQ6NSnnFQfBC4WgOCRcqhOY0CjL/RUJgMufe3T2c+uDDZX8DJPb28vTPrX8fHN7DTuPUfZAUhpvywxnsBvBtpF62wP9bceDF/UHtQUwTdgkTTf7ZHbGCM5fn5uYu3ewYir1nDrj7Oz89XkrloYa38JEB/jf336+u+5V5tuGMt8eoczbu9tdUrsFvl0lmByknpG1x4M8HqNF8NKA/OnsrWEwBbrc9hmFFJ0KexhlBGR32yKZ2XmiChTn93o9LimjMmkxtf2g7e0z/eg43uDOliCLVLV7hGCYOk8ZNZQs39+mYYXEkmSFTHVrb8CXZYz//joruJ1kThfrhKbYPA9Q5CELwu+yh0NbHX/ZHnnxd6HUz+95ada2q7ekygr6znk+WKD27Vq2qU4lEhNuE39s0jj2V6dtLB+JknIXuCdGtL0HbnvCGyweecIoJoq075k0ex/6CgFapGNDWnsFFwn2zpz0qSZAPeAO3qEwxwQLRv8yO1GSxgjEnkkJ2Y/wj9tAAEPHsGw3qO+p+bcj3FZbrF3Ozi5Z6b5dr0WI5N1vjh1I8EbDg7VfRGB1OCxDwkxbhv5CJbjR33qJWk+zMcc9I8LK7WFPP1C6PTvTun6h8coKb23heVRh2XecNPYbHveyz70gb+wMteXYKfQ5kK6znh3o6tdeT4aLWe1bmZe2Ucqe6SqVEzD8badWIWAGjC4UZqZbrTL3hwi22Njs5hsYLyuMmfIlKiOPEXkrFEwurr9+//5ftu4U1IRUGr6Xl7GxVWOEXaPYDlrvQcj6j+js4h9xOY/k/L5al2xEav8BlDoSz7F8GCbVsS8Q0fRDNg+6ctgBYGj6powzNZv/U+fYpmUtLXgLr/0HY8abLQO/cK5+1arORjJXe55k5NH8fdrovXEHy1VizhovWzOIqg+8JKTTjdcwKSMPxsOtTW//zVEa0Bk5cW04FumTz256J4zJiZ95wvy7nj1iPfv/bR17Www2c9r1Y2MMJ0V+0vpQVfRsPrQk1PzGnVQ8PzGAYr5mJwGFlO0kHWeKfywyk+qQ2Jyp1AmCViNfugKXXV5B7fgdXu0oq/Vfxr5WylxsHLSA2I3udcMOhOmGytr7H0LxLEnaXsQ8Cs/vJE0Px2ON2TiIPlpDmRNBMlRuu97ggveo1ogMXptU+0/7qR6rP//D/aEV9efZmyFK70cJ8cFR33EB1noapxmyLjx7vDBst1LLNA/RQaAeYG/Kd9YumwBb5v3AAHfXwEEAWxfCwBB9i6IsznlXhY0zdtaG/ah/hUCwG4EAV3KIkK0upW3PxxtMSl6IaITDDCSOPb9hsyWu8NEXvYR0uFvK1RuBjABd4r76vwAAAP//Au56Dg==" + return "eJzsXe1z2zaT/56/Ysdfqjwnq0nrNjOZaWca5+klk8TxxM49H25uVIhcmTiDAAuAltW//gYvpCgKFCGLtHPTR1/a6GXx28Vi37BLn8Itrl9DRgop7tfPADTVDF/DybvfLs07J88AUlSJpIWmgr+GX58BAPhP4ZNIS4bPAFQmpJ4ngi/pzWtYEqbMuxIZEoWv4YaY76DWlN+o1/DfJ0qxkymcZFoXJ//zDGBJkaXqtSV+Cpzk2ARlXnpdGEJSlIV/J4CriS1HLWmiZv6D5grNVShfivrN0DJ7ljKv/0SOkjBLR+bEfAXIQpS6BlJIkaBSWEMxr23RVK82yCbQmszWpxViJvhN64M9oM3roswXKEEsQwD3IZjzMh8Iw6WjCNxi6VleZxJJOjz7nm4f8zQNrkwYJW1MBdFZLa7Z7i9zeiOJA6ZliQ+T2fu3PYhlyed/lrhD/1CJBYkrLYqC7pA4fjcqwvC/YtG3J+YrgwMgjMWsXXKLlCwYzkfB0VggBg+jShtTNDyQmnIPglSKosB0zsTN8CA8cTDEe3AsSrWeF4KxMdTTEAdPvAfHklCG6VyiEqw0lIeXilsCGkv02VCibo+FET4OhaY5zhQmAzF5XkqJXHvCQDkoTATvtdM55kKuZzm5ny3WOt5bOu/9GkI/6oH6idzTvMyB5KLk2uyLAwGlIjcWuiUKE50hfPcJ85zczz+9+Q7uCCsREsHvUGpMQQv3zecBHsOq3slhO4KBQHSxw0hFVpR657N9hKOINxfQQhMW/MaeHapePbtRvRpn1kpfGW0Spe7QnzZESXTbcQ6N8Lc7lEZBHD5R6qLUdt3Y7S+wy94ftf0k0fQuxPxexiOYbjhZu4RjYM+GVIgSwTkmGtuB1LCg6lU6cYV3QYi2Lg+xCYyJhIzD8hX9C13ki5t1LB8Re1GqkbchR/N/yi4ESynyh+F0vnFUpN79er0x7teYcIOvYiJWh0pGc6rnfCDnWXkkXkMVBXJYUobKeDor0So/2Y8sEXkhUcWbmZioIZzpNpddFO0V968av/K+1WGrMDCy+X/PE5GbTKcSMaaQEk1swEC1ggKlD3oifVbYbw+J+XOpb8SgmI2/m1vtHxn6F6IR7ELO1j8Qf8slURE+s+OdjI7A5DGPho1bHy18a5pXyq3pYkTpw9TsKTSshh2JMSf3IyOs/ILRoXiMtcq7lGwEl1ole4fh6coijkVzbegeiEUpNhtfPldXHx+Aa1w5PQxTWNePRVTp9+GYxsVzGJaMyHQ+HKCg+5L4Z4lKq6ByPDTsdAqxCTrrRfZjUSK5Ra0COjFc/FutESmVMaFESqWgxQG1nb0o+kKKvSkd5RpvUB6bK9lszvIEaSlNCHmLkiM7XRDziU4KUAWjSXddtYl4KbE7CBoIsVnDIT7SfhyFZ1d/NpLsO1eo1EPj0wcEpbOusHAAKXzxAYtnqRGmx2yORdcVAQ6Frg78PMZoYKOrjtwWXnRRQqn4utb/A90xQUGP7d2B9WhK8xBwj6Y4B4JbSsE18m6XEpOlPjA1vcX1fK9CQYSI2lj2ZFQfcL0lpor3yJSvhtuX+w0CttrX2+NAezsyl1gqnBXJ/rxaJYRhOl8yQbq+WN29FSiTcAJ1AJPVcbfo2hweYpwXJLn9m2qxZz1GWkHYT6rN3eDbeBOSZLZhQdyWe+rNQ1wV2IzULAd+tR5U85wGer3GwuQWi40L/mJ0Mc8xn9v77WNTlAOjhREDhU/NW3uxtHx+C6F/Ho+rvrxIWcgy91rk/Za4rzPN/cijNBDANoU8a6NTemvhg3surzTRChLBmLuutZeEA7RbhqNgTXQZTsRvcb0Ssu0heqR0ZenB5OvlFN5+/tfFFC4+f3wzhU+/vb+4noKQ7v8md5Q8n81moV6QJrwV0pssvM8PLeA4kjBZClkZU/XcIlMo71BufcG9FWxZacJMxYobZRgUaEUUJpv2oOcz+L2Bewo6o8p32lBlbw86sEB9M7rKBMOKxBS40PZtVebVrXm9cvUTL4aIy1TBkeu5YTwoi7DF6GuWquhaIjB58UsV8Ezh5S81Iz/84mDavfzxF1cT+77q7+vbwqr999trhIXJC7sRSyqVBsqVJjzBKbwEp6FGMaZATEwgQPA+Ro2QaIJz868Bz72jateAye9fPl9c//PircNdb9ab384/VO/W2yYkEL52P9wcueh9o/zROuHe2E4mynsQiVI/MqTuvq/6gpAoPU8ywjsimgedy0Zbs7NOoChPcHOH+fXy9FfjBMwem/+e/vr1ErQkXFFDs78jXmj9BJ6+uo+qAFRUjOvftoiwypCDYmKlNJG7qR1VvhfMqjkXlaVebn5jvkO5+1bvyXURwai3GsbuKyA1g0TV604Bqc5QWiFwXO3QqnISy60VjcTTlKqC6CSj/MY5L+9MvO+yUQpILITU1oHtUDXibuNrbkEDYZ9GSWN00lmHjT9ceO/fVi7TDvN87yHRpVuK8huzvcjJgvWCS4S4pQNa5HNLz2ucR+nxeR22Jtd/4lav/uWNc59RESSdLwgjPKH8Zk7YjZBUZ+HxmQfx8FGQFOoVoF5h+O6V6M7Fsa6bz8u8ZMT2jfJQc0hM2RS1pCM3BVbmYYMMtNg2Fx5GTIuDyZ/IXQjUsYir7mOPE6o2+1yBMEBrH/XyxQ9nh9amR5VwsyFoG2pvS9BW19k4d/7v6E2GSm/MygburKO5u42OaI15oUdr3AiJElBpsmBUZbnx7B5C3KEqFT4uVrtkDDaassOb5b7hUQZbVGnqP7kjlNkZMRNyuK2IA/4YHXEf3fUWPxb3Vml03P105dlOdYZxWhzDleG9QDOqQ+XhMQGaJeM6aIaPKlLk43juL96z+RVggQnxN0cKk1JSvTaam6DsvQcB+IeN36/PL13oTlWTHIHchPmu2+XUS8rQ1sbiypLh7qVAk+y76+seupnWG8ImhiWyoG3SPSKeL9bzzUmdm9+OcQ2xR+xGfA3p1EbfIDmIjfqi8ml5qJpWYhmwQ9vpI7STuoU2sV2ocrqTffrgO0i40d1WEV9RnYlSbwJgohS94VHRrxfEuL2ZdZy2LfYIeCil2Jm/GwKa1yu/wAyuxCYNLYRS1PhNq2oKiMQez2bMBxLJ1qBR5pS7Z1TUg1UJo8j1FBa4FNKVpirFzYixM8jt7GQnbYkkdVDbRDt/4j62KUf3zEr9tYQJtTVm1fmDOyKpKBUsyEar26C6bWzFtknRnOXsLpjAls+rKjiPlFo2gTYXt4kmF7YC5U6bPdJBqj01pp56UpjkijoYZEXWVuzxo50mOxrtPF3s9N6CzogG5IkouUZpMHOvw1quKb8BLYKk6mx5k9JHmk13exQkaj/WRNtYmDDWaGHwm1SwUtlLgGbCZsUFXIQPEFFKJNROTBobDAQKIjVNSkbqi7yJKpMMiGqWtCAjd0YAPCwAP7nbc+0HIw0sbV7f3pySDc9qBdt0qLZqFMhIYcxZu1YRYuDxJoJ4LxtPPpDTBVFigvQuylJTc9gTLMaeJG+ss69y1pFBqUJwFd9XE51CPYp9deDr0OW6cfvOE1am2HYvKdEkbBUl4WqJUgFZCPt8jMW66YYm/gFhM2M6Z94W+68+D58rF0hZ/+itZ3/49A9YSaorjkDwRizhx0pgshL8Ow0LBOdQ0vYtmIH4vIP8klBWSgRSFMy6niVl2rCthY/GWgoRcRDHLxvXOx1XN66u6K/PL2N8Rk+2f1zjtUcel/HvZN5Bmv3ZuEn+7K3fibDoIMkwubV1lZOYmTCtQ27yMad+X96P7Yy8Xa+2x4YtL+/vIRFpbFXzhycB+cNhIH98EpA/Hgby7ElAnh0G8qcnAfnTYSCtw3kCmM7RGaAKJoUUWiSCOT8W/YSpDEm6A36AWESi9aoHm7SDavmdj4DpXyd6LYi8C4KYzYb4DYfQk2jcblXChRWRnPLgA/OCm33owGD0bo9XymxcUfii5gETHuMWFhvDx/GQ9g+cHVHytY/mLSXWcxFuqSfM6rYfuXng1OTRhiNE+JGmbS52FOPwasFePvqvdgfho6FTrLro5bgKcTXQE1AGnRvanM940EGzaSP5aKM5zBhMcCoBenrJIoXkJxT8hYNVwwwJ05njdAafucl0eh3j14sPDvOvUPJbLlZdhfv3F++rL1JONSWM/rX78NYa3ufzD//88sV826ffNqTp+PbHs88fPG2LHgpin/pkDCBZo4SzKXABZWH23b6jQKPSJhH3bYmdlK8/f722lB2ll6dnPVcaH8/OP19A6yeNkm4hxYJhPrWpMt6TvAi2r2y/Ts6bjTHLUmF6AhOdFCCVfm5TzgsBUpQaQQvIhNInMKFJXoQrEgAff+6R2c+dP2yJ5GeYXF19fN4nlp+/XF3C1s8ovyOMppsywylsR7BdpF71QH+154fnzR8aC2CbsAlj610yW3sEZy/ObMzdu1kpVUanTgU/PXtx1omlJcZXMDEB/vdXn64ve4X5qiXMV0cI8+r6apvUdoFvWwg2B2lmZN3xoEjHaBH8aEH+dPrKLjAFutz0GcUUnUo3jDICsutNkczePFENWohbcx6XlFOVdZjaftDu6zPz03G8wbUrQZZMd3uEaJgmTxk1lKweyWlgxcSS5Aa53i+9g4uNnYs9wiNJHeNOybwsVhll2J4HKIuYAxF22cOhrcdfNiMv4S6U5vxekGbjunpDqqqgb83zwQKNbze8TU0qkdlwm4Rjk9bYX5O2tXwkyapb4J4Y0fUe+OuJYLB45ISRfYxl551Ju/ehrxBgWDo2pHWv6CLBzsxJn2gixAN+8A6lHRMsOf3T3kQpmiIQN5MScx8R3reBAMbuYdVu0LxTC19H+Ku21LicrVuyyn37XosYzvc/DHwgxlsND85+EYn1cFiOhNu2DPOBznBtPg0SdZ5mbcc9E8Kr6+FAPxDbzM50nl9oPRAkWFt4HFG45fdMGocNj3/Q7U6QN3aG2jF2Cn0OZN+s5x50zddOT4aPWd2D9peuUcrNdFXCidj+rqlViNCA0ZnC3Har1eb+EMYWa5fdfAP75ZixKl+hsvxYljdMweT88uv3b/7l6k5RTUiV4Xt6HltXNZbZFUpsPJO6fxK//rto2/hHPM4jOb+vziVblrr/plUTSWAsH4ZJddwDRAz9mMuDfTnsADAMfVvGmdrL/6l3bNOqlhZ9hdf9N8DG05YRn5vXqDpbznzteZKTe/vv563eGz9YrjM7a7zszCDqPvCKkEk3XsCkijy4iLc2vf03R0nAZOTEt+E4pA+e3QwqjM2JH1lh/l3PHrGe/betY2+KwVanfT82pnBSFiedP6qLnu0fLQm1f7ZUaVGcWECpWPGTiELKRknHUfF3ZW5THZLamUqTIBgxiqUvcLnjFdWOv8erHWW1/sva11qYi7WHFhG70Z1u2IEwfaGq8fyHWDxLwrqLmEfh+Z0wdjgeZ8zGQfTOGcqCSJKj9tf1Fhe8Qb1C5PDCptp/uHfNXv3xH/4fRlB/nL4covR+FBNvPfUtF+Csp3WaMecicMcLw3Yrdah5jAwi5QA7Kr+3dtkG2KH3AwPcPgMHAew8CANDDB2KqjgXPBUuztg6G+6t/hMC0W4EIlzJIUx2upSuO59gMKlEKZMRhhlImgb+LNmGuMb7UPQS0+HuKNdPBLIM7GP32f8FAAD//xOaAGg=" } diff --git a/metricbeat/module/haproxy/info/_meta/fields.yml b/metricbeat/module/haproxy/info/_meta/fields.yml index ba7ce5a88f1a..fb4474db8987 100644 --- a/metricbeat/module/haproxy/info/_meta/fields.yml +++ b/metricbeat/module/haproxy/info/_meta/fields.yml @@ -153,14 +153,17 @@ - name: in type: long description: > + Incoming compressed data in bits per second. - name: out type: long description: > + Outgoing compressed data in bits per second. - name: rate_limit type: long description: > + Rate limit of compressed data in bits per second. - name: connection type: group @@ -175,14 +178,17 @@ - name: value type: long description: > + Number of connections in the last second. - name: limit type: long description: > + Rate limit of connections. - name: max type: long description: > + Maximum rate of connections. - name: current type: long @@ -221,14 +227,17 @@ - name: requests.total type: long description: > + Total number of requests. - name: sockets.max type: long description: > + Maximum number of sockets. - name: requests.max type: long description: > + Maximum number of requests. - name: pipes type: group @@ -237,14 +246,17 @@ - name: used type: integer description: > + Number of used pipes during kernel-based tcp splicing. - name: free type: integer description: > + Number of free pipes. - name: max type: integer description: > + Maximum number of used pipes. - name: session type: group @@ -253,14 +265,17 @@ - name: rate.value type: integer description: > + Rate of session per seconds. - name: rate.limit type: integer description: > + Rate limit of sessions. - name: rate.max type: integer description: > + Maximum rate of sessions. - name: ssl @@ -269,15 +284,18 @@ fields: - name: rate.value type: integer - description: + description: > + Rate of SSL requests. - name: rate.limit type: integer - description: + description: > + Rate limit of SSL requests. - name: rate.max type: integer - description: + description: > + Maximum rate of SSL requests. - name: frontend type: group @@ -285,16 +303,19 @@ fields: - name: key_rate.value type: integer - description: + description: > + Key rate of SSL frontend. - name: key_rate.max type: integer - description: + description: > + Maximum key rate of SSL frontend. - name: session_reuse.pct type: scaled_float format: percent - description: + description: > + Rate of reuse of SSL frontend sessions. - name: backend type: group @@ -302,17 +323,21 @@ fields: - name: key_rate.value type: integer - description: + description: > + Key rate of SSL backend sessions. - name: key_rate.max type: integer - description: MaxConnRate + description: > + Maximum key rate of SSL backend sessions. - name: cached_lookups type: long - description: + description: > + Number of SSL cache lookups. - name: cache_misses type: long - description: + description: > + Number of SSL cache misses. - name: zlib_mem_usage @@ -323,12 +348,15 @@ - name: value type: integer description: > + Memory usage of zlib. - name: max type: integer description: > + Maximum memory usage of zlib. - name: idle.pct type: scaled_float format: percent description: > + Percentage of idle time. From da1e55f3c442b80508c613e3619e5033f38247d6 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 2 Jul 2020 14:18:45 +0200 Subject: [PATCH 23/35] introduce journalbeat/pkg in order to provide reusable shared code (#19581) This PR moves some functionality into journalbeat/pkg for reuse: - field conversion - adding field filters - low-level reader The change is mostly moving code around with minor cleanups (e.g. by introducing more dedicated types and tests). Functionality from original read is split into: reader, matchers, and conversion --- NOTICE.txt | 422 +++++++++--------- go.mod | 1 + journalbeat/config/config.go | 38 -- journalbeat/config/config_test.go | 94 ---- journalbeat/input/config.go | 13 +- journalbeat/pkg/journalfield/conv.go | 119 +++++ journalbeat/pkg/journalfield/conv_test.go | 119 +++++ journalbeat/pkg/journalfield/default.go | 88 ++++ journalbeat/pkg/journalfield/default_other.go | 46 ++ journalbeat/pkg/journalfield/matcher.go | 120 +++++ journalbeat/pkg/journalfield/matcher_test.go | 80 ++++ journalbeat/pkg/journalread/mode.go | 57 +++ journalbeat/pkg/journalread/mode_test.go | 58 +++ journalbeat/pkg/journalread/reader.go | 199 +++++++++ journalbeat/reader/config.go | 9 +- journalbeat/reader/fields.go | 75 ---- journalbeat/reader/journal.go | 269 ++--------- journalbeat/reader/journal_test.go | 184 -------- 18 files changed, 1159 insertions(+), 832 deletions(-) delete mode 100644 journalbeat/config/config_test.go create mode 100644 journalbeat/pkg/journalfield/conv.go create mode 100644 journalbeat/pkg/journalfield/conv_test.go create mode 100644 journalbeat/pkg/journalfield/default.go create mode 100644 journalbeat/pkg/journalfield/default_other.go create mode 100644 journalbeat/pkg/journalfield/matcher.go create mode 100644 journalbeat/pkg/journalfield/matcher_test.go create mode 100644 journalbeat/pkg/journalread/mode.go create mode 100644 journalbeat/pkg/journalread/mode_test.go create mode 100644 journalbeat/pkg/journalread/reader.go delete mode 100644 journalbeat/reader/journal_test.go diff --git a/NOTICE.txt b/NOTICE.txt index 8a2746cb5647..97ddd49332dc 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13547,6 +13547,217 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/urso/sderr +Version: v0.0.0-20200210124243-c2a16f3d43ec +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/vmware/govmomi Version: v0.0.0-20170802214208-2cad15190b41 @@ -36368,217 +36579,6 @@ Contents of probable licence file $GOMODCACHE/github.com/urso/qcgen@v0.0.0-20180 limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/urso/sderr -Version: v0.0.0-20200210124243-c2a16f3d43ec -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/vbatts/tar-split Version: v0.11.1 diff --git a/go.mod b/go.mod index aa84ec00f940..44231c67d658 100644 --- a/go.mod +++ b/go.mod @@ -143,6 +143,7 @@ require ( github.com/stretchr/testify v1.6.1 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786 + github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 // indirect diff --git a/journalbeat/config/config.go b/journalbeat/config/config.go index a5d9127641de..1771a2a4e7c9 100644 --- a/journalbeat/config/config.go +++ b/journalbeat/config/config.go @@ -21,56 +21,18 @@ package config import ( - "fmt" - "github.com/elastic/beats/v7/libbeat/common" ) -// SeekMode is specifies how a journal is read -type SeekMode uint8 - // Config stores the configuration of Journalbeat type Config struct { Inputs []*common.Config `config:"inputs"` RegistryFile string `config:"registry_file"` } -const ( - // SeekInvalid is an invalid value for seek - SeekInvalid SeekMode = iota - // SeekHead option seeks to the head of a journal - SeekHead - // SeekTail option seeks to the tail of a journal - SeekTail - // SeekCursor option seeks to the position specified in the cursor - SeekCursor - - seekHeadStr = "head" - seekTailStr = "tail" - seekCursorStr = "cursor" -) - var ( // DefaultConfig are the defaults of a Journalbeat instance DefaultConfig = Config{ RegistryFile: "registry", } - - seekModes = map[string]SeekMode{ - seekHeadStr: SeekHead, - seekTailStr: SeekTail, - seekCursorStr: SeekCursor, - } ) - -// Unpack validates and unpack "seek" config option -func (m *SeekMode) Unpack(value string) error { - mode, ok := seekModes[value] - if !ok { - return fmt.Errorf("invalid seek mode '%s'", value) - } - - *m = mode - - return nil -} diff --git a/journalbeat/config/config_test.go b/journalbeat/config/config_test.go deleted file mode 100644 index f842b812b1fd..000000000000 --- a/journalbeat/config/config_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package config - -import ( - "testing" -) - -func TestUnpack(t *testing.T) { - - tests := []struct { - mode SeekMode - modeStr string - }{ - { - mode: SeekHead, - modeStr: seekHeadStr, - }, - { - mode: SeekTail, - modeStr: seekTailStr, - }, - { - mode: SeekCursor, - modeStr: seekCursorStr, - }, - } - - for _, tc := range tests { - tc := tc - - t.Run(tc.modeStr, func(t *testing.T) { - t.Parallel() - - m := SeekInvalid - err := m.Unpack(tc.modeStr) - if err != nil { - t.Fatal(err) - } - - if m != tc.mode { - t.Errorf("wrong mode, expected %v, got %v", tc.mode, m) - } - }) - } -} - -func TestUnpackFailure(t *testing.T) { - - tests := []struct { - modeStr string - }{ - { - modeStr: "invalid", - }, - { - modeStr: "", - }, - { - modeStr: "unknown", - }, - } - - for _, tc := range tests { - tc := tc - - t.Run(tc.modeStr, func(t *testing.T) { - t.Parallel() - - m := SeekInvalid - err := m.Unpack(tc.modeStr) - if err == nil { - t.Errorf("an error was expected, got %v", m) - } - }) - } -} diff --git a/journalbeat/input/config.go b/journalbeat/input/config.go index 63c31ccfce7e..6c202030b357 100644 --- a/journalbeat/input/config.go +++ b/journalbeat/input/config.go @@ -20,7 +20,8 @@ package input import ( "time" - "github.com/elastic/beats/v7/journalbeat/config" + "github.com/elastic/beats/v7/journalbeat/pkg/journalfield" + "github.com/elastic/beats/v7/journalbeat/pkg/journalread" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/libbeat/processors" @@ -38,11 +39,11 @@ type Config struct { // MaxBackoff is the limit of the backoff time. MaxBackoff time.Duration `config:"max_backoff" validate:"min=0,nonzero"` // Seek is the method to read from journals. - Seek config.SeekMode `config:"seek"` + Seek journalread.SeekMode `config:"seek"` // CursorSeekFallback sets where to seek if registry file is not available. - CursorSeekFallback config.SeekMode `config:"cursor_seek_fallback"` + CursorSeekFallback journalread.SeekMode `config:"cursor_seek_fallback"` // Matches store the key value pairs to match entries. - Matches []string `config:"include_matches"` + Matches []journalfield.Matcher `config:"include_matches"` // SaveRemoteHostname defines if the original source of the entry needs to be saved. SaveRemoteHostname bool `config:"save_remote_hostname"` @@ -59,8 +60,8 @@ var ( DefaultConfig = Config{ Backoff: 1 * time.Second, MaxBackoff: 20 * time.Second, - Seek: config.SeekCursor, - CursorSeekFallback: config.SeekHead, + Seek: journalread.SeekCursor, + CursorSeekFallback: journalread.SeekHead, SaveRemoteHostname: false, } ) diff --git a/journalbeat/pkg/journalfield/conv.go b/journalbeat/pkg/journalfield/conv.go new file mode 100644 index 000000000000..47214c3f91d9 --- /dev/null +++ b/journalbeat/pkg/journalfield/conv.go @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalfield + +import ( + "fmt" + "strconv" + "strings" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +// FieldConversion provides the mappings and conversion rules for raw fields of journald entries. +type FieldConversion map[string]Conversion + +// Conversion configures the conversion rules for a field. +type Conversion struct { + Names []string + IsInteger bool + Dropped bool +} + +// Converter applis configured conversion rules to journald entries, producing +// a new common.MapStr. +type Converter struct { + log *logp.Logger + conversions FieldConversion +} + +// NewConverter creates a new Converter from the given conversion rules. If +// conversions is nil, internal default conversion rules will be applied. +func NewConverter(log *logp.Logger, conversions FieldConversion) *Converter { + if conversions == nil { + conversions = journaldEventFields + } + + return &Converter{log: log, conversions: conversions} +} + +// Convert creates a common.MapStr from the raw fields by applying the +// configured conversion rules. +// Field type conversion errors are logged to at debug level and the original +// value is added to the map. +func (c *Converter) Convert(entryFields map[string]string) common.MapStr { + fields := common.MapStr{} + var custom common.MapStr + + for entryKey, v := range entryFields { + if fieldConversionInfo, ok := c.conversions[entryKey]; !ok { + if custom == nil { + custom = common.MapStr{} + } + normalized := strings.ToLower(strings.TrimLeft(entryKey, "_")) + custom.Put(normalized, v) + } else if !fieldConversionInfo.Dropped { + value, err := convertValue(fieldConversionInfo, v) + if err != nil { + value = v + c.log.Debugf("Journald mapping error: %v", err) + } + for _, name := range fieldConversionInfo.Names { + fields.Put(name, value) + } + } + } + + if len(custom) != 0 { + fields.Put("journald.custom", custom) + } + + return fields +} + +func convertValue(fc Conversion, value string) (interface{}, error) { + if fc.IsInteger { + v, err := strconv.ParseInt(value, 10, 64) + if err != nil { + // On some versions of systemd the 'syslog.pid' can contain the username + // appended to the end of the pid. In most cases this does not occur + // but in the cases that it does, this tries to strip ',\w*' from the + // value and then perform the conversion. + s := strings.Split(value, ",") + v, err = strconv.ParseInt(s[0], 10, 64) + if err != nil { + return value, fmt.Errorf("failed to convert field %s \"%v\" to int: %v", fc.Names[0], value, err) + } + } + return v, nil + } + return value, nil +} + +// helpers for creating a field conversion table. + +var ignoredField = Conversion{Dropped: true} + +func text(names ...string) Conversion { + return Conversion{Names: names, IsInteger: false, Dropped: false} +} + +func integer(names ...string) Conversion { + return Conversion{Names: names, IsInteger: true, Dropped: false} +} diff --git a/journalbeat/pkg/journalfield/conv_test.go b/journalbeat/pkg/journalfield/conv_test.go new file mode 100644 index 000000000000..a6514a955455 --- /dev/null +++ b/journalbeat/pkg/journalfield/conv_test.go @@ -0,0 +1,119 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build linux,cgo + +package journalfield + +import ( + "testing" + + "github.com/coreos/go-systemd/v22/sdjournal" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestConversion(t *testing.T) { + tests := map[string]struct { + fields map[string]string + want common.MapStr + }{ + "field name from fields.go": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_BOOT_ID: "123456", + }, + want: common.MapStr{ + "host": common.MapStr{ + "boot_id": "123456", + }, + }, + }, + "'syslog.pid' field without user append": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "pid": int64(123456), + }, + }, + }, + "'syslog.priority' field with junk": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_PRIORITY: "123456, ", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "priority": int64(123456), + }, + "log": common.MapStr{ + "syslog": common.MapStr{ + "priority": int64(123456), + }, + }, + }, + }, + "'syslog.pid' field with user append": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456,root", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "pid": int64(123456), + }, + }, + }, + "'syslog.pid' field empty": { + fields: map[string]string{ + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "", + }, + want: common.MapStr{ + "syslog": common.MapStr{ + "pid": "", + }, + }, + }, + "custom field": { + fields: map[string]string{ + "my_custom_field": "value", + }, + want: common.MapStr{ + "journald": common.MapStr{ + "custom": common.MapStr{ + "my_custom_field": "value", + }, + }, + }, + }, + "dropped field": { + fields: map[string]string{ + "_SOURCE_MONOTONIC_TIMESTAMP": "value", + }, + want: common.MapStr{}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + log := logp.NewLogger("test") + converted := NewConverter(log, nil).Convert(test.fields) + assert.Equal(t, test.want, converted) + }) + } +} diff --git a/journalbeat/pkg/journalfield/default.go b/journalbeat/pkg/journalfield/default.go new file mode 100644 index 000000000000..a8b3860e9561 --- /dev/null +++ b/journalbeat/pkg/journalfield/default.go @@ -0,0 +1,88 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build linux,cgo + +package journalfield + +import "github.com/coreos/go-systemd/v22/sdjournal" + +// journaldEventFields provides default field mappings and conversions rules. +var journaldEventFields = FieldConversion{ + // provided by systemd journal + "COREDUMP_UNIT": text("journald.coredump.unit"), + "COREDUMP_USER_UNIT": text("journald.coredump.user_unit"), + "OBJECT_AUDIT_LOGINUID": integer("journald.object.audit.login_uid"), + "OBJECT_AUDIT_SESSION": integer("journald.object.audit.session"), + "OBJECT_CMDLINE": text("journald.object.cmd"), + "OBJECT_COMM": text("journald.object.name"), + "OBJECT_EXE": text("journald.object.executable"), + "OBJECT_GID": integer("journald.object.gid"), + "OBJECT_PID": integer("journald.object.pid"), + "OBJECT_SYSTEMD_OWNER_UID": integer("journald.object.systemd.owner_uid"), + "OBJECT_SYSTEMD_SESSION": text("journald.object.systemd.session"), + "OBJECT_SYSTEMD_UNIT": text("journald.object.systemd.unit"), + "OBJECT_SYSTEMD_USER_UNIT": text("journald.object.systemd.user_unit"), + "OBJECT_UID": integer("journald.object.uid"), + "_KERNEL_DEVICE": text("journald.kernel.device"), + "_KERNEL_SUBSYSTEM": text("journald.kernel.subsystem"), + "_SYSTEMD_INVOCATION_ID": text("systemd.invocation_id"), + "_SYSTEMD_USER_SLICE": text("systemd.user_slice"), + "_UDEV_DEVLINK": text("journald.kernel.device_symlinks"), + "_UDEV_DEVNODE": text("journald.kernel.device_node_path"), + "_UDEV_SYSNAME": text("journald.kernel.device_name"), + sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: integer("process.audit.login_uid"), + sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: text("process.audit.session"), + sdjournal.SD_JOURNAL_FIELD_BOOT_ID: text("host.boot_id"), + sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: text("process.capabilites"), + sdjournal.SD_JOURNAL_FIELD_CMDLINE: text("process.cmd"), + sdjournal.SD_JOURNAL_FIELD_CODE_FILE: text("journald.code.file"), + sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: text("journald.code.func"), + sdjournal.SD_JOURNAL_FIELD_CODE_LINE: integer("journald.code.line"), + sdjournal.SD_JOURNAL_FIELD_COMM: text("process.name"), + sdjournal.SD_JOURNAL_FIELD_EXE: text("process.executable"), + sdjournal.SD_JOURNAL_FIELD_GID: integer("process.uid"), + sdjournal.SD_JOURNAL_FIELD_HOSTNAME: text("host.hostname"), + sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: text("host.id"), + sdjournal.SD_JOURNAL_FIELD_MESSAGE: text("message"), + sdjournal.SD_JOURNAL_FIELD_PID: integer("process.pid"), + sdjournal.SD_JOURNAL_FIELD_PRIORITY: integer("syslog.priority", "log.syslog.priority"), + sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: integer("syslog.facility", "log.syslog.facility.name"), + sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: text("syslog.identifier"), + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: integer("syslog.pid"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: text("systemd.cgroup"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: integer("systemd.owner_uid"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: text("systemd.session"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: text("systemd.slice"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: text("systemd.unit"), + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: text("systemd.user_unit"), + sdjournal.SD_JOURNAL_FIELD_TRANSPORT: text("systemd.transport"), + sdjournal.SD_JOURNAL_FIELD_UID: integer("process.uid"), + + // docker journald fields from: https://docs.docker.com/config/containers/logging/journald/ + "CONTAINER_ID": text("container.id_truncated"), + "CONTAINER_ID_FULL": text("container.id"), + "CONTAINER_NAME": text("container.name"), + "CONTAINER_TAG": text("container.log.tag"), + "CONTAINER_PARTIAL_MESSAGE": text("container.partial"), + + // dropped fields + sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: ignoredField, // saved in the registry + sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: ignoredField, // saved in the registry + sdjournal.SD_JOURNAL_FIELD_CURSOR: ignoredField, // saved in the registry + "_SOURCE_MONOTONIC_TIMESTAMP": ignoredField, // received timestamp stored in @timestamp +} diff --git a/journalbeat/pkg/journalfield/default_other.go b/journalbeat/pkg/journalfield/default_other.go new file mode 100644 index 000000000000..ca3d26c92668 --- /dev/null +++ b/journalbeat/pkg/journalfield/default_other.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build !linux !cgo + +package journalfield + +// journaldEventFields provides default field mappings and conversions rules. +var journaldEventFields = FieldConversion{ + // provided by systemd journal + "COREDUMP_UNIT": text("journald.coredump.unit"), + "COREDUMP_USER_UNIT": text("journald.coredump.user_unit"), + "OBJECT_AUDIT_LOGINUID": integer("journald.object.audit.login_uid"), + "OBJECT_AUDIT_SESSION": integer("journald.object.audit.session"), + "OBJECT_CMDLINE": text("journald.object.cmd"), + "OBJECT_COMM": text("journald.object.name"), + "OBJECT_EXE": text("journald.object.executable"), + "OBJECT_GID": integer("journald.object.gid"), + "OBJECT_PID": integer("journald.object.pid"), + "OBJECT_SYSTEMD_OWNER_UID": integer("journald.object.systemd.owner_uid"), + "OBJECT_SYSTEMD_SESSION": text("journald.object.systemd.session"), + "OBJECT_SYSTEMD_UNIT": text("journald.object.systemd.unit"), + "OBJECT_SYSTEMD_USER_UNIT": text("journald.object.systemd.user_unit"), + "OBJECT_UID": integer("journald.object.uid"), + "_KERNEL_DEVICE": text("journald.kernel.device"), + "_KERNEL_SUBSYSTEM": text("journald.kernel.subsystem"), + "_SYSTEMD_INVOCATION_ID": text("systemd.invocation_id"), + "_SYSTEMD_USER_SLICE": text("systemd.user_slice"), + "_UDEV_DEVLINK": text("journald.kernel.device_symlinks"), + "_UDEV_DEVNODE": text("journald.kernel.device_node_path"), + "_UDEV_SYSNAME": text("journald.kernel.device_name"), +} diff --git a/journalbeat/pkg/journalfield/matcher.go b/journalbeat/pkg/journalfield/matcher.go new file mode 100644 index 000000000000..0227b2aae832 --- /dev/null +++ b/journalbeat/pkg/journalfield/matcher.go @@ -0,0 +1,120 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalfield + +import ( + "fmt" + "strings" +) + +// Matcher is a single field condition for filtering journal entries. +// +// The Matcher type can be used as is with Beats configuration unpacking. The +// internal default conversion table will be used, similar to BuildMatcher. +type Matcher struct { + str string +} + +// MatcherBuilder can be used to create a custom builder for creating matchers +// based on a conversion table. +type MatcherBuilder struct { + Conversions map[string]Conversion +} + +type journal interface { + AddMatch(string) error + AddDisjunction() error +} + +var defaultBuilder = MatcherBuilder{Conversions: journaldEventFields} + +// Build creates a new Matcher using the configured conversion table. +// If no table has been configured the internal default table will be used. +func (b MatcherBuilder) Build(in string) (Matcher, error) { + elems := strings.Split(in, "=") + if len(elems) != 2 { + return Matcher{}, fmt.Errorf("invalid match format: %s", in) + } + + conversions := b.Conversions + if conversions == nil { + conversions = journaldEventFields + } + + for journalKey, eventField := range conversions { + for _, name := range eventField.Names { + if elems[0] == name { + return Matcher{journalKey + "=" + elems[1]}, nil + } + } + } + + // pass custom fields as is + return Matcher{in}, nil +} + +// BuildMatcher creates a Matcher from a field filter string. +func BuildMatcher(in string) (Matcher, error) { + return defaultBuilder.Build(in) +} + +// IsValid returns true if the matcher was initialized correctly. +func (m Matcher) IsValid() bool { return m.str != "" } + +// String returns the string representation of the field match. +func (m Matcher) String() string { return m.str } + +// Apply adds the field match to an open journal for filtering. +func (m Matcher) Apply(j journal) error { + if !m.IsValid() { + return fmt.Errorf("can not apply invalid matcher to a journal") + } + + err := j.AddMatch(m.str) + if err != nil { + return fmt.Errorf("error adding match '%s' to journal: %v", m.str, err) + } + return nil +} + +// Unpack initializes the Matcher from a given string representation. Unpack +// fails if the input string is invalid. +// Unpack can be used with Beats configuration loading. +func (m *Matcher) Unpack(value string) error { + tmp, err := BuildMatcher(value) + if err != nil { + return err + } + *m = tmp + return nil +} + +// ApplyMatchersOr adds a list of matchers to a journal, calling AddDisjunction after each matcher being added. +func ApplyMatchersOr(j journal, matchers []Matcher) error { + for _, m := range matchers { + if err := m.Apply(j); err != nil { + return err + } + + if err := j.AddDisjunction(); err != nil { + return fmt.Errorf("error adding disjunction to journal: %v", err) + } + } + + return nil +} diff --git a/journalbeat/pkg/journalfield/matcher_test.go b/journalbeat/pkg/journalfield/matcher_test.go new file mode 100644 index 000000000000..9ce8c63c8a44 --- /dev/null +++ b/journalbeat/pkg/journalfield/matcher_test.go @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//+build linux,cgo + +package journalfield + +import ( + "testing" + + "github.com/coreos/go-systemd/v22/sdjournal" +) + +func TestApplyMatchersOr(t *testing.T) { + cases := map[string]struct { + filters []string + wantErr bool + }{ + "correct filter expression": { + filters: []string{"systemd.unit=nginx"}, + wantErr: false, + }, + "custom field": { + filters: []string{"_MY_CUSTOM_FIELD=value"}, + wantErr: false, + }, + "mixed filters": { + filters: []string{"systemd.unit=nginx", "_MY_CUSTOM_FIELD=value"}, + wantErr: false, + }, + "same field filters": { + filters: []string{"systemd.unit=nginx", "systemd.unit=mysql"}, + wantErr: false, + }, + "incorrect separator": { + filters: []string{"systemd.unit~nginx"}, + wantErr: true, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + journal, err := sdjournal.NewJournal() + if err != nil { + t.Fatalf("error while creating test journal: %v", err) + } + defer journal.Close() + + matchers := make([]Matcher, len(test.filters)) + for i, str := range test.filters { + m, err := BuildMatcher(str) + if err != nil && !test.wantErr { + t.Fatalf("unexpected error compiling the filters: %v", err) + } + matchers[i] = m + } + + // double check if journald likes our filters + err = ApplyMatchersOr(journal, matchers) + fail := (test.wantErr && err == nil) || (!test.wantErr && err != nil) + if fail { + t.Errorf("unexpected outcome: error: '%v', expected error: %v", err, test.wantErr) + } + }) + } +} diff --git a/journalbeat/pkg/journalread/mode.go b/journalbeat/pkg/journalread/mode.go new file mode 100644 index 000000000000..3eff2d93c5d9 --- /dev/null +++ b/journalbeat/pkg/journalread/mode.go @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalread + +import ( + "errors" + "fmt" +) + +// SeekMode is used by (*Reader).Seek to decide where to advance the read pointer to. +type SeekMode uint + +const ( + // SeekInvalid is an invalid value for seek + SeekInvalid SeekMode = iota + // SeekHead option seeks to the head of a journal + SeekHead + // SeekTail option seeks to the tail of a journal + SeekTail + // SeekCursor option seeks to the position specified in the cursor + SeekCursor +) + +var seekModes = map[string]SeekMode{ + "head": SeekHead, + "tail": SeekTail, + "cursor": SeekCursor, +} + +var errInvalidSeekFallback = errors.New("invalid setting for cursor_seek_fallback") + +// Unpack validates and unpack "seek" config options. It returns an error if +// the string is no valid seek mode. +func (m *SeekMode) Unpack(value string) error { + mode, ok := seekModes[value] + if !ok { + return fmt.Errorf("invalid seek mode '%s'", value) + } + + *m = mode + return nil +} diff --git a/journalbeat/pkg/journalread/mode_test.go b/journalbeat/pkg/journalread/mode_test.go new file mode 100644 index 000000000000..aef0ed4150cc --- /dev/null +++ b/journalbeat/pkg/journalread/mode_test.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package journalread + +import ( + "testing" +) + +func TestMode_Unpack(t *testing.T) { + t.Run("ok", func(t *testing.T) { + tests := map[string]SeekMode{ + "head": SeekHead, + "tail": SeekTail, + "cursor": SeekCursor, + } + + for str, want := range tests { + t.Run(str, func(t *testing.T) { + var m SeekMode + if err := m.Unpack(str); err != nil { + t.Fatal(err) + } + + if m != want { + t.Errorf("wrong mode, expected %v, got %v", want, m) + } + }) + } + }) + + t.Run("failing", func(t *testing.T) { + cases := []string{"invalid", "", "unknown"} + + for _, str := range cases { + t.Run(str, func(t *testing.T) { + var m SeekMode + if err := m.Unpack(str); err == nil { + t.Errorf("an error was expected, got %v", m) + } + }) + } + }) +} diff --git a/journalbeat/pkg/journalread/reader.go b/journalbeat/pkg/journalread/reader.go new file mode 100644 index 000000000000..7d5ac50e9652 --- /dev/null +++ b/journalbeat/pkg/journalread/reader.go @@ -0,0 +1,199 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux,cgo + +package journalread + +import ( + "fmt" + "io" + "os" + "syscall" + "time" + + "github.com/coreos/go-systemd/v22/sdjournal" + "github.com/urso/sderr" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common/backoff" + "github.com/elastic/beats/v7/libbeat/common/cleanup" + "github.com/elastic/beats/v7/libbeat/logp" +) + +// Reader implements a Journald base reader with backoff support. The reader +// will block until a new entry can be read from the journal. +type Reader struct { + log *logp.Logger + backoff backoff.Backoff + journal journal +} + +type journal interface { + Close() error + + Next() (uint64, error) + Wait(time.Duration) int + GetEntry() (*sdjournal.JournalEntry, error) + + SeekHead() error + SeekTail() error + SeekCursor(string) error +} + +// LocalSystemJournalID is the ID of the local system journal. +const localSystemJournalID = "LOCAL_SYSTEM_JOURNAL" + +// NewReader creates a new Reader for an already opened journal. The reader assumed to take +// ownership of the journal, and needs to be closed. +func NewReader(log *logp.Logger, journal journal, backoff backoff.Backoff) *Reader { + return &Reader{log: log, journal: journal, backoff: backoff} +} + +// Open opens a journal and creates a reader for it. +// Additonal settings can be applied to the journal by passing functions to with. +// Open returns an error if the journal can not be opened, or if one with-function failed. +// +// Open will opend the systems journal if the path is empty or matches LOCAL_SYSTEM_JOURNAL. +// The path can optionally point to a file or a directory. +func Open(log *logp.Logger, path string, backoff backoff.Backoff, with ...func(j *sdjournal.Journal) error) (*Reader, error) { + j, err := openJournal(path) + if err != nil { + return nil, err + } + + ok := false + defer cleanup.IfNot(&ok, func() { j.Close() }) + + for _, w := range with { + if err := w(j); err != nil { + return nil, err + } + } + + ok = true + return NewReader(log, j, backoff), nil +} + +func openJournal(path string) (*sdjournal.Journal, error) { + if path == localSystemJournalID || path == "" { + j, err := sdjournal.NewJournal() + if err != nil { + err = sderr.Wrap(err, "failed to open local journal") + } + return j, err + } + + stat, err := os.Stat(path) + if err != nil { + return nil, sderr.Wrap(err, "failed to read meta data for %{path}", path) + } + + if stat.IsDir() { + j, err := sdjournal.NewJournalFromDir(path) + if err != nil { + err = sderr.Wrap(err, "failed to open journal directory %{path}", path) + } + return j, err + } + + j, err := sdjournal.NewJournalFromFiles(path) + if err != nil { + err = sderr.Wrap(err, "failed to open journal file %{path}", path) + } + return j, err +} + +// Close closes the journal. +func (r *Reader) Close() error { + return r.journal.Close() +} + +// Seek moves the read pointer to a new position. +// If a cursor or SeekTail is given, Seek tries to ignore the entry at the +// given position, jumping right to the next entry. +func (r *Reader) Seek(mode SeekMode, cursor string) (err error) { + switch mode { + case SeekHead: + err = r.journal.SeekHead() + case SeekTail: + if err = r.journal.SeekTail(); err == nil { + _, err = r.journal.Next() + } + case SeekCursor: + if err = r.journal.SeekCursor(cursor); err == nil { + _, err = r.journal.Next() + } + default: + return fmt.Errorf("invalid seek mode '%v'", mode) + } + return err +} + +// Next reads a new journald entry from the journal. It blocks if there is +// currently no entry available in the journal, or until an error has occured. +func (r *Reader) Next(cancel input.Canceler) (*sdjournal.JournalEntry, error) { + for cancel.Err() == nil { + c, err := r.journal.Next() + if err != nil && err != io.EOF { + return nil, err + } + + switch { + // error while reading next entry + case c < 0: + return nil, fmt.Errorf("error while reading next entry %+v", syscall.Errno(-c)) + // no new entry, so wait + case c == 0: + hasNewEntry, err := r.checkForNewEvents() + if err != nil { + return nil, err + } + if !hasNewEntry { + // TODO: backoff support is currently not cancellable :( + r.backoff.Wait() + } + continue + // new entries are available + default: + } + + entry, err := r.journal.GetEntry() + if err != nil { + return nil, err + } + r.backoff.Reset() + + return entry, nil + } + return nil, cancel.Err() +} + +func (r *Reader) checkForNewEvents() (bool, error) { + c := r.journal.Wait(100 * time.Millisecond) + switch c { + case sdjournal.SD_JOURNAL_NOP: + return false, nil + // new entries are added or the journal has changed (e.g. vacuum, rotate) + case sdjournal.SD_JOURNAL_APPEND, sdjournal.SD_JOURNAL_INVALIDATE: + return true, nil + default: + } + + r.log.Errorf("Unknown return code from Wait: %d\n", c) + return false, nil +} diff --git a/journalbeat/reader/config.go b/journalbeat/reader/config.go index f8dbabe5feef..e559681aea79 100644 --- a/journalbeat/reader/config.go +++ b/journalbeat/reader/config.go @@ -20,7 +20,8 @@ package reader import ( "time" - "github.com/elastic/beats/v7/journalbeat/config" + "github.com/elastic/beats/v7/journalbeat/pkg/journalfield" + "github.com/elastic/beats/v7/journalbeat/pkg/journalread" ) // Config stores the options of a reder. @@ -29,16 +30,16 @@ type Config struct { Path string // Seek specifies the seeking stategy. // Possible values: head, tail, cursor. - Seek config.SeekMode + Seek journalread.SeekMode // CursorSeekFallback sets where to seek if registry file is not available. - CursorSeekFallback config.SeekMode + CursorSeekFallback journalread.SeekMode // MaxBackoff is the limit of the backoff time. MaxBackoff time.Duration // Backoff is the current interval to wait before // attemting to read again from the journal. Backoff time.Duration // Matches store the key value pairs to match entries. - Matches []string + Matches []journalfield.Matcher // SaveRemoteHostname defines if the original source of the entry needs to be saved. SaveRemoteHostname bool // CheckpointID is the identifier to use when persisting state. diff --git a/journalbeat/reader/fields.go b/journalbeat/reader/fields.go index d0230849cba3..0b6efc3cdc8b 100644 --- a/journalbeat/reader/fields.go +++ b/journalbeat/reader/fields.go @@ -18,78 +18,3 @@ //+build linux,cgo package reader - -import "github.com/coreos/go-systemd/v22/sdjournal" - -type fieldConversion struct { - names []string - isInteger bool - dropped bool -} - -var ( - journaldEventFields = map[string]fieldConversion{ - // provided by systemd journal - "COREDUMP_UNIT": fieldConversion{[]string{"journald.coredump.unit"}, false, false}, - "COREDUMP_USER_UNIT": fieldConversion{[]string{"journald.coredump.user_unit"}, false, false}, - "OBJECT_AUDIT_LOGINUID": fieldConversion{[]string{"journald.object.audit.login_uid"}, true, false}, - "OBJECT_AUDIT_SESSION": fieldConversion{[]string{"journald.object.audit.session"}, true, false}, - "OBJECT_CMDLINE": fieldConversion{[]string{"journald.object.cmd"}, false, false}, - "OBJECT_COMM": fieldConversion{[]string{"journald.object.name"}, false, false}, - "OBJECT_EXE": fieldConversion{[]string{"journald.object.executable"}, false, false}, - "OBJECT_GID": fieldConversion{[]string{"journald.object.gid"}, true, false}, - "OBJECT_PID": fieldConversion{[]string{"journald.object.pid"}, true, false}, - "OBJECT_SYSTEMD_OWNER_UID": fieldConversion{[]string{"journald.object.systemd.owner_uid"}, true, false}, - "OBJECT_SYSTEMD_SESSION": fieldConversion{[]string{"journald.object.systemd.session"}, false, false}, - "OBJECT_SYSTEMD_UNIT": fieldConversion{[]string{"journald.object.systemd.unit"}, false, false}, - "OBJECT_SYSTEMD_USER_UNIT": fieldConversion{[]string{"journald.object.systemd.user_unit"}, false, false}, - "OBJECT_UID": fieldConversion{[]string{"journald.object.uid"}, true, false}, - "_KERNEL_DEVICE": fieldConversion{[]string{"journald.kernel.device"}, false, false}, - "_KERNEL_SUBSYSTEM": fieldConversion{[]string{"journald.kernel.subsystem"}, false, false}, - "_SYSTEMD_INVOCATION_ID": fieldConversion{[]string{"systemd.invocation_id"}, false, false}, - "_SYSTEMD_USER_SLICE": fieldConversion{[]string{"systemd.user_slice"}, false, false}, - "_UDEV_DEVLINK": fieldConversion{[]string{"journald.kernel.device_symlinks"}, false, false}, // TODO aggregate multiple elements - "_UDEV_DEVNODE": fieldConversion{[]string{"journald.kernel.device_node_path"}, false, false}, - "_UDEV_SYSNAME": fieldConversion{[]string{"journald.kernel.device_name"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: fieldConversion{[]string{"process.audit.login_uid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: fieldConversion{[]string{"process.audit.session"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_BOOT_ID: fieldConversion{[]string{"host.boot_id"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: fieldConversion{[]string{"process.capabilites"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CMDLINE: fieldConversion{[]string{"process.cmd"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CODE_FILE: fieldConversion{[]string{"journald.code.file"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: fieldConversion{[]string{"journald.code.func"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_CODE_LINE: fieldConversion{[]string{"journald.code.line"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_COMM: fieldConversion{[]string{"process.name"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_EXE: fieldConversion{[]string{"process.executable"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_GID: fieldConversion{[]string{"process.uid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_HOSTNAME: fieldConversion{[]string{"host.hostname"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: fieldConversion{[]string{"host.id"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_MESSAGE: fieldConversion{[]string{"message"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_PID: fieldConversion{[]string{"process.pid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_PRIORITY: fieldConversion{[]string{"syslog.priority", "log.syslog.priority"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: fieldConversion{[]string{"syslog.facility", "log.syslog.facility.name"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: fieldConversion{[]string{"syslog.identifier"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: fieldConversion{[]string{"syslog.pid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: fieldConversion{[]string{"systemd.cgroup"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: fieldConversion{[]string{"systemd.owner_uid"}, true, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: fieldConversion{[]string{"systemd.session"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: fieldConversion{[]string{"systemd.slice"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: fieldConversion{[]string{"systemd.unit"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: fieldConversion{[]string{"systemd.user_unit"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_TRANSPORT: fieldConversion{[]string{"systemd.transport"}, false, false}, - sdjournal.SD_JOURNAL_FIELD_UID: fieldConversion{[]string{"process.uid"}, true, false}, - - // docker journald fields from: https://docs.docker.com/config/containers/logging/journald/ - "CONTAINER_ID": fieldConversion{[]string{"container.id_truncated"}, false, false}, - "CONTAINER_ID_FULL": fieldConversion{[]string{"container.id"}, false, false}, - "CONTAINER_NAME": fieldConversion{[]string{"container.name"}, false, false}, - "CONTAINER_TAG": fieldConversion{[]string{"container.log.tag"}, false, false}, - "CONTAINER_PARTIAL_MESSAGE": fieldConversion{[]string{"container.partial"}, false, false}, - - // dropped fields - sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: fieldConversion{nil, false, true}, // saved in the registry - sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: fieldConversion{nil, false, true}, // saved in the registry - sdjournal.SD_JOURNAL_FIELD_CURSOR: fieldConversion{nil, false, true}, // saved in the registry - "_SOURCE_MONOTONIC_TIMESTAMP": fieldConversion{nil, false, true}, // received timestamp stored in @timestamp - } -) diff --git a/journalbeat/reader/journal.go b/journalbeat/reader/journal.go index 7a254f4f6d2d..fb5b91c5019a 100644 --- a/journalbeat/reader/journal.go +++ b/journalbeat/reader/journal.go @@ -20,28 +20,23 @@ package reader import ( - "fmt" - "io" - "os" - "strconv" - "strings" - "syscall" "time" "github.com/coreos/go-systemd/v22/sdjournal" - "github.com/pkg/errors" "github.com/elastic/beats/v7/journalbeat/checkpoint" "github.com/elastic/beats/v7/journalbeat/cmd/instance" - "github.com/elastic/beats/v7/journalbeat/config" + "github.com/elastic/beats/v7/journalbeat/pkg/journalfield" + "github.com/elastic/beats/v7/journalbeat/pkg/journalread" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/backoff" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/ctxtool" ) // Reader reads entries from journal(s). type Reader struct { + r *journalread.Reader journal *sdjournal.Journal config Config done chan struct{} @@ -51,223 +46,83 @@ type Reader struct { // New creates a new journal reader and moves the FP to the configured position. func New(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) { - f, err := os.Stat(c.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to open file") - } - - var j *sdjournal.Journal - if f.IsDir() { - j, err = sdjournal.NewJournalFromDir(c.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to open journal directory") - } - } else { - j, err = sdjournal.NewJournalFromFiles(c.Path) - if err != nil { - return nil, errors.Wrap(err, "failed to open journal file") - } - } - - l := logger.With("path", c.Path) - l.Debug("New journal is opened for reading") - - return newReader(l, done, c, j, state) + return newReader(c.Path, c, done, state, logger) } // NewLocal creates a reader to read form the local journal and moves the FP // to the configured position. func NewLocal(c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) { - j, err := sdjournal.NewJournal() - if err != nil { - return nil, errors.Wrap(err, "failed to open local journal") - } - - l := logger.With("path", "local") - l.Debug("New local journal is opened for reading") - - return newReader(l, done, c, j, state) + return newReader(LocalSystemJournalID, c, done, state, logger) } -func newReader(logger *logp.Logger, done chan struct{}, c Config, journal *sdjournal.Journal, state checkpoint.JournalState) (*Reader, error) { - err := setupMatches(journal, c.Matches) +func newReader(path string, c Config, done chan struct{}, state checkpoint.JournalState, logger *logp.Logger) (*Reader, error) { + logger = logger.With("path", path) + backoff := backoff.NewExpBackoff(done, c.Backoff, c.MaxBackoff) + + var journal *sdjournal.Journal + r, err := journalread.Open(logger, c.Path, backoff, func(j *sdjournal.Journal) error { + journal = j + return journalfield.ApplyMatchersOr(j, c.Matches) + }) if err != nil { return nil, err } - r := &Reader{ - journal: journal, - config: c, - done: done, - logger: logger, - backoff: backoff.NewExpBackoff(done, c.Backoff, c.MaxBackoff), + if err := r.Seek(seekBy(logger, c, state)); err != nil { + logger.Error("Continue from current position. Seek failed with: %v", err) } - r.seek(state.Cursor) + logger.Debug("New journal is opened for reading") instance.AddJournalToMonitor(c.Path, journal) - return r, nil + return &Reader{ + journal: journal, + config: c, + done: done, + logger: logger, + backoff: backoff, + }, nil } -func setupMatches(j *sdjournal.Journal, matches []string) error { - for _, m := range matches { - elems := strings.Split(m, "=") - if len(elems) != 2 { - return fmt.Errorf("invalid match format: %s", m) - } - - var p string - for journalKey, eventField := range journaldEventFields { - for _, name := range eventField.names { - if elems[0] == name { - p = journalKey + "=" + elems[1] - } - } - } - - // pass custom fields as is - if p == "" { - p = m - } - - logp.Debug("journal", "Added matcher expression: %s", p) - - err := j.AddMatch(p) - if err != nil { - return fmt.Errorf("error adding match to journal %v", err) - } - - err = j.AddDisjunction() - if err != nil { - return fmt.Errorf("error adding disjunction to journal: %v", err) +func seekBy(log *logp.Logger, c Config, state checkpoint.JournalState) (journalread.SeekMode, string) { + mode := c.Seek + if mode == journalread.SeekCursor && state.Cursor == "" { + mode = c.CursorSeekFallback + if mode != journalread.SeekHead && mode != journalread.SeekTail { + log.Error("Invalid option for cursor_seek_fallback") + mode = journalread.SeekHead } } - return nil + return mode, state.Cursor } -// seek seeks to the position determined by the coniguration and cursor state. -func (r *Reader) seek(cursor string) { - switch r.config.Seek { - case config.SeekCursor: - if cursor == "" { - switch r.config.CursorSeekFallback { - case config.SeekHead: - r.journal.SeekHead() - r.logger.Debug("Seeking method set to cursor, but no state is saved for reader. Starting to read from the beginning") - case config.SeekTail: - r.journal.SeekTail() - r.journal.Next() - r.logger.Debug("Seeking method set to cursor, but no state is saved for reader. Starting to read from the end") - default: - r.logger.Error("Invalid option for cursor_seek_fallback") - } - return - } - r.journal.SeekCursor(cursor) - _, err := r.journal.Next() - if err != nil { - r.logger.Error("Error while seeking to cursor") - } - r.logger.Debug("Seeked to position defined in cursor") - case config.SeekTail: - r.journal.SeekTail() - r.journal.Next() - r.logger.Debug("Tailing the journal file") - case config.SeekHead: - r.journal.SeekHead() - r.logger.Debug("Reading from the beginning of the journal file") - default: - r.logger.Error("Invalid seeking mode") - } +// Close closes the underlying journal reader. +func (r *Reader) Close() { + instance.StopMonitoringJournal(r.config.Path) + r.r.Close() } // Next waits until a new event shows up and returns it. // It blocks until an event is returned or an error occurs. func (r *Reader) Next() (*beat.Event, error) { - for { - select { - case <-r.done: - return nil, nil - default: - } - - c, err := r.journal.Next() - if err != nil && err != io.EOF { - return nil, err - } - - switch { - // error while reading next entry - case c < 0: - return nil, fmt.Errorf("error while reading next entry %+v", syscall.Errno(-c)) - // no new entry, so wait - case c == 0: - hasNewEntry, err := r.checkForNewEvents() - if err != nil { - return nil, err - } - if !hasNewEntry { - r.backoff.Wait() - } - continue - // new entries are available - default: - } - - entry, err := r.journal.GetEntry() - if err != nil { - return nil, err - } - event := r.toEvent(entry) - r.backoff.Reset() - - return event, nil - } -} - -func (r *Reader) checkForNewEvents() (bool, error) { - c := r.journal.Wait(100 * time.Millisecond) - switch c { - case sdjournal.SD_JOURNAL_NOP: - return false, nil - // new entries are added or the journal has changed (e.g. vacuum, rotate) - case sdjournal.SD_JOURNAL_APPEND, sdjournal.SD_JOURNAL_INVALIDATE: - return true, nil - default: + entry, err := r.r.Next(ctxtool.FromChannel(r.done)) + if err != nil { + return nil, err } - r.logger.Errorf("Unknown return code from Wait: %d\n", c) - return false, nil + event := toEvent(r.logger, r.config.CheckpointID, entry, r.config.SaveRemoteHostname) + return event, nil } // toEvent creates a beat.Event from journal entries. -func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { - fields := common.MapStr{ - "event": common.MapStr{ - "kind": "event", - }, - } - custom := common.MapStr{} - - for entryKey, v := range entry.Fields { - if fieldConversionInfo, ok := journaldEventFields[entryKey]; !ok { - normalized := strings.ToLower(strings.TrimLeft(entryKey, "_")) - custom.Put(normalized, v) - } else if !fieldConversionInfo.dropped { - value := r.convertNamedField(fieldConversionInfo, v) - for _, name := range fieldConversionInfo.names { - fields.Put(name, value) - } - } - } - - if len(custom) != 0 { - fields.Put("journald.custom", custom) - } +func toEvent(logger *logp.Logger, id string, entry *sdjournal.JournalEntry, saveRemoteHostname bool) *beat.Event { + created := time.Now() + fields := journalfield.NewConverter(logger, nil).Convert(entry.Fields) + fields.Put("event.kind", "event") // if entry is coming from a remote journal, add_host_metadata overwrites the source hostname, so it // has to be copied to a different field - if r.config.SaveRemoteHostname { + if saveRemoteHostname { remoteHostname, err := fields.GetValue("host.hostname") if err == nil { fields.Put("log.source.address", remoteHostname) @@ -275,13 +130,13 @@ func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { } state := checkpoint.JournalState{ - Path: r.config.CheckpointID, + Path: id, Cursor: entry.Cursor, RealtimeTimestamp: entry.RealtimeTimestamp, MonotonicTimestamp: entry.MonotonicTimestamp, } - fields.Put("event.created", time.Now()) + fields.Put("event.created", created) receivedByJournal := time.Unix(0, int64(entry.RealtimeTimestamp)*1000) event := beat.Event{ @@ -291,29 +146,3 @@ func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { } return &event } - -func (r *Reader) convertNamedField(fc fieldConversion, value string) interface{} { - if fc.isInteger { - v, err := strconv.ParseInt(value, 10, 64) - if err != nil { - // On some versions of systemd the 'syslog.pid' can contain the username - // appended to the end of the pid. In most cases this does not occur - // but in the cases that it does, this tries to strip ',\w*' from the - // value and then perform the conversion. - s := strings.Split(value, ",") - v, err = strconv.ParseInt(s[0], 10, 64) - if err != nil { - r.logger.Debugf("Failed to convert field: %v \"%v\" to int: %v", fc.names, value, err) - return value - } - } - return v - } - return value -} - -// Close closes the underlying journal reader. -func (r *Reader) Close() { - instance.StopMonitoringJournal(r.config.Path) - r.journal.Close() -} diff --git a/journalbeat/reader/journal_test.go b/journalbeat/reader/journal_test.go deleted file mode 100644 index 4293b4b31e16..000000000000 --- a/journalbeat/reader/journal_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//+build linux,cgo - -package reader - -import ( - "reflect" - "testing" - - "github.com/coreos/go-systemd/v22/sdjournal" - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/journalbeat/checkpoint" - "github.com/elastic/beats/v7/journalbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" -) - -type ToEventTestCase struct { - entry sdjournal.JournalEntry - expectedFields common.MapStr -} - -type SetupMatchesTestCase struct { - matches []string - expectError bool -} - -func TestToEvent(t *testing.T) { - tests := []ToEventTestCase{ - // field name from fields.go - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_BOOT_ID: "123456", - }, - }, - expectedFields: common.MapStr{ - "host": common.MapStr{ - "boot_id": "123456", - }, - }, - }, - // 'syslog.pid' field without user append - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "pid": int64(123456), - }, - }, - }, - // 'syslog.priority' field with junk - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_PRIORITY: "123456, ", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "priority": int64(123456), - }, - "log": common.MapStr{ - "syslog": common.MapStr{ - "priority": int64(123456), - }, - }, - }, - }, - // 'syslog.pid' field with user append - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "123456,root", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "pid": int64(123456), - }, - }, - }, - // 'syslog.pid' field empty - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "", - }, - }, - expectedFields: common.MapStr{ - "syslog": common.MapStr{ - "pid": "", - }, - }, - }, - // custom field - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - "my_custom_field": "value", - }, - }, - expectedFields: common.MapStr{ - "journald": common.MapStr{ - "custom": common.MapStr{ - "my_custom_field": "value", - }, - }, - }, - }, - // dropped field - ToEventTestCase{ - entry: sdjournal.JournalEntry{ - Fields: map[string]string{ - "_SOURCE_MONOTONIC_TIMESTAMP": "value", - }, - }, - expectedFields: common.MapStr{}, - }, - } - - instance.SetupJournalMetrics() - r, err := NewLocal(Config{Path: "dummy.journal"}, nil, checkpoint.JournalState{}, logp.NewLogger("test")) - if err != nil { - t.Fatalf("error creating test journal: %v", err) - } - for _, test := range tests { - event := r.toEvent(&test.entry) - event.Fields.Delete("event") - assert.True(t, reflect.DeepEqual(event.Fields, test.expectedFields)) - } -} - -func TestSetupMatches(t *testing.T) { - tests := []SetupMatchesTestCase{ - // correct filter expression - SetupMatchesTestCase{ - matches: []string{"systemd.unit=nginx"}, - expectError: false, - }, - // custom field - SetupMatchesTestCase{ - matches: []string{"_MY_CUSTOM_FIELD=value"}, - expectError: false, - }, - // incorrect separator - SetupMatchesTestCase{ - matches: []string{"systemd.unit~nginx"}, - expectError: true, - }, - } - journal, err := sdjournal.NewJournal() - if err != nil { - t.Fatalf("error while creating test journal: %v", err) - } - - for _, test := range tests { - err = setupMatches(journal, test.matches) - if err != nil && !test.expectError { - t.Errorf("unexpected outcome of setupMatches: error: '%v', expected error: %v", err, test.expectError) - } - } -} From 6dd12f67f7494c7f54f18ca6e50eea3f88e15cc8 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 2 Jul 2020 14:42:41 +0200 Subject: [PATCH 24/35] Implement memlog store operations (#19533) Implement store operation that get/set key value pairs from the in-memory store, and log update operations. This change also include unit tests and store compliance tests. The addition of the statestore package is split up into multiple changeset to ease review. The final version of the package can be found [here](https://github.com/urso/beats/tree/fb-input-v2-combined/libbeat/statestore). Once finalized, the libbeat/statestore package contains: - The statestore frontend and interface for use within Beats - Interfaces for the store backend - A common set of tests store backends need to support - a storetest package for testing new features that require a store. The testing helpers use map[string]interface{} that can be initialized or queried after the test run for validation purposes. - The default memlog backend + tests --- libbeat/statestore/backend/memlog/memlog.go | 7 +- .../statestore/backend/memlog/memlog_test.go | 258 ++++++++++++++++++ libbeat/statestore/backend/memlog/store.go | 177 +++++++++++- .../backend/memlog/testdata/1/Readme.md | 1 + .../memlog/testdata/1/commit_wrong_id/1.json | 5 + .../testdata/1/commit_wrong_id/Readme.md | 2 + .../testdata/1/commit_wrong_id/expected.json | 9 + .../testdata/1/commit_wrong_id/log.json | 6 + .../testdata/1/commit_wrong_id/meta.json | 1 + .../memlog/testdata/1/data_and_log/1.json | 5 + .../memlog/testdata/1/data_and_log/Readme.md | 1 + .../testdata/1/data_and_log/expected.json | 12 + .../memlog/testdata/1/data_and_log/log.json | 6 + .../memlog/testdata/1/data_and_log/meta.json | 1 + .../memlog/testdata/1/datafile_only/1.json | 8 + .../memlog/testdata/1/datafile_only/Readme.md | 2 + .../testdata/1/datafile_only/expected.json | 12 + .../memlog/testdata/1/datafile_only/meta.json | 1 + .../memlog/testdata/1/incomplete_op/1.json | 5 + .../memlog/testdata/1/incomplete_op/Readme.md | 2 + .../testdata/1/incomplete_op/expected.json | 9 + .../memlog/testdata/1/incomplete_op/log.json | 1 + .../memlog/testdata/1/incomplete_op/meta.json | 1 + .../testdata/1/logfile_incomplete/Readme.md | 3 + .../1/logfile_incomplete/expected.json | 12 + .../testdata/1/logfile_incomplete/log.json | 13 + .../testdata/1/logfile_incomplete/meta.json | 1 + .../memlog/testdata/1/logfile_only/Readme.md | 2 + .../testdata/1/logfile_only/expected.json | 12 + .../memlog/testdata/1/logfile_only/log.json | 12 + .../memlog/testdata/1/logfile_only/meta.json | 1 + .../memlog/testdata/1/old_datafiles/1.json | 8 + .../memlog/testdata/1/old_datafiles/2.json | 3 + .../memlog/testdata/1/old_datafiles/3.json | 5 + .../memlog/testdata/1/old_datafiles/Readme.md | 2 + .../testdata/1/old_datafiles/expected.json | 9 + .../memlog/testdata/1/old_datafiles/meta.json | 1 + .../testdata/1/old_entries_in_log/5.json | 5 + .../testdata/1/old_entries_in_log/Readme.md | 3 + .../1/old_entries_in_log/expected.json | 9 + .../testdata/1/old_entries_in_log/log.json | 8 + .../testdata/1/old_entries_in_log/meta.json | 1 + libbeat/statestore/backend/memlog/util.go | 43 +++ 43 files changed, 671 insertions(+), 14 deletions(-) create mode 100644 libbeat/statestore/backend/memlog/memlog_test.go create mode 100644 libbeat/statestore/backend/memlog/testdata/1/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/data_and_log/1.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/data_and_log/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/data_and_log/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/data_and_log/log.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/data_and_log/meta.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/datafile_only/1.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/datafile_only/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/datafile_only/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/datafile_only/meta.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/incomplete_op/1.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/incomplete_op/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/incomplete_op/log.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/incomplete_op/meta.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_only/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_only/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_only/log.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/logfile_only/meta.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_datafiles/1.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_datafiles/2.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_datafiles/3.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_datafiles/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_datafiles/meta.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json create mode 100644 libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json diff --git a/libbeat/statestore/backend/memlog/memlog.go b/libbeat/statestore/backend/memlog/memlog.go index 265b53a3df1c..fc93d85e85d9 100644 --- a/libbeat/statestore/backend/memlog/memlog.go +++ b/libbeat/statestore/backend/memlog/memlog.go @@ -54,6 +54,9 @@ type Settings struct { // Checkpoint predicate that can trigger a registry file rotation. If not // configured, memlog will automatically trigger a checkpoint every 10MB. Checkpoint CheckpointPredicate + + // If set memlog will not check the version of the meta file. + IgnoreVersionCheck bool } // CheckpointPredicate is the type for configurable checkpoint checks. @@ -62,7 +65,7 @@ type CheckpointPredicate func(fileSize uint64) bool const defaultFileMode os.FileMode = 0600 -const defaultBufferSize = 4096 +const defaultBufferSize = 4 * 1024 func defaultCheckpoint(filesize uint64) bool { const limit = 10 * 1 << 20 // set rotation limit to 10MB by default @@ -110,7 +113,7 @@ func (r *Registry) Access(name string) (backend.Store, error) { home := filepath.Join(r.settings.Root, name) fileMode := r.settings.FileMode bufSz := r.settings.BufferSize - store, err := openStore(logger, home, fileMode, bufSz, r.settings.Checkpoint) + store, err := openStore(logger, home, fileMode, bufSz, r.settings.IgnoreVersionCheck, r.settings.Checkpoint) if err != nil { return nil, err } diff --git a/libbeat/statestore/backend/memlog/memlog_test.go b/libbeat/statestore/backend/memlog/memlog_test.go new file mode 100644 index 000000000000..d8fe048f69c7 --- /dev/null +++ b/libbeat/statestore/backend/memlog/memlog_test.go @@ -0,0 +1,258 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memlog + +import ( + "encoding/json" + "io" + "io/ioutil" + "math" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/backend" + "github.com/elastic/beats/v7/libbeat/statestore/internal/storecompliance" +) + +func init() { + logp.DevelopmentSetup() +} + +func TestCompliance_Default(t *testing.T) { + storecompliance.TestBackendCompliance(t, func(testPath string) (backend.Registry, error) { + return New(logp.NewLogger("test"), Settings{Root: testPath}) + }) +} + +func TestCompliance_AlwaysCheckpoint(t *testing.T) { + storecompliance.TestBackendCompliance(t, func(testPath string) (backend.Registry, error) { + return New(logp.NewLogger("test"), Settings{ + Root: testPath, + Checkpoint: func(filesize uint64) bool { + return true + }, + }) + }) +} + +func TestLoadVersion1(t *testing.T) { + dataHome := "testdata/1" + + list, err := ioutil.ReadDir(dataHome) + if err != nil { + t.Fatal(err) + } + + cases := list[:0] + for _, info := range list { + if info.IsDir() { + cases = append(cases, info) + } + } + + for _, info := range cases { + name := filepath.Base(info.Name()) + t.Run(name, func(t *testing.T) { + testLoadVersion1Case(t, filepath.Join(dataHome, info.Name())) + }) + } +} + +func testLoadVersion1Case(t *testing.T, dataPath string) { + path, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create temporary test directory: %v", err) + } + defer os.RemoveAll(path) + + t.Logf("Test tmp dir: %v", path) + + if err := copyPath(path, dataPath); err != nil { + t.Fatalf("Failed to copy test file to the temporary directory: %v", err) + } + + // load expected test results + raw, err := ioutil.ReadFile(filepath.Join(path, "expected.json")) + if err != nil { + t.Fatalf("Failed to load expected.json: %v", err) + } + + expected := struct { + Txid uint64 + Datafile string + Entries map[string]interface{} + }{} + if err := json.Unmarshal(raw, &expected); err != nil { + t.Fatalf("Failed to parse expected.json: %v", err) + } + + // load store: + store, err := openStore(logp.NewLogger("test"), path, 0660, 4096, true, func(_ uint64) bool { + return false + }) + if err != nil { + t.Fatalf("Failed to load test store: %v", err) + } + defer store.Close() + + disk := store.disk + disk.removeOldDataFiles() + + // validate store: + assert.Equal(t, expected.Txid, disk.nextTxID-1) + if expected.Datafile != "" { + assert.Equal(t, filepath.Join(path, expected.Datafile), disk.activeDataFile.path) + } + + // check all keys in expected are known and do match stored values: + func() { + for key, val := range expected.Entries { + var tmp interface{} + err := store.Get(key, &tmp) + require.NoError(t, err, "error reading entry (key=%v)", key) + + assert.Equal(t, val, tmp, "failed when checking key '%s'", key) + } + }() + + // check store does not contain any additional keys + func() { + err = store.Each(func(key string, val statestore.ValueDecoder) (bool, error) { + _, exists := expected.Entries[string(key)] + if !exists { + t.Errorf("unexpected key: %s", key) + } + return true, nil + }) + assert.NoError(t, err) + }() +} + +func TestTxIDLessEqual(t *testing.T) { + cases := map[string]struct { + a, b uint64 + want bool + }{ + "is equal": {10, 10, true}, + "is less": {8, 9, true}, + "is bigger": {9, 8, false}, + "is less 0 with integer overflow": { + math.MaxUint64 - 2, 0, true, + }, + "is less random value with integer overflow": { + math.MaxUint64 - 2, 10, true, + }, + "is less with large ids": { + math.MaxUint64 - 10, math.MaxUint64 - 9, true, + }, + "is bigger with large ids": { + math.MaxUint64 - 9, math.MaxUint64 - 10, false, + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + got := isTxIDLessEqual(test.a, test.b) + if got != test.want { + t.Fatalf("%v <= %v should be %v", test.a, test.b, test.want) + } + }) + } +} + +func copyPath(to, from string) error { + info, err := os.Stat(from) + if err != nil { + return err + } + + if info.IsDir() { + return copyDir(to, from) + } + if info.Mode().IsRegular() { + return copyFile(to, from) + } + + // ignore other file types + return nil +} + +func copyDir(to, from string) error { + if !isDir(to) { + info, err := os.Stat(from) + if err != nil { + return err + } + + if err := os.MkdirAll(to, info.Mode()); err != nil { + return err + } + } + + list, err := ioutil.ReadDir(from) + if err != nil { + return err + } + + for _, file := range list { + name := file.Name() + err := copyPath(filepath.Join(to, name), filepath.Join(from, name)) + if err != nil { + return err + } + } + return nil +} + +func copyFile(to, from string) error { + in, err := os.Open(from) + if err != nil { + return err + } + defer in.Close() + + info, err := in.Stat() + if err != nil { + return err + } + + out, err := os.OpenFile(to, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode()) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + return err +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +func isFile(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} diff --git a/libbeat/statestore/backend/memlog/store.go b/libbeat/statestore/backend/memlog/store.go index 68bb6124a3af..29880a6aae8b 100644 --- a/libbeat/statestore/backend/memlog/store.go +++ b/libbeat/statestore/backend/memlog/store.go @@ -18,9 +18,12 @@ package memlog import ( + "fmt" "os" + "path/filepath" "sync" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore/backend" @@ -56,51 +59,201 @@ type entry struct { // If an error in in the log file is detected, the store opening routine continues from the last known valid state and will trigger a checkpoint // operation on subsequent writes, also truncating the log file. // Old data files are scheduled for deletion later. -func openStore(log *logp.Logger, home string, mode os.FileMode, bufSz uint, checkpoint CheckpointPredicate) (*store, error) { - panic("TODO: implement me") +func openStore(log *logp.Logger, home string, mode os.FileMode, bufSz uint, ignoreVersionCheck bool, checkpoint CheckpointPredicate) (*store, error) { + fi, err := os.Stat(home) + if os.IsNotExist(err) { + err = os.MkdirAll(home, os.ModeDir|0770) + if err != nil { + return nil, err + } + + err = writeMetaFile(home, mode) + if err != nil { + return nil, err + } + } else if !fi.Mode().IsDir() { + return nil, fmt.Errorf("'%v' is not a directory", home) + } else { + if err := pathEnsurePermissions(filepath.Join(home, metaFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update meta file permissions: %w", err) + } + } + + if !ignoreVersionCheck { + meta, err := readMetaFile(home) + if err != nil { + return nil, err + } + if err := checkMeta(meta); err != nil { + return nil, err + } + } + + if err := pathEnsurePermissions(filepath.Join(home, activeDataFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update active file permissions: %w", err) + } + + dataFiles, err := listDataFiles(home) + if err != nil { + return nil, err + } + for _, df := range dataFiles { + if err := pathEnsurePermissions(df.path, mode); err != nil { + return nil, fmt.Errorf("failed to update data file permissions: %w", err) + } + } + if err := pathEnsurePermissions(filepath.Join(home, logFileName), mode); err != nil { + return nil, fmt.Errorf("failed to update log file permissions: %w", err) + } + + tbl := map[string]entry{} + var txid uint64 + if L := len(dataFiles); L > 0 { + active := dataFiles[L-1] + txid = active.txid + if err := loadDataFile(active.path, tbl); err != nil { + return nil, err + } + } + + logp.Info("Loading data file of '%v' succeeded. Active transaction id=%v", home, txid) + + var entries uint + memstore := memstore{tbl} + txid, entries, err = loadLogFile(&memstore, txid, home) + logp.Info("Finished loading transaction log file for '%v'. Active transaction id=%v", home, txid) + + if err != nil { + // Error indicates the log file was incomplete or corrupted. + // Anyways, we already have the table in a valid state and will + // continue opening the store from here. + logp.Warn("Incomplete or corrupted log file in %v. Continue with last known complete and consistent state. Reason: %v", home, err) + } + + diskstore, err := newDiskStore(log, home, dataFiles, txid, mode, entries, err != nil, bufSz, checkpoint) + if err != nil { + return nil, err + } + + return &store{ + disk: diskstore, + mem: memstore, + }, nil } // Close closes access to the update log file and clears the in memory key // value store. Access to the store after close can lead to a panic. func (s *store) Close() error { - panic("TODO: implement me") + s.lock.Lock() + defer s.lock.Unlock() + s.mem = memstore{} + return s.disk.Close() } // Has checks if the key is known. The in memory store does not report any // errors. func (s *store) Has(key string) (bool, error) { - panic("TODO: implement me") + s.lock.RLock() + defer s.lock.RUnlock() + return s.mem.Has(key), nil } // Get retrieves and decodes the key-value pair into to. func (s *store) Get(key string, to interface{}) error { - panic("TODO: implement me") + s.lock.RLock() + defer s.lock.RUnlock() + + dec := s.mem.Get(key) + if dec == nil { + return errKeyUnknown + } + return dec.Decode(to) } // Set inserts or overwrites a key-value pair. // If encoding was successful the in-memory state will be updated and a // set-operation is logged to the diskstore. func (s *store) Set(key string, value interface{}) error { - panic("TODO: implement me") + var tmp common.MapStr + if err := typeconv.Convert(&tmp, value); err != nil { + return err + } + + s.lock.Lock() + defer s.lock.Unlock() + + s.mem.Set(key, tmp) + return s.logOperation(&opSet{K: key, V: tmp}) } // Remove removes a key from the in memory store and logs a remove operation to // the diskstore. The operation does not check if the key exists. func (s *store) Remove(key string) error { - panic("TODO: implement me") + s.lock.Lock() + defer s.lock.Unlock() + + s.mem.Remove(key) + return s.logOperation(&opRemove{K: key}) +} + +// lopOperation ensures that the diskstore reflects the recent changes to the +// in memory store by either triggering a checkpoint operations or adding the +// operation type to the update log file. +func (s *store) logOperation(op op) error { + if s.disk.mustCheckpoint() { + err := s.disk.WriteCheckpoint(s.mem.table) + if err != nil { + // if writing the new checkpoint file failed we try to fallback to + // appending the log operation. + // TODO: make append configurable and retry checkpointing with backoff. + s.disk.LogOperation(op) + } + + return err + } + + return s.disk.LogOperation(op) } // Each iterates over all key-value pairs in the store. func (s *store) Each(fn func(string, backend.ValueDecoder) (bool, error)) error { - panic("TODO: implement me") + s.lock.RLock() + defer s.lock.RUnlock() + + for k, entry := range s.mem.table { + cont, err := fn(k, entry) + if !cont || err != nil { + return err + } + } + + return nil +} + +func (m *memstore) Has(key string) bool { + _, exists := m.table[key] + return exists +} + +func (m *memstore) Get(key string) backend.ValueDecoder { + entry, exists := m.table[key] + if !exists { + return nil + } + return entry } -func (s *memstore) Set(key string, value interface{}) error { - panic("TODO: implement me") +func (m *memstore) Set(key string, value common.MapStr) { + m.table[key] = entry{value: value} } -func (s *memstore) Remove(key string) error { - panic("TODO: implement me") +func (m *memstore) Remove(key string) bool { + _, exists := m.table[key] + if !exists { + return false + } + delete(m.table, key) + return true } func (e entry) Decode(to interface{}) error { diff --git a/libbeat/statestore/backend/memlog/testdata/1/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/Readme.md new file mode 100644 index 000000000000..7532ab9b9425 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/Readme.md @@ -0,0 +1 @@ +Sample disk stores version 1 diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json new file mode 100644 index 000000000000..90bd45d5de1c --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md new file mode 100644 index 000000000000..9c7b4584885d --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/Readme.md @@ -0,0 +1,2 @@ +Store file with transaction ID and log with with transaction id 3. +The data file 2.json is missing, which leads to all entries in the log file to be ignored. diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json new file mode 100644 index 000000000000..f691496a5b23 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json new file mode 100644 index 000000000000..5b5d8f4e0f62 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/log.json @@ -0,0 +1,6 @@ +{"op": "set", "id": 3} +{"K":"key3","V":{"a":3}} +{"op": "set", "id": 3} +{"K":"key3","V":{"a":3}} +{"op": "set", "id": 3} +{"K":"key5","V":{"a":5}} diff --git a/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json new file mode 100644 index 000000000000..bcb050a28217 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/commit_wrong_id/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/1.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/1.json new file mode 100644 index 000000000000..90bd45d5de1c --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/Readme.md new file mode 100644 index 000000000000..5272ae569bd2 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/Readme.md @@ -0,0 +1 @@ +Store with valid data file and valid log file with logged updates. diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/expected.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/expected.json new file mode 100644 index 000000000000..c7a3023283ca --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 4, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/log.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/log.json new file mode 100644 index 000000000000..c7edd4f783b3 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/log.json @@ -0,0 +1,6 @@ +{"op":"set", "id": 2} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 3} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 4} +{"K":"key5","V":{"a":5}} diff --git a/libbeat/statestore/backend/memlog/testdata/1/data_and_log/meta.json b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/meta.json new file mode 100644 index 000000000000..bcb050a28217 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/data_and_log/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/1.json b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/1.json new file mode 100644 index 000000000000..0460d15de490 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/1.json @@ -0,0 +1,8 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2}, +{"_key":"key3","a":3}, +{"_key":"key4","a":4}, +{"_key":"key5","a":5} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/Readme.md new file mode 100644 index 000000000000..d5cc1f554404 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/Readme.md @@ -0,0 +1,2 @@ +Valid store without log.json. All entries are read from 1.json only. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/expected.json b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/expected.json new file mode 100644 index 000000000000..8c643ce7fda2 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/datafile_only/meta.json b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/meta.json new file mode 100644 index 000000000000..bcb050a28217 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/datafile_only/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/1.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/1.json new file mode 100644 index 000000000000..90bd45d5de1c --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/1.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md new file mode 100644 index 000000000000..a3df15b92838 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/Readme.md @@ -0,0 +1,2 @@ +Valid data file with incomplete log.json. The last entry in the log file has +the data part missing, which will fail the log file to be read. diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/expected.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/expected.json new file mode 100644 index 000000000000..f691496a5b23 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 1, + "datafile": "1.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/log.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/log.json new file mode 100644 index 000000000000..50462314cb6a --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/log.json @@ -0,0 +1 @@ +{"op":"set", "id": 2} diff --git a/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/meta.json b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/meta.json new file mode 100644 index 000000000000..bcb050a28217 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/incomplete_op/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md new file mode 100644 index 000000000000..0c8ac8f51abd --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/Readme.md @@ -0,0 +1,3 @@ +Checkpoint file does not exist yet. All entries are read from the log file. +Contents is missing with the last operation in the log file triggereing a parse error. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json new file mode 100644 index 000000000000..356fb55782d9 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 6, + "datafile": "", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json new file mode 100644 index 000000000000..a0cc0a36eb91 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/log.json @@ -0,0 +1,13 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"set", "id": 4} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 5} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 6} +{"K":"key5","V":{"a":5}} +{"op":"set", diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json new file mode 100644 index 000000000000..cd576d0b504a --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_incomplete/meta.json @@ -0,0 +1 @@ +{"version":"1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/Readme.md new file mode 100644 index 000000000000..725042043f30 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/Readme.md @@ -0,0 +1,2 @@ +Checkpoint file does not exist yet. All entries are read from the log file. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/expected.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/expected.json new file mode 100644 index 000000000000..356fb55782d9 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/expected.json @@ -0,0 +1,12 @@ +{ + "txid": 6, + "datafile": "", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2}, + "key3": {"a": 3}, + "key4": {"a": 4}, + "key5": {"a": 5} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/log.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/log.json new file mode 100644 index 000000000000..f00679f22555 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/log.json @@ -0,0 +1,12 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"set", "id": 4} +{"K":"key3","V":{"a":3}} +{"op":"set", "id": 5} +{"K":"key4","V":{"a":4}} +{"op":"set", "id": 6} +{"K":"key5","V":{"a":5}} diff --git a/libbeat/statestore/backend/memlog/testdata/1/logfile_only/meta.json b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/meta.json new file mode 100644 index 000000000000..cd576d0b504a --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/logfile_only/meta.json @@ -0,0 +1 @@ +{"version":"1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/1.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/1.json new file mode 100644 index 000000000000..0460d15de490 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/1.json @@ -0,0 +1,8 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2}, +{"_key":"key3","a":3}, +{"_key":"key4","a":4}, +{"_key":"key5","a":5} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/2.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/2.json new file mode 100644 index 000000000000..c3d3f12f859a --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/2.json @@ -0,0 +1,3 @@ +[ +{"_key":"key0","a":0} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/3.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/3.json new file mode 100644 index 000000000000..9ac6abf9862f --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/3.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":30}, +{"_key":"key1","a":31}, +{"_key":"key2","a":32} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md new file mode 100644 index 000000000000..cdea11598196 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/Readme.md @@ -0,0 +1,2 @@ +Store with old data files that failed to be cleaned up. Only entries from data file 3.json should be loaded. + diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/expected.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/expected.json new file mode 100644 index 000000000000..f671203914aa --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 3, + "datafile": "3.json", + "entries": { + "key0": {"a": 30}, + "key1": {"a": 31}, + "key2": {"a": 32} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/meta.json b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/meta.json new file mode 100644 index 000000000000..bcb050a28217 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_datafiles/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json new file mode 100644 index 000000000000..90bd45d5de1c --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/5.json @@ -0,0 +1,5 @@ +[ +{"_key":"key0","a":0}, +{"_key":"key1","a":1}, +{"_key":"key2","a":2} +] diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md new file mode 100644 index 000000000000..9d99a9ae95aa --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/Readme.md @@ -0,0 +1,3 @@ +Due to restart the log file was not truncated after the last checkpoint was +written. Update with ID 0 adds the removed key0 again to the store. All entries +in log.json should be ignored. diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json new file mode 100644 index 000000000000..4e9547000e72 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/expected.json @@ -0,0 +1,9 @@ +{ + "txid": 5, + "datafile": "5.json", + "entries": { + "key0": {"a": 0}, + "key1": {"a": 1}, + "key2": {"a": 2} + } +} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json new file mode 100644 index 000000000000..426ce6a4f619 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/log.json @@ -0,0 +1,8 @@ +{"op":"set", "id": 1} +{"K":"key0","V":{"a":0}} +{"op":"set", "id": 2} +{"K":"key1","V":{"a":1}} +{"op":"set", "id": 3} +{"K":"key2","V":{"a":2}} +{"op":"remove", "id": 4} +{"K":"key0"} diff --git a/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json new file mode 100644 index 000000000000..bcb050a28217 --- /dev/null +++ b/libbeat/statestore/backend/memlog/testdata/1/old_entries_in_log/meta.json @@ -0,0 +1 @@ +{"version": "1"} diff --git a/libbeat/statestore/backend/memlog/util.go b/libbeat/statestore/backend/memlog/util.go index f02311392b9e..2027c87adca0 100644 --- a/libbeat/statestore/backend/memlog/util.go +++ b/libbeat/statestore/backend/memlog/util.go @@ -20,6 +20,7 @@ package memlog import ( "io" "os" + "runtime" "syscall" ) @@ -76,3 +77,45 @@ func trySyncPath(path string) { defer f.Close() syncFile(f) } + +// pathEnsurePermissions checks if the file permissions for the given file match wantPerm. +// The permissions are updated using chmod if needed. +// No file will be created if the file does not yet exist. +func pathEnsurePermissions(path string, wantPerm os.FileMode) error { + f, err := os.OpenFile(path, os.O_RDWR, wantPerm) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + defer f.Close() + return fileEnsurePermissions(f, wantPerm) +} + +// fileEnsurePermissions checks if the file permissions for the given file +// matches wantPerm. If not fileEnsurePermissions tries to update +// the current permissions via chmod. +// The file is not created or updated if it does not exist. +func fileEnsurePermissions(f *os.File, wantPerm os.FileMode) error { + if runtime.GOOS == "windows" { + return nil + } + + fi, err := f.Stat() + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + + wantPerm = wantPerm & os.ModePerm + perm := fi.Mode() & os.ModePerm + if wantPerm == perm { + return nil + } + + return f.Chmod((fi.Mode() &^ os.ModePerm) | wantPerm) +} From 6bfe793be95de5c1e6a7ef2dbb1bfe74d063d903 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 2 Jul 2020 07:16:27 -0600 Subject: [PATCH 25/35] Accept prefix as metric_types for stackdriver metricset in GCP (#19345) * Add metric_types for stackdriver metricset in GCP --- CHANGELOG.next.asciidoc | 1 + .../stackdriver/_meta/docs.asciidoc | 42 +++++++++-- .../stackdriver/metrics_requester.go | 7 +- .../googlecloud/stackdriver/metricset.go | 72 +++++++++++-------- 4 files changed, 82 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 331f348f9927..b8a100c089fe 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -517,6 +517,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add memory metrics into compute googlecloud. {pull}18802[18802] - Add new fields to HAProxy module. {issue}18523[18523] - Add Tomcat overview dashboard {pull}14026[14026] +- Accept prefix as metric_types config parameter in googlecloud stackdriver metricset. {pull}19345[19345] - Update Couchbase to version 6.5 {issue}18595[18595] {pull}19055[19055] - Add dashboards for googlecloud load balancing metricset. {pull}18369[18369] - Add support for v1 consumer API in Cloud Foundry module, use it by default. {pull}19268[19268] diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc index 47ff699566cb..2f42a919e3d0 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/_meta/docs.asciidoc @@ -16,19 +16,20 @@ call. [float] == Metricset config and parameters -* *metric_types*: Required, a list of metric type strings. Each call of the +* *metric_types*: Required, a list of metric type strings, or a list of metric +type prefixes. For example, `instance/cpu` is the prefix for metric type +`instance/cpu/usage_time`, `instance/cpu/utilization` etc Each call of the `ListTimeSeries` API can return any number of time series from a single metric type. Metric type is to used for identifying a specific time series. * *aligner*: A single string with which aggregation operation need to be applied onto time series data for ListTimeSeries API. If it's not given, default aligner -is set to be `ALIGN_NONE`. Google Cloud also supports `ALIGN_DELTA`, `ALIGN_RATE`, -`ALIGN_MIN`, `ALIGN_MAX`, `ALIGN_MEAN`, `ALIGN_COUNT`, `ALIGN_SUM` and etc. +is `ALIGN_NONE`. Google Cloud also supports `ALIGN_DELTA`, `ALIGN_RATE`, +`ALIGN_MIN`, `ALIGN_MAX`, `ALIGN_MEAN`, `ALIGN_COUNT`, `ALIGN_SUM` etc. Please see https://cloud.google.com/monitoring/api/ref_v3/rpc/google.monitoring.v3#aligner[Aggregation Aligner] for the full list of aligners. - [float] === Example Configuration * `stackdriver` metricset is enabled to collect metrics from all zones under @@ -37,7 +38,7 @@ are specified: first one is to collect CPU usage time and utilization with aggregation aligner ALIGN_MEAN; second one is to collect uptime with aggregation aligner ALIGN_SUM. These metric types all have 240 seconds ingest delay time and 60 seconds sample period. With `period` specified as `300s` in the config below, -Metricbeat will collect compute metrics from googlecloud every 5-minute with +Metricbeat will collect compute metrics from Google Cloud every 5-minute with given aggregation aligner applied for each metric type. + [source,yaml] @@ -69,7 +70,7 @@ are specified: first one is to collect CPU usage time and utilization with aggregation aligner ALIGN_MEAN; second one is to collect uptime with aggregation aligner ALIGN_SUM. These metric types all have 240 seconds ingest delay time and 60 seconds sample period. With `period` specified as `60s` in the config below, -Metricbeat will collect compute metrics from googlecloud every minute with no +Metricbeat will collect compute metrics from Google Cloud every minute with no aggregation. This case, the aligners specified in the configuration will be ignored. + @@ -94,3 +95,32 @@ ignored. metric_types: - "instance/uptime" ---- + +* `stackdriver` metricset is enabled to collect metrics from all zones under +`europe-west1-c` region in `elastic-observability` project. One set of metrics +will be collected: metric types that starts with `instance/cpu` under `compute` +service with aligner ALIGN_NONE. These metric types all have 240 seconds ingest +delay time and 60 seconds sample period. With `period` specified as `1m` in +the config below, Metricbeat will collect compute metrics from Google Cloud +every minute with no aggregation. The metric types in `compute` service with +`instance/cpu` prefix are: `instance/cpu/reserved_cores`, +`instance/cpu/scheduler_wait_time`, `instance/cpu/usage_time`, and +`instance/cpu/utilization`. + ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - stackdriver + zone: "europe-west1-c" + project_id: elastic-observability + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + metrics: + - aligner: ALIGN_NONE + service: compute + metric_types: + - "instance/cpu" +---- diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 4b48cf06ff0a..a3f9a3250387 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -76,17 +76,16 @@ func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, sdc stackDriv results := make([]timeSeriesWithAligner, 0) aligner := sdc.Aligner - serviceName := sdc.ServiceName - for _, mt := range sdc.MetricTypes { + for mt, meta := range metricsMeta { wg.Add(1) + metricMeta := meta go func(mt string) { defer wg.Done() - metricMeta := metricsMeta[mt] r.logger.Debugf("For metricType %s, metricMeta = %s", mt, metricMeta) interval, aligner := getTimeIntervalAligner(metricMeta.ingestDelay, metricMeta.samplePeriod, r.config.period, aligner) - ts := r.Metric(ctx, serviceName+".googleapis.com/"+mt, interval, aligner) + ts := r.Metric(ctx, mt, interval, aligner) lock.Lock() defer lock.Unlock() results = append(results, ts) diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go index 053db176f211..a81bca2b86e9 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go @@ -9,13 +9,12 @@ import ( "fmt" "time" - "github.com/golang/protobuf/ptypes/duration" - monitoring "cloud.google.com/go/monitoring/apiv3" - + "github.com/golang/protobuf/ptypes/duration" "github.com/pkg/errors" - + "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/genproto/googleapis/api/metric" monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" "github.com/elastic/beats/v7/libbeat/common" @@ -216,41 +215,54 @@ func (mc *stackDriverConfig) Validate() error { // (sample period and ingest delay) of each given metric type func (m *MetricSet) metricDescriptor(ctx context.Context, client *monitoring.MetricClient) (map[string]metricMeta, error) { metricsWithMeta := make(map[string]metricMeta, 0) + req := &monitoringpb.ListMetricDescriptorsRequest{ + Name: "projects/" + m.config.ProjectID, + } for _, sdc := range m.stackDriverConfig { for _, mt := range sdc.MetricTypes { - req := &monitoringpb.ListMetricDescriptorsRequest{ - Name: "projects/" + m.config.ProjectID, - Filter: fmt.Sprintf(`metric.type = "%s"`, sdc.ServiceName+".googleapis.com/"+mt), - } - + req.Filter = fmt.Sprintf(`metric.type = starts_with("%s")`, sdc.ServiceName+".googleapis.com/"+mt) it := client.ListMetricDescriptors(ctx, req) - out, err := it.Next() - if err != nil { - err = errors.Errorf("Could not make ListMetricDescriptors request: %s: %v", mt, err) - m.Logger().Error(err) - return metricsWithMeta, err - } - // Set samplePeriod default to 60 seconds and ingestDelay default to 0. - meta := metricMeta{ - samplePeriod: 60 * time.Second, - ingestDelay: 0 * time.Second, + for { + out, err := it.Next() + if err != nil && err != iterator.Done { + err = errors.Errorf("Could not make ListMetricDescriptors request for metric type %s: %v", mt, err) + m.Logger().Error(err) + return metricsWithMeta, err + } + + if out != nil { + metricsWithMeta = m.getMetadata(out, metricsWithMeta) + } + + if err == iterator.Done { + break + } } + } + } - if out.Metadata.SamplePeriod != nil { - m.Logger().Debugf("For metric type %s: sample period = %s", mt, out.Metadata.SamplePeriod) - meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second - } + return metricsWithMeta, nil +} - if out.Metadata.IngestDelay != nil { - m.Logger().Debugf("For metric type %s: ingest delay = %s", mt, out.Metadata.IngestDelay) - meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second - } +func (m *MetricSet) getMetadata(out *metric.MetricDescriptor, metricsWithMeta map[string]metricMeta) map[string]metricMeta { + // Set samplePeriod default to 60 seconds and ingestDelay default to 0. + meta := metricMeta{ + samplePeriod: 60 * time.Second, + ingestDelay: 0 * time.Second, + } - metricsWithMeta[mt] = meta - } + if out.Metadata.SamplePeriod != nil { + m.Logger().Debugf("For metric type %s: sample period = %s", out.Type, out.Metadata.SamplePeriod) + meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second } - return metricsWithMeta, nil + if out.Metadata.IngestDelay != nil { + m.Logger().Debugf("For metric type %s: ingest delay = %s", out.Type, out.Metadata.IngestDelay) + meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second + } + + metricsWithMeta[out.Type] = meta + return metricsWithMeta } From 1a3cc632ef6b2799b9696bf6696764223d8cd38b Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 2 Jul 2020 15:36:54 +0200 Subject: [PATCH 26/35] [Ingest Manager] Make Agent beta and Constraints experimental (#19586) * make agent beta * make constraints experimental --- .../elastic-agent/docs/elastic-agent_configuration_example.yml | 2 ++ .../pkg/agent/application/filters/constraints_filter.go | 1 + x-pack/elastic-agent/pkg/agent/warn/warn.go | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml index 5673f4e724e1..261862e7e7f2 100644 --- a/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml +++ b/x-pack/elastic-agent/docs/elastic-agent_configuration_example.yml @@ -110,6 +110,7 @@ inputs: name: epm/nginx version: 1.7.0 dataset.namespace: prod + # constraints are still Experimental and should not be used in production. constraints?: # Contraints look are not final - os.platform: { in: "windows" } @@ -129,6 +130,7 @@ inputs: name: epm/nginx version: 1.7.0 dataset.namespace: prod + # constraints are still Experimental and should not be used in production. constraints?: # Contraints look are not final - os.platform: { in: "windows" } diff --git a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go index c891cefb056f..d9abd6c7aa31 100644 --- a/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go +++ b/x-pack/elastic-agent/pkg/agent/application/filters/constraints_filter.go @@ -28,6 +28,7 @@ var ( ) // ConstraintFilter filters ast based on included constraints. +// constraints are still Experimental and should not be used in production. func ConstraintFilter(log *logger.Logger, ast *transpiler.AST) error { // get datasources inputsNode, found := transpiler.Lookup(ast, inputsKey) diff --git a/x-pack/elastic-agent/pkg/agent/warn/warn.go b/x-pack/elastic-agent/pkg/agent/warn/warn.go index c3b97079aa11..03d746992f6e 100644 --- a/x-pack/elastic-agent/pkg/agent/warn/warn.go +++ b/x-pack/elastic-agent/pkg/agent/warn/warn.go @@ -11,7 +11,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -const message = "The Elastic Agent is currently in Experimental and should not be used in production" +const message = "The Elastic Agent is currently in BETA and should not be used in production" // LogNotGA warns the users in the log that the Elastic Agent is not GA. func LogNotGA(log *logger.Logger) { From afffe2bac08830bf17f44bc6ebcac112c6aa9162 Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 2 Jul 2020 09:12:26 -0500 Subject: [PATCH 27/35] [Filebeat] Fix tls mapping in suricata module (#19494) * Fix tls mapping in suricata module - add suricata.eve.tls.ja3s.string field - add suricata.eve.tls.ja3s.hash field - add suricata.eve.tls.ja3.string field - add suricata.eve.tls.ja3.hash field - set default_field to false for ja3 & ja3s fields - map suricata.eve.tls.ja3.hash to tls.client.ja3 - map suricata.eve.tls.ja3s.hash to tls.server.ja3s - perform suricata.eve.tls.* -> tls.* mappings for all event types Closes #19492 --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 30 ++++ .../module/suricata/eve/_meta/fields.yml | 18 +++ .../module/suricata/eve/config/eve.yml | 37 +++-- .../module/suricata/eve/test/eve-alerts.log | 2 + .../eve/test/eve-alerts.log-expected.json | 141 ++++++++++++++++++ .../eve/test/eve-small.log-expected.json | 3 + x-pack/filebeat/module/suricata/fields.go | 2 +- 8 files changed, 213 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b8a100c089fe..4726420da201 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -199,6 +199,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix date and timestamp formats for fortigate module {pull}19316[19316] - Fix memory leak in tcp and unix input sources. {pull}19459[19459] - Add missing `default_field: false` to aws filesets fields.yml. {pull}19568[19568] +- Fix tls mapping in suricata module {issue}19492[19492] {pull}19494[19494] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 71c438adde9a..6e8112eb2689 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -34596,6 +34596,36 @@ type: keyword -- + +*`suricata.eve.tls.ja3s.string`*:: ++ +-- +type: keyword + +-- + +*`suricata.eve.tls.ja3s.hash`*:: ++ +-- +type: keyword + +-- + + +*`suricata.eve.tls.ja3.string`*:: ++ +-- +type: keyword + +-- + +*`suricata.eve.tls.ja3.hash`*:: ++ +-- +type: keyword + +-- + *`suricata.eve.app_proto_ts`*:: + -- diff --git a/x-pack/filebeat/module/suricata/eve/_meta/fields.yml b/x-pack/filebeat/module/suricata/eve/_meta/fields.yml index 7529cba1b525..45980b888b00 100644 --- a/x-pack/filebeat/module/suricata/eve/_meta/fields.yml +++ b/x-pack/filebeat/module/suricata/eve/_meta/fields.yml @@ -665,6 +665,24 @@ - name: subject type: keyword + - name: ja3s + type: group + default_field: false + fields: + - name: string + type: keyword + - name: hash + type: keyword + + - name: ja3 + type: group + default_field: false + fields: + - name: string + type: keyword + - name: hash + type: keyword + - name: app_proto_ts type: keyword diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml index 700c8db17e98..438c30e63731 100644 --- a/x-pack/filebeat/module/suricata/eve/config/eve.yml +++ b/x-pack/filebeat/module/suricata/eve/config/eve.yml @@ -373,26 +373,23 @@ processors: addTlsVersion(evt); cleanupTlsSni(evt); } - - if: - equals: - suricata.eve.event_type: tls - then: - - convert: - ignore_missing: true - ignore_failure: true - mode: copy - fields: - - {from: suricata.eve.tls.subject, to: tls.server.subject} - - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer} - - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean} - - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1} - - {from: suricata.eve.tls.sni, to: tls.client.server_name} - - {from: suricata.eve.tls.sni, to: destination.domain} - - {from: suricata.eve.tls.notbefore, to: tls.server.not_before} - - {from: suricata.eve.tls.notafter, to: tls.server.not_after} - - {from: suricata.eve.tls.ja3s, to: tls.server.ja3s} - - {from: suricata.eve.tls.certificate, to: tls.server.certificate} - - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain} + - convert: + ignore_missing: true + fail_on_error: false + mode: copy + fields: + - {from: suricata.eve.tls.subject, to: tls.server.subject} + - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer} + - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean} + - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1} + - {from: suricata.eve.tls.sni, to: tls.client.server_name} + - {from: suricata.eve.tls.sni, to: destination.domain} + - {from: suricata.eve.tls.notbefore, to: tls.server.not_before} + - {from: suricata.eve.tls.notafter, to: tls.server.not_after} + - {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s} + - {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3} + - {from: suricata.eve.tls.certificate, to: tls.server.certificate} + - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain} - drop_fields: ignore_missing: true fields: diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log index 81bc39dbf4fe..6587a913f52a 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log @@ -18,3 +18,5 @@ {"timestamp":"2018-10-04T09:35:00.897009+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":7,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/binary-amd64\/by-hash\/SHA256\/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":2687},"app_proto":"http","flow":{"pkts_toserver":330,"pkts_toclient":591,"bytes_toserver":23758,"bytes_toclient":884342,"start":"2018-10-04T09:34:58.926006+0000"}} {"timestamp":"2018-10-04T09:35:01.362208+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":8,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/universe\/i18n\/by-hash\/SHA256\/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":524,"pkts_toclient":979,"bytes_toserver":36819,"bytes_toclient":1467603,"start":"2018-10-04T09:34:58.926006+0000"}} {"timestamp":"2018-10-04T09:35:01.575088+0000","flow_id":112424506237238,"in_iface":"enp0s3","event_type":"alert","src_ip":"192.168.1.146","src_port":52340,"dest_ip":"91.189.91.23","dest_port":80,"proto":"TCP","tx_id":9,"alert":{"action":"allowed","gid":1,"signature_id":2013504,"rev":5,"signature":"ET POLICY GNU\/Linux APT User-Agent Outbound likely related to package management","category":"Not Suspicious Traffic","severity":3},"http":{"hostname":"archive.ubuntu.com","url":"\/ubuntu\/dists\/bionic-updates\/multiverse\/binary-amd64\/by-hash\/SHA256\/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16","http_user_agent":"Debian APT-HTTP\/1.3 (1.6.3ubuntu0.1)","http_method":"GET","protocol":"HTTP\/1.1","length":0},"app_proto":"http","flow":{"pkts_toserver":575,"pkts_toclient":1079,"bytes_toserver":40452,"bytes_toclient":1618380,"start":"2018-10-04T09:34:58.926006+0000"}} +{"tls":{"ja3s":{"string":"333,55555,66666-22","hash":"0993626a07ad09e1ce91293be7aa5721"},"ja3":{"string":"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0","hash":"d92325c876e7279f4eb8c62415e3a6b7"},"notafter":"2024-07-16T14:52:35","notbefore":"2019-07-17T14:52:35","version":"TLS 1.2","sni":"hostname.domain.net","fingerprint":"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33","serial":"00:11:22:33:44:55:66:77:88","issuerdn":"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN","subject":"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com"},"proto":"TCP","dest_port":9080,"dest_ip":"10.232.0.237","src_port":45884,"src_ip":"10.126.2.140","event_type":"tls","in_iface":"enp5s0","flow_id":1091813059495729,"timestamp":"2018-10-04T09:35:02.796615+0000"} +{"flow":{"start":"2020-06-26T11:00:02.970011-0400","bytes_toclient":4660,"bytes_toserver":1074,"pkts_toclient":8,"pkts_toserver":7},"app_proto":"tls","tls":{"ja3s":{"string":"742,48172,30210-30","hash":"391231ba5675e42807b9e1f457b2614e"},"ja3":{"string":"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3","hash":"3f1ea03f5822e8021b60cc3e4b233181"},"notafter":"2026-06-25T17:36:29","notbefore":"2016-06-27T17:36:29","version":"TLS 1.2","sni":"host.domain.net","fingerprint":"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc","serial":"72:A9:2C:51","issuerdn":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown","subject":"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown"},"alert":{"severity":3,"category":"","signature":"SURICATA TLS on unusual port","rev":1,"signature_id":2610003,"gid":1,"action":"allowed"},"proto":"TCP","dest_port":8443,"dest_ip":"10.128.2.48","src_port":64389,"src_ip":"10.137.3.54","event_type":"alert","in_iface":"enp0s31f6","flow_id":991192778198299,"timestamp":"2020-06-26T11:00:03.342282-0400"} diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 793ce1647464..6e06de5c66ea 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -1536,5 +1536,146 @@ "user_agent.original": "Debian APT-HTTP/1.3 (1.6.3ubuntu0.1)", "user_agent.os.name": "Debian", "user_agent.version": "1.3" + }, + { + "@timestamp": "2018-10-04T09:35:02.796Z", + "destination.address": "10.232.0.237", + "destination.domain": "hostname.domain.net", + "destination.ip": "10.232.0.237", + "destination.port": 9080, + "event.category": [ + "network" + ], + "event.dataset": "suricata.eve", + "event.kind": "event", + "event.module": "suricata", + "event.original": "{\"tls\":{\"ja3s\":{\"string\":\"333,55555,66666-22\",\"hash\":\"0993626a07ad09e1ce91293be7aa5721\"},\"ja3\":{\"string\":\"001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0\",\"hash\":\"d92325c876e7279f4eb8c62415e3a6b7\"},\"notafter\":\"2024-07-16T14:52:35\",\"notbefore\":\"2019-07-17T14:52:35\",\"version\":\"TLS 1.2\",\"sni\":\"hostname.domain.net\",\"fingerprint\":\"00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33\",\"serial\":\"00:11:22:33:44:55:66:77:88\",\"issuerdn\":\"CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN\",\"subject\":\"C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com\"},\"proto\":\"TCP\",\"dest_port\":9080,\"dest_ip\":\"10.232.0.237\",\"src_port\":45884,\"src_ip\":\"10.126.2.140\",\"event_type\":\"tls\",\"in_iface\":\"enp5s0\",\"flow_id\":1091813059495729,\"timestamp\":\"2018-10-04T09:35:02.796615+0000\"}", + "event.type": [ + "protocol" + ], + "fileset.name": "eve", + "input.type": "log", + "log.offset": 16546, + "network.community_id": "1:qsGDjYDIWp+kHhxotTdhPbUaWSo=", + "network.protocol": "tls", + "network.transport": "tcp", + "related.hash": [ + "00112233445566778899AABBCCDDEEFF00112233" + ], + "related.ip": [ + "10.126.2.140", + "10.232.0.237" + ], + "service.type": "suricata", + "source.address": "10.126.2.140", + "source.ip": "10.126.2.140", + "source.port": 45884, + "suricata.eve.event_type": "tls", + "suricata.eve.flow_id": 1091813059495729, + "suricata.eve.in_iface": "enp5s0", + "suricata.eve.tls.fingerprint": "00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff:00:11:22:33", + "suricata.eve.tls.issuerdn": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN", + "suricata.eve.tls.ja3.hash": "d92325c876e7279f4eb8c62415e3a6b7", + "suricata.eve.tls.ja3.string": "001,22222-33333-00-44444-66666-333-333-55555-55555-22-55555-44444-22-33-66666-77777-22-88888-99999-333-22-66666-77777-22-99999-96611-22-33-88888-33333-88888-333-22222-33333-333-222-88888-444-99999-22222-666-777-888,22-33-44-55-6,77-88-99-0-11-11-22-33-44-55,0", + "suricata.eve.tls.ja3s.hash": "0993626a07ad09e1ce91293be7aa5721", + "suricata.eve.tls.ja3s.string": "333,55555,66666-22", + "suricata.eve.tls.notafter": "2024-07-16T14:52:35", + "suricata.eve.tls.notbefore": "2019-07-17T14:52:35", + "suricata.eve.tls.serial": "00:11:22:33:44:55:66:77:88", + "suricata.eve.tls.sni": "hostname.domain.net", + "suricata.eve.tls.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com", + "suricata.eve.tls.version": "TLS 1.2", + "tags": [ + "suricata" + ], + "tls.client.ja3": "d92325c876e7279f4eb8c62415e3a6b7", + "tls.client.server_name": "hostname.domain.net", + "tls.server.hash.sha1": "00112233445566778899AABBCCDDEEFF00112233", + "tls.server.issuer": "CN=UNKNOWN/DC=UNKNOWN/DC=UNKNOWN/C=UNKNOWN/ST=UNKNOWN/O=UNK-NOWN/OU=UNKNOWN", + "tls.server.ja3s": "0993626a07ad09e1ce91293be7aa5721", + "tls.server.not_after": "2024-07-16T14:52:35", + "tls.server.not_before": "2019-07-17T14:52:35", + "tls.server.subject": "C=UNKNOWN, ST=UNKNOWN, L=UNKNOWN, O=UNKNOWN, OU=UNKNOWN, CN=hostname.domain.net/emailAddress=user@domain.com", + "tls.version": "1.2", + "tls.version_protocol": "tls" + }, + { + "@timestamp": "2020-06-26T15:00:03.342Z", + "destination.address": "10.128.2.48", + "destination.bytes": 4660, + "destination.domain": "host.domain.net", + "destination.ip": "10.128.2.48", + "destination.packets": 8, + "destination.port": 8443, + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "suricata.eve", + "event.kind": "alert", + "event.module": "suricata", + "event.original": "{\"flow\":{\"start\":\"2020-06-26T11:00:02.970011-0400\",\"bytes_toclient\":4660,\"bytes_toserver\":1074,\"pkts_toclient\":8,\"pkts_toserver\":7},\"app_proto\":\"tls\",\"tls\":{\"ja3s\":{\"string\":\"742,48172,30210-30\",\"hash\":\"391231ba5675e42807b9e1f457b2614e\"},\"ja3\":{\"string\":\"718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3\",\"hash\":\"3f1ea03f5822e8021b60cc3e4b233181\"},\"notafter\":\"2026-06-25T17:36:29\",\"notbefore\":\"2016-06-27T17:36:29\",\"version\":\"TLS 1.2\",\"sni\":\"host.domain.net\",\"fingerprint\":\"36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc\",\"serial\":\"72:A9:2C:51\",\"issuerdn\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\",\"subject\":\"C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown\"},\"alert\":{\"severity\":3,\"category\":\"\",\"signature\":\"SURICATA TLS on unusual port\",\"rev\":1,\"signature_id\":2610003,\"gid\":1,\"action\":\"allowed\"},\"proto\":\"TCP\",\"dest_port\":8443,\"dest_ip\":\"10.128.2.48\",\"src_port\":64389,\"src_ip\":\"10.137.3.54\",\"event_type\":\"alert\",\"in_iface\":\"enp0s31f6\",\"flow_id\":991192778198299,\"timestamp\":\"2020-06-26T11:00:03.342282-0400\"}", + "event.severity": 3, + "event.start": "2020-06-26T15:00:02.970Z", + "event.type": [ + "allowed" + ], + "fileset.name": "eve", + "input.type": "log", + "log.offset": 17606, + "message": "", + "network.bytes": 5734, + "network.community_id": "1:W6fjhboFUwyEchJ3ELaqSBzDEJE=", + "network.packets": 15, + "network.protocol": "tls", + "network.transport": "tcp", + "related.hash": [ + "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC" + ], + "related.ip": [ + "10.137.3.54", + "10.128.2.48" + ], + "rule.id": "2610003", + "rule.name": "SURICATA TLS on unusual port", + "service.type": "suricata", + "source.address": "10.137.3.54", + "source.bytes": 1074, + "source.ip": "10.137.3.54", + "source.packets": 7, + "source.port": 64389, + "suricata.eve.alert.category": "", + "suricata.eve.alert.gid": 1, + "suricata.eve.alert.rev": 1, + "suricata.eve.alert.signature": "SURICATA TLS on unusual port", + "suricata.eve.alert.signature_id": 2610003, + "suricata.eve.event_type": "alert", + "suricata.eve.flow_id": 991192778198299, + "suricata.eve.in_iface": "enp0s31f6", + "suricata.eve.tls.fingerprint": "36:3f:ee:2a:1c:fa:de:ad:be:ef:42:99:cf:a9:b0:91:01:eb:a9:cc", + "suricata.eve.tls.issuerdn": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "suricata.eve.tls.ja3.hash": "3f1ea03f5822e8021b60cc3e4b233181", + "suricata.eve.tls.ja3.string": "718,4682-2687-2686-41992-41911-53292-53297-41969-22905-41926-41924-94181-94711-15-23-95-12-11-205,0-33-50-53-6-61-39-23-34-85-81,93-04-52,3-9-3", + "suricata.eve.tls.ja3s.hash": "391231ba5675e42807b9e1f457b2614e", + "suricata.eve.tls.ja3s.string": "742,48172,30210-30", + "suricata.eve.tls.notafter": "2026-06-25T17:36:29", + "suricata.eve.tls.notbefore": "2016-06-27T17:36:29", + "suricata.eve.tls.serial": "72:A9:2C:51", + "suricata.eve.tls.sni": "host.domain.net", + "suricata.eve.tls.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "suricata.eve.tls.version": "TLS 1.2", + "tags": [ + "suricata" + ], + "tls.client.ja3": "3f1ea03f5822e8021b60cc3e4b233181", + "tls.client.server_name": "host.domain.net", + "tls.server.hash.sha1": "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC", + "tls.server.issuer": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "tls.server.ja3s": "391231ba5675e42807b9e1f457b2614e", + "tls.server.not_after": "2026-06-25T17:36:29", + "tls.server.not_before": "2016-06-27T17:36:29", + "tls.server.subject": "C=Unknown, ST=Unknown, L=Unknown, O=Unknown, OU=Unknown, CN=Unknown", + "tls.version": "1.2", + "tls.version_protocol": "tls" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index ec02bba8dd18..5d44c5bd12f2 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -43,6 +43,7 @@ "@timestamp": "2018-07-05T19:07:20.910Z", "destination.address": "192.168.156.70", "destination.bytes": 343, + "destination.domain": "l2.io", "destination.ip": "192.168.156.70", "destination.packets": 3, "destination.port": 443, @@ -96,6 +97,8 @@ "tags": [ "suricata" ], + "tls.client.server_name": "l2.io", + "tls.resumed": true, "tls.version": "1.2", "tls.version_protocol": "tls" }, diff --git a/x-pack/filebeat/module/suricata/fields.go b/x-pack/filebeat/module/suricata/fields.go index 578c30c9d6bf..4cba3d5ee745 100644 --- a/x-pack/filebeat/module/suricata/fields.go +++ b/x-pack/filebeat/module/suricata/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSuricata returns asset data. // This is the base64 encoded gzipped contents of module/suricata. func AssetSuricata() string { - return "eJzsXEuP27oV3udXaDerGE3QBMUsim7SRYG2iwDdEsfkkcyar5CUPe6vLyR7PLJFyuIDc3Fzk1UwCT+e94tH87HZ4+m5cb3lFDx8aBrPvcDn5vvbTxg6arnxXKvn5q8fmqZp/qlZL7BptW12oJjgqmv8Dptv//nW/OP7v//VCN25xljNeoqs2Z6ueJsPTdNyFMw9j0gfGwUSbygY/viTweems7o3l58EqBj+/H3Ealqr5UjB6z0jKUJ3TcsFbi7/fXrx9HI84PVnobsX7p/QgC9GW39mdyaMyYF7Ku4oUZ4MFNz88ytRezwdtWXXfwtigDHEWO010ZZ3+TiemuDhe8nEeLpBIq2AGymsJucNxrhdBGCrtUBQjwCudBBPy0gBui8jxZ1UIYAHf28kiUxMxFGoGet8GTctzxLH23GBXLW6kr26HXwqE8hA0PC3CAoIDnORG/C789HN8NeH6nshnEUuEFp172BBzmuLMRpWqr4DE7O+ta6wg89fvpZxItmXQlHw/2Vr++Zs0MA5lSaeFuLafj3P0HkypKfg+RCBZ/KGc1zBkPU2N8eDtzhLcy5xurcUV+CPWS0RXKE/arvfeAvKrbiCgiFUhVl4LOdBAjycNh/yPzkX1qG6P56dj+eGlBrwbQ2MhQi5FoO91YyZEKVh1HtRcNpSzQplkEH9NU0JfZxzn1AjogR+z3521vXg+6xC5CbKJbvfNMY98sExDgdUtlbgO+9rFdWL4lpKOQMNG4vOaOVwc4a55Slmq8i4RRor9Vaa63A96R1aAh3OAu0aBt4Ob4YOhysQjy4dMwfVMU9NId1iizZf7D96dH4zgtgJTuTG3sZIXhSQFasls9POZ1aqwzVMS+APKzSBqvOx5m29rW41O222J49ulaYk+p2OBcfVqrpDWbqQauUj/Xta580lOg8yNZI9/e168ulBHFOEt0BLpgwCI4Veeiij4LHT9lRYgeMBLfcxlCWFj2OXzQwgGgcPBRm/K2vaeKfA97awZgA6pLtsSeneUy0fJowrsTkV1hVkNvXJNjPB4wknBBcDvMsr5IDWhcS5UikTXnXrj2CxFPHNJewhmqx+Io6nNVGtJomCWfCzPOHt0SoUxADdY2DmtiYQzMCY1YEBShYUb/PBrhWLGXJQQYRj6OM1Zp7YQ7kqjahA3V5GkkRJYX4sTW0SZe/msk5hrOUCyTjGq8qeNqjIgF1mTAxbC/dPGIWChxcygJIdL/ZAbg5/jkKEyYsTeFv+6T5I3Tr6Juq10ElUVbAsgnMotyIw8V2LNpHcfGD7S3KP3VXoY1V38HToaHOjyCTsswrBrDwiokTbEakZElQeQ48T2YA6VEqloc2fVFMROJVLrpMAEo9c60CcgUDSePcEFhJpUX3rsGeatMDD3pokIqfI2azHKq1Ua8qh9YSBh9EWBZiRzFIi0Q2Fd2kuPMutlJaTCr2s58tJaStBVBFT4MH+3Yx9ytwBBGeE7pDuXS+LlT8muYpWerHMOjoU3PkaygvtJyQKylsESRgavyMWge6Ko8O1KjiRKqYxweuKFfAaYBhvWxJ8ZEvDU5oEa5e0boBqVnmcAYeOmL0nwef7NBZvB9N5GOUFgoU8IU8iuSml4SBAkR9c/SjEeVK9EE+FIMIPKFGQ/GajV64355W/8MR/LaUT0e898VoTJyFIclauKJRfV1zlLbbJFfrFlWa9r9HvfyVckQrUcLPkqPlG+VofcLMwUU2OB/q3bw/Z7LGqqSCuug43xL3SOoMGBo2JdSa81EppYwddWtRLI0odzy0E8N9b34zWmWIzQb9Dq7C0qh0ic62Ahoh/+dPnTzB/5k+q8QKNaPmcnXRCb6HUhi5Y4b3Z95/cC30ksqtbDlt9dGTbu/kjehp/A3GOXMaxVbCUroM2cuj23JjiBo4K7ZARY3tVjKXwWAfoLC2LUh+KsbYnM7SUlVgcV5DrsXgxB8JVeet8RhxnKuVt/QAl4UVgaZC3lYlCaQLbMSkxB4whAk6Ve/DFwUDBY1Cw7Eh+BwrWY6kowffjDGLGqTQJV9OpYEy5ikjhyUUqkpM15VSHJunye6ilNaYcFC6Dk71khVG0htbS/hmslgFss1Gujv/yK5r8wQNAHW/7+dwkbs1x/1hek1Pab7GN7w+xabMUweDO9WhZbB917bKj4mUAsYFZGhXn51Ri0fWy9NvIlqsOrbE8ura6fhGUB/rfJAylPbQ+WvutUbTrt//N/IBkWoKed1Ajm56rUAIlZ64DjM8+xOvF7eKllerp90erPm5wHgLrjI8vumy535yOXGH2vhpP99uukSuhK9karfAx86siF5eml5i+fMe5SocWwZWGmouSSuldqR9UsWC2xvPHFdy8cDjz/KDjPv4CePYNWDgzBj4HXZtVJ7Hp/vdLJMSmQAmUG5ssNZ7MBJZoZhK4IK3V812TJJgdiixC5sLFF4N0bkxJ4X/+y0gey/j/AQAA//93dIZ1" + return "eJzsXEuP67YV3t9fod2sYjRpExSzKLpJFwXaLgJ0SxyTRxJjvi5J2eP++kKyxyNbpCw+MEWTO6uBZ/jxvHhePPR3zQHPr40bLKfg4UvTeO4Fvja/fHzC0FHLjedavTZ/+dI0TfMPzQaBTatt04Nigquu8T02P//75+bvv/zrn43QnWuM1WygyJr9+Ya3+9I0LUfB3OuE9F2jQOIdBeOPPxt8bTqrB3P9JEDF+PO3CatprZYTBe/7TKQI3TUtF7i7/vt84/nmeMTbZ6G9V/af0YBvRlt/YXchjNmCRyoeKFGejBTc/fmdqAOeT9qy29+CGGAMMVZ7TbTlXT6Opya4+FEyMZ7ukEgr4E4Km8n5gDGujwDstRYI6hnAjQ7iaRkpQA9lpLizKgTw4B+NJJGJmTgKNWOdL+Om5Vni+FgukKtWV7JX18P3ZQIZCRp/i6CA4LAUuQHfX5buxl+fqu+NcBbZQGjVfYIFOa8txmjYqPoOTMz6th6FHn748acyTiT7sVAU/D/Z2r5bGzRwTqWJh4W4tt/XM3SejOEpuD5E4IW8cR1XMEa93d3y4C7O0pxNnB4sxQ34U1RLBFfoT9oedt6Cchu2oGAIVWEWnst5lAAPh82n/M/WhXWoHpdnx+OlIaU6fFsDY8VDbsVgHzljJkSpG/VeFKy2VLNCGWRQfwtTQp+W3CfkiCiBP7KfHXU9+CErEbnzcsnHb+7jnp3ByQ8HVLZV4L33tZLqVXGthZyRhp1FZ7RyuLvA3PMUs1Vk3CKNpXobzXXcngwOLYEOF452CwMfi3djhcMViGebTpGD6thJTSHdYos2X+xfB3R+N4HYGU5kx8HGSF4VkBWbJdNr5zMz1XEbpiXwpxmaQNX5WPG23Vb3mp13+7NHt0lTEn2vY85xs6oeUNY2pFr5SP2eVnlzic6DTPVkL3+9rXx54scU4S3Qki6DwEiil+7KKHjstD0XZuB4RMt9DGVN4VPbZbcAiPrBY0HE78qKNt4p8IMtzBmAjuEuW1J68FTLpwHjRmxOhnUDWXR9ss1M8HjACcHFAB/iCjmidSFxblTKjFfd+hNYLEX8OBL2GA1WvyGO5zlRrSKJglk5Z3nCO6BVKIgBesBAz22LI1iAMasDDZQsKN7mg90yFjPGoAIPx9DHc8w8sYdiVRpRgby9jCSJksJyWZraJMrBLWWdwljLBZKpjVeVPW1QkRG7zJgYthYerzAKBQ9vZAQlPS8+gdwc/xSFCJMXJ/A+/dNDkLpt9M3Ua6GTqKpgWQTnUO5FoOO7FW0muWXD9pvknh9XoU9Vj4OnY0Wb60Vmbp9VcGblHhEl2o5IzZCg8hi6nMgG1KFUKg1teaWaisCpXDs6CSBxz7UNxBkIBI1PD2AhkRbltw4HpkkLPHxak0TkFLmY9ZSllWpNObSeMPAw2aIAM5FZSiS6MfEujYUXuZXSclahm/V8OSltJYgqYgpc2H+asc+ZO4LgjNAe6cENslj5U5CraKVXy6yjQ8Gdr6G80HxCoqC8RZCEofE9sQi0L/YOt6zgTKqYxgyvK1bAu4NhvG1J8JItDU9pEsxd0qoBqlnldgYcO2IOngSv79NYvG9M52GUJwgW8oQ88+SmlIajAEW+cvW1EOdFDUK8FIIIP6JEQfKLjUG5wVxG/sId/62UzkR/8MRrTZyEIMlZsaJQfl1xlrdaJleoFzea9aFGvf8T4YpUoIabtYOab5Tv+QE3Kx3VZH+g//flIVtcVjUVxFX3wI1+rzTPoIFGY2KeCW+1QtpUQZcm9dKI0oPnVhz4/1vdjNaZYjNB36NVWJrVjp65lkNDxD//4YfvYXnNn5TjBQrR8j476YTeQ6kNXbHCc7Of37kX+kRkVzcdtvrkyH5wy0v0NP5G4hy5tmOrYCldB23i0B24McUFHBXaISPGDqoYS+GpDtBFWhalPhZj7c9mLCkrsTiNINdj8WoOhKvy0vmCOPVUysv6EUrCm8BSJ28rE4XSBKZjUnwOGEMEnCvX4KuNgYLLoGDakXwPFMzHUlGC98cZxExdaRLOplPBmHIVkcKdi1QkJ2vKqQ5N0uXXUGtjTDkoXAY7e8kKo2gNraX9C1gtA9hno9wO/ts3b/I7dwB1Tttv75jErTl+PtbH5JT2e2zj80NsXixFMLhzA1oWm0fdOuyoeBlArGGWRsXlOpVYdIMsfRvZctWhNZZHx1a3D4LyQP2bhKG0h9ZHc78tinbD/tfiByS/wh/TegQMWxiEJ5Mhj8mBCBQMzyOA85ar5TBanP4lRA+rfmm7AH6H/M9rkMsQcmTUdxNKoObI9YDTvR/xenW8fG2mfv4AbdPrFuchMM/6fKPrM4e71ZEtzMFX4+lx3DmyJXQlY8MVXrO/K3J1an6N6etD3k06tAiuNNZclVRK70b9oIpFsy2uf5rBzouHi5MfPLjPn4AvHgGGU6PAe+CtadXMNz1+wUiCbwrkwLm+yVLjyUJgiWYmgQvSWr0cNkqC6VFkEbIULr4ZpEtjSnL/y2+jeS7j/wYAAP//9F32EA==" } From 9a98beeab5f80f62b3403380a64baa92dbad861a Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 2 Jul 2020 17:17:31 +0200 Subject: [PATCH 28/35] Cursor input and manager implementation (#19571) This change finally implements the input manager and actual input execution for stateful inputs. --- filebeat/input/v2/input-cursor/input.go | 120 +++++++++++++++++++++- filebeat/input/v2/input-cursor/manager.go | 112 +++++++++++++++++++- 2 files changed, 225 insertions(+), 7 deletions(-) diff --git a/filebeat/input/v2/input-cursor/input.go b/filebeat/input/v2/input-cursor/input.go index 94faeeaadda8..a768823d9f6c 100644 --- a/filebeat/input/v2/input-cursor/input.go +++ b/filebeat/input/v2/input-cursor/input.go @@ -18,11 +18,19 @@ package cursor import ( + "context" "fmt" + "runtime/debug" "time" + "github.com/urso/sderr" + + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/unison" + input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/logp" ) // Input interface for cursor based inputs. This interface must be implemented @@ -62,7 +70,29 @@ func (inp *managedInput) Name() string { return inp.input.Name() } // Test runs the Test method for each configured source. func (inp *managedInput) Test(ctx input.TestContext) error { - panic("TODO: implement me") + var grp unison.MultiErrGroup + for _, source := range inp.sources { + source := source + grp.Go(func() (err error) { + return inp.testSource(ctx, source) + }) + } + + errs := grp.Wait() + if len(errs) > 0 { + return sderr.WrapAll(errs, "input tests failed") + } + return nil +} + +func (inp *managedInput) testSource(ctx input.TestContext, source Source) (err error) { + defer func() { + if v := recover(); v != nil { + err = fmt.Errorf("input panic with: %+v\n%s", v, debug.Stack()) + ctx.Logger.Errorf("Input crashed with: %+v", err) + } + }() + return inp.input.Test(source, ctx) } // Run creates a go-routine per source, waiting until all go-routines have @@ -73,7 +103,68 @@ func (inp *managedInput) Run( ctx input.Context, pipeline beat.PipelineConnector, ) (err error) { - panic("TODO: implement me") + // Setup cancellation using a custom cancel context. All workers will be + // stopped if one failed badly by returning an error. + cancelCtx, cancel := context.WithCancel(ctxtool.FromCanceller(ctx.Cancelation)) + defer cancel() + ctx.Cancelation = cancelCtx + + var grp unison.MultiErrGroup + for _, source := range inp.sources { + source := source + grp.Go(func() (err error) { + // refine per worker context + inpCtx := ctx + inpCtx.ID = ctx.ID + "::" + source.Name() + inpCtx.Logger = ctx.Logger.With("source", source.Name()) + + if err = inp.runSource(inpCtx, inp.manager.store, source, pipeline); err != nil { + cancel() + } + return err + }) + } + + if errs := grp.Wait(); len(errs) > 0 { + return sderr.WrapAll(errs, "input %{id} failed", ctx.ID) + } + return nil +} + +func (inp *managedInput) runSource( + ctx input.Context, + store *store, + source Source, + pipeline beat.PipelineConnector, +) (err error) { + defer func() { + if v := recover(); v != nil { + err = fmt.Errorf("input panic with: %+v\n%s", v, debug.Stack()) + ctx.Logger.Errorf("Input crashed with: %+v", err) + } + }() + + client, err := pipeline.ConnectWith(beat.ClientConfig{ + CloseRef: ctx.Cancelation, + ACKEvents: newInputACKHandler(ctx.Logger), + }) + if err != nil { + return err + } + defer client.Close() + + resourceKey := inp.createSourceID(source) + resource, err := inp.manager.lock(ctx, resourceKey) + if err != nil { + return err + } + defer releaseResource(resource) + + store.UpdateTTL(resource, inp.cleanTimeout) + + cursor := makeCursor(store, resource) + publisher := &cursorPublisher{canceler: ctx.Cancelation, client: client, cursor: &cursor} + return inp.input.Run(ctx, source, cursor, publisher) } func (inp *managedInput) createSourceID(s Source) string { @@ -82,3 +173,28 @@ func (inp *managedInput) createSourceID(s Source) string { } return fmt.Sprintf("%v::%v", inp.manager.Type, s.Name()) } + +func newInputACKHandler(log *logp.Logger) func([]interface{}) { + return func(private []interface{}) { + var n uint + var last int + for i := 0; i < len(private); i++ { + current := private[i] + if current == nil { + continue + } + + if _, ok := current.(*updateOp); !ok { + continue + } + + n++ + last = i + } + + if n == 0 { + return + } + private[last].(*updateOp).Execute(n) + } +} diff --git a/filebeat/input/v2/input-cursor/manager.go b/filebeat/input/v2/input-cursor/manager.go index ee1b2bc7939b..2a4310dc778e 100644 --- a/filebeat/input/v2/input-cursor/manager.go +++ b/filebeat/input/v2/input-cursor/manager.go @@ -18,15 +18,19 @@ package cursor import ( + "errors" + "sync" "time" + "github.com/urso/sderr" + + "github.com/elastic/go-concert/unison" + input "github.com/elastic/beats/v7/filebeat/input/v2" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore" - - "github.com/elastic/go-concert/unison" ) // InputManager is used to create, manage, and coordinate stateful inputs and @@ -60,7 +64,9 @@ type InputManager struct { // that will be used to collect events from each source. Configure func(cfg *common.Config) ([]Source, Input, error) - store *store + initOnce sync.Once + initErr error + store *store } // Source describe a source the input can collect data from. @@ -70,22 +76,118 @@ type Source interface { Name() string } +var errNoSourceConfigured = errors.New("no source has been configured") +var errNoInputRunner = errors.New("no input runner available") + // StateStore interface and configurations used to give the Manager access to the persistent store. type StateStore interface { Access() (*statestore.Store, error) CleanupInterval() time.Duration } +func (cim *InputManager) init() error { + cim.initOnce.Do(func() { + if cim.DefaultCleanTimeout <= 0 { + cim.DefaultCleanTimeout = 30 * time.Minute + } + + log := cim.Logger.With("input_type", cim.Type) + var store *store + store, cim.initErr = openStore(log, cim.StateStore, cim.Type) + if cim.initErr != nil { + return + } + + cim.store = store + }) + + return cim.initErr +} + // Init starts background processes for deleting old entries from the // persistent store if mode is ModeRun. func (cim *InputManager) Init(group unison.Group, mode v2.Mode) error { - panic("TODO: implement me") + if mode != v2.ModeRun { + return nil + } + + if err := cim.init(); err != nil { + return err + } + + log := cim.Logger.With("input_type", cim.Type) + + store := cim.store + cleaner := &cleaner{log: log} + store.Retain() + err := group.Go(func(canceler unison.Canceler) error { + defer cim.shutdown() + defer store.Release() + interval := cim.StateStore.CleanupInterval() + if interval <= 0 { + interval = 5 * time.Minute + } + cleaner.run(canceler, store, interval) + return nil + }) + if err != nil { + store.Release() + cim.shutdown() + return sderr.Wrap(err, "Can not start registry cleanup process") + } + + return nil +} + +func (cim *InputManager) shutdown() { + cim.store.Release() } // Create builds a new v2.Input using the provided Configure function. // The Input will run a go-routine per source that has been configured. func (cim *InputManager) Create(config *common.Config) (input.Input, error) { - panic("TODO: implement me") + if err := cim.init(); err != nil { + return nil, err + } + + settings := struct { + ID string `config:"id"` + CleanTimeout time.Duration `config:"clean_timeout"` + }{ID: "", CleanTimeout: cim.DefaultCleanTimeout} + if err := config.Unpack(&settings); err != nil { + return nil, err + } + + sources, inp, err := cim.Configure(config) + if err != nil { + return nil, err + } + if len(sources) == 0 { + return nil, errNoSourceConfigured + } + if inp == nil { + return nil, errNoInputRunner + } + + return &managedInput{ + manager: cim, + userID: settings.ID, + sources: sources, + input: inp, + cleanTimeout: settings.CleanTimeout, + }, nil +} + +// Lock locks a key for exclusive access and returns an resource that can be used to modify +// the cursor state and unlock the key. +func (cim *InputManager) lock(ctx input.Context, key string) (*resource, error) { + resource := cim.store.Get(key) + err := lockResource(ctx.Logger, resource, ctx.Cancelation) + if err != nil { + resource.Release() + return nil, err + } + return resource, nil } func lockResource(log *logp.Logger, resource *resource, canceler input.Canceler) error { From 36e2978945a95f2adc75d321412d31926a4f4ce3 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 2 Jul 2020 17:21:20 +0200 Subject: [PATCH 29/35] Prepare input/file for changes in the registrar (#19516) Planned changes in the registrar will introduce a key-value store. This changes prepares the input/file package for upcoming updates. --- filebeat/input/file/state.go | 20 ++++++++++---------- filebeat/input/file/states.go | 12 +++++++++++- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/filebeat/input/file/state.go b/filebeat/input/file/state.go index dde3c6c54217..ef255243b4c1 100644 --- a/filebeat/input/file/state.go +++ b/filebeat/input/file/state.go @@ -30,16 +30,16 @@ import ( // State is used to communicate the reading state of a file type State struct { - Id string `json:"-"` // local unique id to make comparison more efficient - Finished bool `json:"-"` // harvester state - Fileinfo os.FileInfo `json:"-"` // the file info - Source string `json:"source"` - Offset int64 `json:"offset"` - Timestamp time.Time `json:"timestamp"` - TTL time.Duration `json:"ttl"` - Type string `json:"type"` - Meta map[string]string `json:"meta"` - FileStateOS file.StateOS + Id string `json:"-" struct:"-"` // local unique id to make comparison more efficient + Finished bool `json:"-" struct:"-"` // harvester state + Fileinfo os.FileInfo `json:"-" struct:"-"` // the file info + Source string `json:"source" struct:"source"` + Offset int64 `json:"offset" struct:"offset"` + Timestamp time.Time `json:"timestamp" struct:"timestamp"` + TTL time.Duration `json:"ttl" struct:"ttl"` + Type string `json:"type" struct:"type"` + Meta map[string]string `json:"meta" struct:"meta,omitempty"` + FileStateOS file.StateOS `json:"FileStateOS" struct:"FileStateOS"` } // NewState creates a new file state diff --git a/filebeat/input/file/states.go b/filebeat/input/file/states.go index fc50dd904c0b..34704b41dba1 100644 --- a/filebeat/input/file/states.go +++ b/filebeat/input/file/states.go @@ -94,6 +94,12 @@ func (s *States) findPrevious(id string) int { // The number of states that were cleaned up and number of states that can be // cleaned up in the future is returned. func (s *States) Cleanup() (int, int) { + return s.CleanupWith(nil) +} + +// CleanupWith cleans up the state array. It calls `fn` with the state ID, for +// each entry to be removed. +func (s *States) CleanupWith(fn func(string)) (int, int) { s.Lock() defer s.Unlock() @@ -114,7 +120,11 @@ func (s *States) Cleanup() (int, int) { continue } - delete(s.idx, state.ID()) + id := state.ID() + delete(s.idx, id) + if fn != nil { + fn(id) + } logp.Debug("state", "State removed for %v because of older: %v", state.Source, state.TTL) L-- From 58edbb4e56c17e58995ec76071ef313d2aaad83d Mon Sep 17 00:00:00 2001 From: Vijay Samuel Date: Thu, 2 Jul 2020 08:29:34 -0700 Subject: [PATCH 30/35] Fix ordering and duplicate configs on autodiscover (#19317) #18979 introduced a pod level event which is generated after all container events. The ordering is wrong in that pod events are sent last which would generate a valid event similar to container events. The ordering needs to be pod first and container events next so that pod events dont override valid container events. One other issue was that the pod level hint generates a single config with all hosts and it wont get over written by container hints causing more than one config to be spun up for the same hint (one with a container meta and one without). --- .../autodiscover/providers/kubernetes/pod.go | 19 +- .../providers/kubernetes/pod_test.go | 521 +++++++++++++++--- .../autodiscover/builder/hints/metrics.go | 39 +- .../builder/hints/metrics_test.go | 62 ++- 4 files changed, 538 insertions(+), 103 deletions(-) diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index 649c372bfd63..30ed913060ba 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -284,6 +284,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet var ( annotations = common.MapStr{} nsAnn = common.MapStr{} + events = make([]bus.Event, 0) ) for k, v := range pod.GetObjectMeta().GetAnnotations() { safemapstr.Put(annotations, k, v) @@ -299,7 +300,6 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet } } - emitted := 0 // Emit container and port information for _, c := range containers { // If it doesn't have an ID, container doesn't exist in @@ -345,8 +345,7 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet "kubernetes": meta, }, } - p.publish(event) - emitted++ + events = append(events, event) } for _, port := range c.Ports { @@ -361,16 +360,16 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet "kubernetes": meta, }, } - p.publish(event) - emitted++ + events = append(events, event) } } - // Finally publish a pod level event so that hints that have no exposed ports can get processed. + // Publish a pod level event so that hints that have no exposed ports can get processed. // Log hints would just ignore this event as there is no ${data.container.id} - // Publish the pod level hint only if atleast one container level hint was emitted. This ensures that there is + // Publish the pod level hint only if at least one container level hint was generated. This ensures that there is // no unnecessary pod level events emitted prematurely. - if emitted != 0 { + // We publish the pod level hint first so that it doesn't override a valid container level event. + if len(events) != 0 { meta := p.metagen.Generate(pod) // Information that can be used in discovering a workload @@ -392,6 +391,10 @@ func (p *pod) emitEvents(pod *kubernetes.Pod, flag string, containers []kubernet }, } p.publish(event) + } + // Publish the container level hints finally. + for _, event := range events { + p.publish(event) } } diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 7e5d51b23b06..2dacee2b9fc6 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -353,7 +353,7 @@ func TestEmitEvent(t *testing.T) { Message string Flag string Pod *kubernetes.Pod - Expected bus.Event + Expected []bus.Event }{ { Message: "Test common pod start", @@ -389,44 +389,227 @@ func TestEmitEvent(t *testing.T) { }, }, }, - Expected: bus.Event{ - "start": true, - "host": "127.0.0.1", - "port": 0, - "id": cid, - "provider": UUID, - "kubernetes": common.MapStr{ - "container": common.MapStr{ - "id": "foobar", - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - "runtime": "docker", + Expected: []bus.Event{ + { + "start": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "pod": common.MapStr{ - "name": "filebeat", - "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, - "node": common.MapStr{ - "name": "node", + "config": []*common.Config{}, + }, + { + "start": true, + "host": "127.0.0.1", + "port": 0, + "id": cid, + "provider": UUID, + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "id": "foobar", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "docker", + }, + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "namespace": "default", - "annotations": common.MapStr{}, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + }, + }, + { + Message: "Test common pod start with multiple ports exposed", + Flag: "start", + Pod: &kubernetes.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, }, - "meta": common.MapStr{ + TypeMeta: typeMeta, + Status: v1.PodStatus{ + PodIP: podIP, + ContainerStatuses: []kubernetes.PodContainerStatus{ + { + Name: name, + ContainerID: containerID, + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{}, + }, + }, + }, + }, + Spec: v1.PodSpec{ + NodeName: node, + Containers: []kubernetes.Container{ + { + Image: containerImage, + Name: name, + Ports: []v1.ContainerPort{ + { + ContainerPort: 8080, + }, + { + ContainerPort: 9090, + }, + }, + }, + }, + }, + }, + Expected: []bus.Event{ + { + "start": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "start": true, + "host": "127.0.0.1", + "port": int32(8080), + "id": cid, + "provider": UUID, + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "id": "foobar", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "docker", + }, + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "start": true, + "host": "127.0.0.1", + "port": int32(9090), + "id": cid, + "provider": UUID, "kubernetes": common.MapStr{ - "namespace": "default", "container": common.MapStr{ - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - }, "pod": common.MapStr{ + "id": "foobar", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "docker", + }, + "pod": common.MapStr{ "name": "filebeat", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - }, "node": common.MapStr{ + }, + "node": common.MapStr{ "name": "node", }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, + "config": []*common.Config{}, }, - "config": []*common.Config{}, }, }, { @@ -522,44 +705,75 @@ func TestEmitEvent(t *testing.T) { }, }, }, - Expected: bus.Event{ - "stop": true, - "host": "", - "id": cid, - "port": 0, - "provider": UUID, - "kubernetes": common.MapStr{ - "container": common.MapStr{ - "id": "", - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - "runtime": "", - }, - "pod": common.MapStr{ - "name": "filebeat", - "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + Expected: []bus.Event{ + { + "stop": true, + "host": "", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "node": common.MapStr{ - "name": "node", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, - "namespace": "default", - "annotations": common.MapStr{}, + "config": []*common.Config{}, }, - "meta": common.MapStr{ + { + "stop": true, + "host": "", + "id": cid, + "port": 0, + "provider": UUID, "kubernetes": common.MapStr{ - "namespace": "default", "container": common.MapStr{ - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - }, "pod": common.MapStr{ + "id": "", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "", + }, + "pod": common.MapStr{ "name": "filebeat", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - }, "node": common.MapStr{ + }, + "node": common.MapStr{ "name": "node", }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, + "config": []*common.Config{}, }, - "config": []*common.Config{}, }, }, { @@ -592,44 +806,176 @@ func TestEmitEvent(t *testing.T) { }, }, }, - Expected: bus.Event{ - "stop": true, - "host": "127.0.0.1", - "port": 0, - "id": cid, - "provider": UUID, - "kubernetes": common.MapStr{ - "container": common.MapStr{ - "id": "", - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - "runtime": "", + Expected: []bus.Event{ + { + "stop": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "pod": common.MapStr{ - "name": "filebeat", - "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "stop": true, + "host": "127.0.0.1", + "port": 0, + "id": cid, + "provider": UUID, + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "id": "", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "", + }, + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, }, - "node": common.MapStr{ - "name": "node", + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, }, - "namespace": "default", - "annotations": common.MapStr{}, + "config": []*common.Config{}, }, - "meta": common.MapStr{ + }, + }, + { + Message: "Test stop pod without container id", + Flag: "stop", + Pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + TypeMeta: typeMeta, + Status: v1.PodStatus{ + PodIP: podIP, + ContainerStatuses: []kubernetes.PodContainerStatus{ + { + Name: name, + }, + }, + }, + Spec: v1.PodSpec{ + NodeName: node, + Containers: []kubernetes.Container{ + { + Image: containerImage, + Name: name, + }, + }, + }, + }, + Expected: []bus.Event{ + { + "stop": true, + "host": "127.0.0.1", + "id": uid, + "provider": UUID, + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, + "node": common.MapStr{ + "name": "node", + }, + "namespace": "default", + "annotations": common.MapStr{}, + }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, + }, + { + "stop": true, + "host": "127.0.0.1", + "port": 0, + "id": cid, + "provider": UUID, "kubernetes": common.MapStr{ - "namespace": "default", "container": common.MapStr{ - "name": "filebeat", - "image": "elastic/filebeat:6.3.0", - }, "pod": common.MapStr{ + "id": "", + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + "runtime": "", + }, + "pod": common.MapStr{ "name": "filebeat", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - }, "node": common.MapStr{ + }, + "node": common.MapStr{ "name": "node", }, + "namespace": "default", + "annotations": common.MapStr{}, }, + "meta": common.MapStr{ + "kubernetes": common.MapStr{ + "namespace": "default", + "container": common.MapStr{ + "name": "filebeat", + "image": "elastic/filebeat:6.3.0", + }, "pod": common.MapStr{ + "name": "filebeat", + "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", + }, "node": common.MapStr{ + "name": "node", + }, + }, + }, + "config": []*common.Config{}, }, - "config": []*common.Config{}, }, }, } @@ -663,14 +1009,17 @@ func TestEmitEvent(t *testing.T) { pod.emit(test.Pod, test.Flag) - select { - case event := <-listener.Events(): - assert.Equal(t, test.Expected, event, test.Message) - case <-time.After(2 * time.Second): - if test.Expected != nil { - t.Fatal("Timeout while waiting for event") + for i := 0; i < len(test.Expected); i++ { + select { + case event := <-listener.Events(): + assert.Equal(t, test.Expected[i], event, test.Message) + case <-time.After(2 * time.Second): + if test.Expected != nil { + t.Fatal("Timeout while waiting for event") + } } } + }) } } diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 52eba34a1ff5..37ec6f150e0f 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -133,7 +133,6 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c moduleConfig := common.MapStr{ "module": mod, "metricsets": msets, - "hosts": hosts, "timeout": tout, "period": ival, "enabled": true, @@ -154,15 +153,30 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c moduleConfig["password"] = password } - logp.Debug("hints.builder", "generated config: %v", moduleConfig) + // If there are hosts that match, ensure that there is a module config for each valid host. + // We do this because every config that is from a Pod that has an exposed port will generate a valid + // module config. However, the pod level hint will generate a config with all hosts that are defined in the + // config. To make sure that these pod level configs get deduped, it is essential that we generate exactly one + // module config per host. + if len(hosts) != 0 { + for _, h := range hosts { + mod := moduleConfig.Clone() + mod["hosts"] = []string{h} + + logp.Debug("hints.builder", "generated config: %v", mod) + + // Create config object + cfg := m.generateConfig(mod) + configs = append(configs, cfg) + } + } else { + logp.Debug("hints.builder", "generated config: %v", moduleConfig) - // Create config object - cfg, err := common.NewConfigFrom(moduleConfig) - if err != nil { - logp.Debug("hints.builder", "config merge failed with error: %v", err) + // Create config object + cfg := m.generateConfig(moduleConfig) + configs = append(configs, cfg) } - logp.Debug("hints.builder", "generated config: %+v", common.DebugString(cfg, true)) - configs = append(configs, cfg) + } // Apply information in event to the template to generate the final config @@ -171,6 +185,15 @@ func (m *metricHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*c return template.ApplyConfigTemplate(event, configs, options...) } +func (m *metricHints) generateConfig(mod common.MapStr) *common.Config { + cfg, err := common.NewConfigFrom(mod) + if err != nil { + logp.Debug("hints.builder", "config merge failed with error: %v", err) + } + logp.Debug("hints.builder", "generated config: %+v", common.DebugString(cfg, true)) + return cfg +} + func (m *metricHints) getModule(hints common.MapStr) string { return builder.GetHintString(hints, m.Key, module) } diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index d2d0462a0476..4badd9c2a026 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -409,6 +409,66 @@ func TestGenerateHints(t *testing.T) { }, }, }, + { + message: "Module with multiple hosts returns the right number of hints. Pod level hints need to be one per host", + event: bus.Event{ + "host": "1.2.3.4", + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "mockmoduledefaults", + "namespace": "test", + "hosts": "${data.host}:9090, ${data.host}:9091", + }, + }, + }, + len: 2, + result: []common.MapStr{ + { + "module": "mockmoduledefaults", + "namespace": "test", + "metricsets": []string{"default"}, + "timeout": "3s", + "period": "1m", + "enabled": true, + "hosts": []interface{}{"1.2.3.4:9090"}, + }, + { + "module": "mockmoduledefaults", + "namespace": "test", + "metricsets": []string{"default"}, + "timeout": "3s", + "period": "1m", + "enabled": true, + "hosts": []interface{}{"1.2.3.4:9091"}, + }, + }, + }, + { + message: "Module with multiple hosts and an exposed port creates a config for just the exposed port", + event: bus.Event{ + "host": "1.2.3.4", + "port": 9091, + "hints": common.MapStr{ + "metrics": common.MapStr{ + "module": "mockmoduledefaults", + "namespace": "test", + "hosts": "${data.host}:9090, ${data.host}:9091", + }, + }, + }, + len: 1, + result: []common.MapStr{ + { + "module": "mockmoduledefaults", + "namespace": "test", + "metricsets": []string{"default"}, + "timeout": "3s", + "period": "1m", + "enabled": true, + "hosts": []interface{}{"1.2.3.4:9091"}, + }, + }, + }, } for _, test := range tests { mockRegister := mb.NewRegister() @@ -423,7 +483,7 @@ func TestGenerateHints(t *testing.T) { logger: logp.NewLogger("hints.builder"), } cfgs := m.CreateConfig(test.event) - assert.Equal(t, len(cfgs), test.len) + assert.Equal(t, len(cfgs), test.len, test.message) // The check below helps skipping config validation if there is no config supposed to be emitted. if len(cfgs) == 0 { From 4b5fca9d4150c42189fd87017fa9a0aa2177b610 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 2 Jul 2020 17:58:12 +0200 Subject: [PATCH 31/35] Update Jenkinsfile to not inspect removed vendor (#19610) --- Jenkinsfile | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6aec0cf79f69..47a29353e0b5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1236,50 +1236,50 @@ def loadConfigEnvVars(){ // Auditbeat depends on metricbeat as framework, but does not include any of // the modules from Metricbeat. // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. - env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat')) - env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat')) + env.BUILD_AUDITBEAT = isChangedOSSCode(getProjectDependencies('auditbeat')) + env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/auditbeat')) // Dockerlogbeat is a standalone Beat that only relies on libbeat. - env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat')) + env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/dockerlogbeat')) // Filebeat depends on libbeat only. // The Filebeat x-pack build contains all functionality from OSS Filebeat. - env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat')) - env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat')) + env.BUILD_FILEBEAT = isChangedOSSCode(getProjectDependencies('filebeat')) + env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/filebeat')) // Metricbeat depends on libbeat only. // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. - env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat')) - env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat')) + env.BUILD_METRICBEAT = isChangedOSSCode(getProjectDependencies('metricbeat')) + env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/metricbeat')) // Functionbeat is a standalone beat that depends on libbeat only. // Functionbeat is available as x-pack build only. - env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat')) + env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/functionbeat')) // Heartbeat depends on libbeat only. // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. - env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat')) - env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat')) + env.BUILD_HEARTBEAT = isChangedOSSCode(getProjectDependencies('heartbeat')) + env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/heartbeat')) // Journalbeat depends on libbeat only. // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. - env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat')) - env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat')) + env.BUILD_JOURNALBEAT = isChangedOSSCode(getProjectDependencies('journalbeat')) + env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/journalbeat')) // Packetbeat depends on libbeat only. // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. - env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat')) - env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat')) + env.BUILD_PACKETBEAT = isChangedOSSCode(getProjectDependencies('packetbeat')) + env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/packetbeat')) // Winlogbeat depends on libbeat only. // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. - env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat')) - env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat')) + env.BUILD_WINLOGBEAT = isChangedOSSCode(getProjectDependencies('winlogbeat')) + env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/winlogbeat')) // Elastic-agent is a self-contained product, that depends on libbeat only. // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. // The agent is available as x-pack build only. - env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/elastic-agent')) + env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/elastic-agent')) // The Kubernetes test use Filebeat and Metricbeat, but only need to be run // if the deployment scripts have been updated. No Beats specific testing is @@ -1287,8 +1287,8 @@ def loadConfigEnvVars(){ env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) def generatorPatterns = ['^generator/.*'] - generatorPatterns.addAll(getVendorPatterns('generator/common/beatgen')) - generatorPatterns.addAll(getVendorPatterns('metricbeat/beater')) + generatorPatterns.addAll(getProjectDependencies('generator/common/beatgen')) + generatorPatterns.addAll(getProjectDependencies('metricbeat/beater')) env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) // Skip all the stages for changes only related to the documentation @@ -1327,7 +1327,7 @@ def isDocChangedOnly(){ /** This method grab the dependencies of a Go module and transform them on regexp */ -def getVendorPatterns(beatName){ +def getProjectDependencies(beatName){ def os = goos() def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" def output = "" @@ -1337,9 +1337,10 @@ def getVendorPatterns(beatName){ "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", ]) { output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - go list -mod=mod -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\ - |awk '{print \$1"/.*"}'\ - |sed -e "s#github.com/elastic/beats/v7/##g" + go list -deps ./${beatName} \ + | grep 'elastic/beats' \ + | sed -e "s#github.com/elastic/beats/v7/##g" \ + | awk '{print "^" \$1 "/.*"}' """) } return output?.split('\n').collect{ item -> item as String } From 1e759ef45985c5cbb523dbee394b48e71cb2a25e Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 2 Jul 2020 21:55:41 +0200 Subject: [PATCH 32/35] Input v2 cursor testing (#19573) Add tests for the stateful input manager and inputs. --- .../input/v2/input-cursor/manager_test.go | 607 ++++++++++++++++++ 1 file changed, 607 insertions(+) create mode 100644 filebeat/input/v2/input-cursor/manager_test.go diff --git a/filebeat/input/v2/input-cursor/manager_test.go b/filebeat/input/v2/input-cursor/manager_test.go new file mode 100644 index 000000000000..8c531f48f84a --- /dev/null +++ b/filebeat/input/v2/input-cursor/manager_test.go @@ -0,0 +1,607 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cursor + +import ( + "context" + "errors" + "fmt" + "runtime" + "sort" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" + "github.com/elastic/beats/v7/libbeat/tests/resources" + "github.com/elastic/go-concert/unison" +) + +type fakeTestInput struct { + OnTest func(Source, input.TestContext) error + OnRun func(input.Context, Source, Cursor, Publisher) error +} + +type stringSource string + +func TestManager_Init(t *testing.T) { + // Integration style tests for the InputManager and the state garbage collector + + t.Run("stopping the taskgroup kills internal go-routines", func(t *testing.T) { + numRoutines := runtime.NumGoroutine() + + var grp unison.TaskGroup + store := createSampleStore(t, nil) + manager := &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: store, + Type: "test", + DefaultCleanTimeout: 10 * time.Millisecond, + } + + err := manager.Init(&grp, v2.ModeRun) + require.NoError(t, err) + + time.Sleep(200 * time.Millisecond) + grp.Stop() + + // wait for all go-routines to be gone + + for numRoutines < runtime.NumGoroutine() { + time.Sleep(1 * time.Millisecond) + } + }) + + t.Run("collect old entries after startup", func(t *testing.T) { + store := createSampleStore(t, map[string]state{ + "test::key": { + TTL: 1 * time.Millisecond, + Updated: time.Now().Add(-24 * time.Hour), + }, + }) + store.GCPeriod = 10 * time.Millisecond + + var grp unison.TaskGroup + defer grp.Stop() + manager := &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: store, + Type: "test", + DefaultCleanTimeout: 10 * time.Millisecond, + } + + err := manager.Init(&grp, v2.ModeRun) + require.NoError(t, err) + + for len(store.snapshot()) > 0 { + time.Sleep(1 * time.Millisecond) + } + }) +} + +func TestManager_Create(t *testing.T) { + t.Run("fail if no source is configured", func(t *testing.T) { + manager := constInput(t, nil, &fakeTestInput{}) + _, err := manager.Create(common.NewConfig()) + require.Error(t, err) + }) + + t.Run("fail if config error", func(t *testing.T) { + manager := failingManager(t, errors.New("oops")) + _, err := manager.Create(common.NewConfig()) + require.Error(t, err) + }) + + t.Run("fail if no input runner is returned", func(t *testing.T) { + manager := constInput(t, sourceList("test"), nil) + _, err := manager.Create(common.NewConfig()) + require.Error(t, err) + }) + + t.Run("configure ok", func(t *testing.T) { + manager := constInput(t, sourceList("test"), &fakeTestInput{}) + _, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + }) + + t.Run("configuring inputs with overlapping sources is allowed", func(t *testing.T) { + manager := simpleManagerWithConfigure(t, func(cfg *common.Config) ([]Source, Input, error) { + config := struct{ Sources []string }{} + err := cfg.Unpack(&config) + return sourceList(config.Sources...), &fakeTestInput{}, err + }) + + _, err := manager.Create(common.MustNewConfigFrom(map[string]interface{}{ + "sources": []string{"a"}, + })) + require.NoError(t, err) + + _, err = manager.Create(common.MustNewConfigFrom(map[string]interface{}{ + "sources": []string{"a"}, + })) + require.NoError(t, err) + }) +} + +func TestManager_InputsTest(t *testing.T) { + var mu sync.Mutex + var seen []string + + sources := sourceList("source1", "source2") + + t.Run("test is run for each source", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ v2.TestContext) error { + mu.Lock() + defer mu.Unlock() + seen = append(seen, source.Name()) + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + err = inp.Test(input.TestContext{}) + require.NoError(t, err) + + sort.Strings(seen) + require.Equal(t, []string{"source1", "source2"}, seen) + }) + + t.Run("cancel gets distributed to all source tests", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(_ Source, ctx v2.TestContext) error { + <-ctx.Cancelation.Done() + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.TODO()) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{Cancelation: ctx}) + }() + + cancel() + wg.Wait() + require.NoError(t, err) + }) + + t.Run("fail if test for one source fails", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + failing := Source(stringSource("source1")) + sources := []Source{failing, stringSource("source2")} + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ v2.TestContext) error { + if source == failing { + t.Log("return error") + return errors.New("oops") + } + t.Log("return ok") + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{}) + t.Logf("Test returned: %v", err) + }() + + wg.Wait() + require.Error(t, err) + }) + + t.Run("panic is captured", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sources, &fakeTestInput{ + OnTest: func(source Source, _ v2.TestContext) error { + panic("oops") + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Test(input.TestContext{Logger: logp.NewLogger("test")}) + t.Logf("Test returned: %v", err) + }() + + wg.Wait() + require.Error(t, err) + }) +} + +func TestManager_InputsRun(t *testing.T) { + // Integration style tests for the InputManager and Input.Run + + t.Run("input returned with error", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, _ Cursor, _ Publisher) error { + return errors.New("oops") + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + require.Error(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("panic is captured", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, _ Cursor, _ Publisher) error { + panic("oops") + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + require.Error(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("shutdown on signal", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + manager := constInput(t, sourceList("test"), &fakeTestInput{ + OnRun: func(ctx input.Context, _ Source, _ Cursor, _ Publisher) error { + <-ctx.Cancelation.Done() + return nil + }, + }) + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var clientCounters pubtest.ClientCounter + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, clientCounters.BuildConnector()) + }() + + cancel() + require.NoError(t, err) + require.Equal(t, 0, clientCounters.Active()) + }) + + t.Run("continue sending from last known position", func(t *testing.T) { + log := logp.NewLogger("test") + + type runConfig struct{ Max int } + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + manager := simpleManagerWithConfigure(t, func(cfg *common.Config) ([]Source, Input, error) { + config := runConfig{} + if err := cfg.Unpack(&config); err != nil { + return nil, nil, err + } + + inp := &fakeTestInput{ + OnRun: func(_ input.Context, _ Source, cursor Cursor, pub Publisher) error { + state := struct{ N int }{} + if !cursor.IsNew() { + if err := cursor.Unpack(&state); err != nil { + return fmt.Errorf("failed to unpack cursor: %w", err) + } + } + + for i := 0; i < config.Max; i++ { + event := beat.Event{Fields: common.MapStr{"n": state.N}} + state.N++ + pub.Publish(event, state) + } + return nil + }, + } + + return sourceList("test"), inp, nil + }) + + var ids []int + pipeline := pubtest.ConstClient(&pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { + id := event.Fields["n"].(int) + ids = append(ids, id) + }, + }) + + // create and run first instance + inp, err := manager.Create(common.MustNewConfigFrom(runConfig{Max: 3})) + require.NoError(t, err) + require.NoError(t, inp.Run(input.Context{ + Logger: log, + Cancelation: context.Background(), + }, pipeline)) + + // create and run second instance instance + inp, err = manager.Create(common.MustNewConfigFrom(runConfig{Max: 3})) + require.NoError(t, err) + inp.Run(input.Context{ + Logger: log, + Cancelation: context.Background(), + }, pipeline) + + // verify + assert.Equal(t, []int{0, 1, 2, 3, 4, 5}, ids) + }) + + t.Run("event ACK triggers execution of update operations", func(t *testing.T) { + defer resources.NewGoroutinesChecker().Check(t) + + store := createSampleStore(t, nil) + var wgSend sync.WaitGroup + wgSend.Add(1) + manager := constInput(t, sourceList("key"), &fakeTestInput{ + OnRun: func(ctx input.Context, _ Source, _ Cursor, pub Publisher) error { + defer wgSend.Done() + fields := common.MapStr{"hello": "world"} + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state1") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state2") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state3") + pub.Publish(beat.Event{Fields: fields}, nil) + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state4") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state5") + pub.Publish(beat.Event{Fields: fields}, "test-cursor-state6") + return nil + }, + }) + manager.StateStore = store + + inp, err := manager.Create(common.NewConfig()) + require.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // setup publishing pipeline and capture ACKer, so we can simulate progress in the Output + var acker func([]interface{}) + var activeEventPrivate []interface{} + + ackEvents := func(n int) { + data, rest := activeEventPrivate[:n], activeEventPrivate[n:] + activeEventPrivate = rest + acker(data) + } + + var wgACKer sync.WaitGroup + wgACKer.Add(1) + pipeline := &pubtest.FakeConnector{ + ConnectFunc: func(cfg beat.ClientConfig) (beat.Client, error) { + defer wgACKer.Done() + acker = cfg.ACKEvents + return &pubtest.FakeClient{ + PublishFunc: func(event beat.Event) { + activeEventPrivate = append(activeEventPrivate, event.Private) + }, + }, nil + }, + } + + // start the input + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err = inp.Run(v2.Context{ + Logger: manager.Logger, + Cancelation: cancelCtx, + }, pipeline) + }() + // wait for test setup to shutdown + defer wg.Wait() + + // wait for setup complete and events being send (pending operations in the pipeline) + wgACKer.Wait() + wgSend.Wait() + + // 1. No cursor state in store yet, all operations are still pending + require.Equal(t, nil, store.snapshot()["test::key"].Cursor) + + // ACK first 2 events and check snapshot state + ackEvents(2) + require.Equal(t, "test-cursor-state2", store.snapshot()["test::key"].Cursor) + + // ACK 1 events and check snapshot state (3 events published) + ackEvents(1) + require.Equal(t, "test-cursor-state3", store.snapshot()["test::key"].Cursor) + + // ACK event without cursor update and check snapshot state not modified + ackEvents(1) + require.Equal(t, "test-cursor-state3", store.snapshot()["test::key"].Cursor) + + // ACK rest + ackEvents(3) + require.Equal(t, "test-cursor-state6", store.snapshot()["test::key"].Cursor) + }) +} + +func TestLockResource(t *testing.T) { + t.Run("can lock unused resource", func(t *testing.T) { + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + res := store.Get("test::key") + err := lockResource(logp.NewLogger("test"), res, context.TODO()) + require.NoError(t, err) + }) + + t.Run("fail to lock resource in use when context is cancelled", func(t *testing.T) { + log := logp.NewLogger("test") + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + resUsed := store.Get("test::key") + err := lockResource(log, resUsed, context.TODO()) + require.NoError(t, err) + + // fail to lock resource in use + ctx, cancel := context.WithCancel(context.TODO()) + cancel() + resFail := store.Get("test::key") + err = lockResource(log, resFail, ctx) + require.Error(t, err) + resFail.Release() + + // unlock and release resource in use -> it should be marked finished now + releaseResource(resUsed) + require.True(t, resUsed.Finished()) + }) + + t.Run("succeed to lock resource after it has been released", func(t *testing.T) { + log := logp.NewLogger("test") + + store := testOpenStore(t, "test", createSampleStore(t, nil)) + defer store.Release() + + resUsed := store.Get("test::key") + err := lockResource(log, resUsed, context.TODO()) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + resOther := store.Get("test::key") + err := lockResource(log, resOther, context.TODO()) + if err == nil { + releaseResource(resOther) + } + }() + + go func() { + time.Sleep(100 * time.Millisecond) + releaseResource(resUsed) + }() + + wg.Wait() // <- block forever if waiting go-routine can not acquire lock + }) +} + +func (s stringSource) Name() string { return string(s) } + +func simpleManagerWithConfigure(t *testing.T, configure func(*common.Config) ([]Source, Input, error)) *InputManager { + return &InputManager{ + Logger: logp.NewLogger("test"), + StateStore: createSampleStore(t, nil), + Type: "test", + Configure: configure, + } +} + +func constConfigureResult(t *testing.T, sources []Source, inp Input, err error) *InputManager { + return simpleManagerWithConfigure(t, func(cfg *common.Config) ([]Source, Input, error) { + return sources, inp, err + }) +} + +func failingManager(t *testing.T, err error) *InputManager { + return constConfigureResult(t, nil, nil, err) +} + +func constInput(t *testing.T, sources []Source, inp Input) *InputManager { + return constConfigureResult(t, sources, inp, nil) +} + +func (f *fakeTestInput) Name() string { return "test" } + +func (f *fakeTestInput) Test(source Source, ctx input.TestContext) error { + if f.OnTest != nil { + return f.OnTest(source, ctx) + } + return nil +} + +func (f *fakeTestInput) Run(ctx input.Context, source Source, cursor Cursor, pub Publisher) error { + if f.OnRun != nil { + return f.OnRun(ctx, source, cursor, pub) + } + return nil +} + +func sourceList(names ...string) []Source { + tmp := make([]Source, len(names)) + for i, name := range names { + tmp[i] = stringSource(name) + } + return tmp +} From 3c9219ff7eca6bcfc8d8d6201125eb626014457f Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 3 Jul 2020 09:59:14 +0200 Subject: [PATCH 33/35] ci: user fixed type of agent (#19625) --- .ci/build-docker-images.groovy | 2 +- .ci/jobs/build-it-docker-images.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/build-docker-images.groovy b/.ci/build-docker-images.groovy index f093acba52cf..417e2fefae4a 100644 --- a/.ci/build-docker-images.groovy +++ b/.ci/build-docker-images.groovy @@ -3,7 +3,7 @@ @Library('apm@current') _ pipeline { - agent { label 'linux && immutable' } + agent { label 'ubuntu-16 && immutable' } environment { REPO = 'beats' BASE_DIR = "src/github.com/elastic/${env.REPO}" diff --git a/.ci/jobs/build-it-docker-images.yml b/.ci/jobs/build-it-docker-images.yml index ac51519e1838..79a14779e6e8 100644 --- a/.ci/jobs/build-it-docker-images.yml +++ b/.ci/jobs/build-it-docker-images.yml @@ -15,6 +15,7 @@ scm: - git: url: git@github.com:elastic/beats.git + refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* wipe-workspace: true name: origin shallow-clone: true From 9c0b63d582e38dc43ad8d56ba1bd059a34f8aa37 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 3 Jul 2020 13:28:09 +0200 Subject: [PATCH 34/35] Cache error responses for cloudfoundry apps metadata (#19181) Cache error responses when requesting Cloud Foundry apps metadata to avoid hitting continuously the API when there are missing applications. --- CHANGELOG.next.asciidoc | 1 + x-pack/libbeat/common/cloudfoundry/cache.go | 44 +++++++---- .../cloudfoundry/cache_integration_test.go | 77 +++++++++++++++++++ .../libbeat/common/cloudfoundry/cache_test.go | 34 +++++--- x-pack/libbeat/common/cloudfoundry/config.go | 6 +- x-pack/libbeat/common/cloudfoundry/hub.go | 2 +- .../add_cloudfoundry_metadata.go | 2 +- .../add_cloudfoundry_metadata_test.go | 3 +- .../docs/add_cloudfoundry_metadata.asciidoc | 4 +- 9 files changed, 142 insertions(+), 31 deletions(-) create mode 100644 x-pack/libbeat/common/cloudfoundry/cache_integration_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4726420da201..a49e352f600b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -316,6 +316,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Change ownership of files in docker images so they can be used in secured environments. {pull}12905[12905] - Upgrade k8s.io/client-go and k8s keystore tests. {pull}18817[18817] - Add support for multiple sets of hints on autodiscover {pull}18883[18883] +- Add a configurable delay between retries when an app metadata cannot be retrieved by `add_cloudfoundry_metadata`. {pull}19181[19181] *Auditbeat* diff --git a/x-pack/libbeat/common/cloudfoundry/cache.go b/x-pack/libbeat/common/cloudfoundry/cache.go index bc87fb118369..22f41f3b23cd 100644 --- a/x-pack/libbeat/common/cloudfoundry/cache.go +++ b/x-pack/libbeat/common/cloudfoundry/cache.go @@ -22,43 +22,57 @@ type cfClient interface { // clientCacheWrap wraps the cloudfoundry client to add a cache in front of GetAppByGuid. type clientCacheWrap struct { - cache *common.Cache - client cfClient - log *logp.Logger + cache *common.Cache + client cfClient + log *logp.Logger + errorTTL time.Duration } // newClientCacheWrap creates a new cache for application data. -func newClientCacheWrap(client cfClient, ttl time.Duration, log *logp.Logger) *clientCacheWrap { +func newClientCacheWrap(client cfClient, ttl time.Duration, errorTTL time.Duration, log *logp.Logger) *clientCacheWrap { return &clientCacheWrap{ - cache: common.NewCacheWithExpireOnAdd(ttl, 100), - client: client, - log: log, + cache: common.NewCacheWithExpireOnAdd(ttl, 100), + client: client, + errorTTL: errorTTL, + log: log, } } +type appResponse struct { + app *cfclient.App + err error +} + // fetchApp uses the cfClient to retrieve an App entity and // stores it in the internal cache func (c *clientCacheWrap) fetchAppByGuid(guid string) (*cfclient.App, error) { app, err := c.client.GetAppByGuid(guid) + resp := appResponse{ + app: &app, + err: err, + } + timeout := time.Duration(0) if err != nil { - return nil, err + // Cache nil, because is what we want to return when there was an error + resp.app = nil + timeout = c.errorTTL } - c.cache.Put(app.Guid, &app) - return &app, nil + c.cache.PutWithTimeout(guid, &resp, timeout) + return resp.app, resp.err } // GetApp returns CF Application info, either from the cache or // using the CF client. func (c *clientCacheWrap) GetAppByGuid(guid string) (*cfclient.App, error) { - cachedApp := c.cache.Get(guid) - if cachedApp == nil { + cachedResp := c.cache.Get(guid) + if cachedResp == nil { return c.fetchAppByGuid(guid) } - app, ok := cachedApp.(*cfclient.App) + resp, ok := cachedResp.(*appResponse) if !ok { - return nil, fmt.Errorf("error converting cached app") + return nil, fmt.Errorf("error converting cached app response (of type %T), this is likely a bug", cachedResp) } - return app, nil + return resp.app, resp.err } // StartJanitor starts a goroutine that will periodically clean the applications cache. diff --git a/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go b/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go new file mode 100644 index 000000000000..f6af11787c9e --- /dev/null +++ b/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build cloudfoundry + +package cloudfoundry + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudfoundry-community/go-cfclient" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test" +) + +func TestGetApps(t *testing.T) { + var conf Config + err := common.MustNewConfigFrom(cftest.GetConfigFromEnv(t)).Unpack(&conf) + require.NoError(t, err) + + log := logp.NewLogger("cloudfoundry") + hub := NewHub(&conf, "filebeat", log) + + client, err := hub.Client() + require.NoError(t, err) + apps, err := client.(*clientCacheWrap).client.(*cfclient.Client).ListApps() + require.NoError(t, err) + + t.Logf("%d applications available", len(apps)) + + t.Run("request one of the available applications", func(t *testing.T) { + if len(apps) == 0 { + t.Skip("no apps in account?") + } + client, err := hub.Client() + require.NoError(t, err) + + guid := apps[0].Guid + app, err := client.GetAppByGuid(guid) + assert.Equal(t, guid, app.Guid) + assert.NoError(t, err) + }) + + t.Run("handle error when application is not available", func(t *testing.T) { + client, err := hub.Client() + require.NoError(t, err) + + testNotExists := func(t *testing.T) { + app, err := client.GetAppByGuid("notexists") + assert.Nil(t, app) + assert.True(t, cfclient.IsAppNotFoundError(err)) + } + + var firstTimeDuration time.Duration + t.Run("first call", func(t *testing.T) { + startTime := time.Now() + testNotExists(t) + firstTimeDuration = time.Now().Sub(startTime) + }) + + t.Run("second call, in cache, faster, same response", func(t *testing.T) { + for i := 0; i < 10; i++ { + startTime := time.Now() + testNotExists(t) + require.True(t, firstTimeDuration > time.Now().Sub(startTime)) + } + }) + }) +} diff --git a/x-pack/libbeat/common/cloudfoundry/cache_test.go b/x-pack/libbeat/common/cloudfoundry/cache_test.go index 678dd74ecfed..9e18a5ac86e6 100644 --- a/x-pack/libbeat/common/cloudfoundry/cache_test.go +++ b/x-pack/libbeat/common/cloudfoundry/cache_test.go @@ -7,7 +7,6 @@ package cloudfoundry import ( - "fmt" "testing" "time" @@ -26,17 +25,27 @@ func TestClientCacheWrap(t *testing.T) { Memory: 1, // use this field to track if from cache or from client } fakeClient := &fakeCFClient{app, 0} - cache := newClientCacheWrap(fakeClient, ttl, logp.NewLogger("cloudfoundry")) + cache := newClientCacheWrap(fakeClient, ttl, ttl, logp.NewLogger("cloudfoundry")) + + missingAppGuid := mustCreateFakeGuid() // should err; different app client doesn't have - _, err := cache.GetAppByGuid(mustCreateFakeGuid()) - assert.Error(t, err) + one, err := cache.GetAppByGuid(missingAppGuid) + assert.Nil(t, one) + assert.True(t, cfclient.IsAppNotFoundError(err)) + assert.Equal(t, 1, fakeClient.callCount) + + // calling again; the miss should be cached + one, err = cache.GetAppByGuid(missingAppGuid) + assert.Nil(t, one) + assert.True(t, cfclient.IsAppNotFoundError(err)) + assert.Equal(t, 1, fakeClient.callCount) // fetched from client for the first time - one, err := cache.GetAppByGuid(guid) + one, err = cache.GetAppByGuid(guid) assert.NoError(t, err) assert.Equal(t, app, *one) - assert.Equal(t, 1, fakeClient.callCount) + assert.Equal(t, 2, fakeClient.callCount) // updated app in fake client, new fetch should not have updated app updatedApp := cfclient.App{ @@ -47,14 +56,14 @@ func TestClientCacheWrap(t *testing.T) { two, err := cache.GetAppByGuid(guid) assert.NoError(t, err) assert.Equal(t, app, *two) - assert.Equal(t, 1, fakeClient.callCount) + assert.Equal(t, 2, fakeClient.callCount) // wait the ttl, then it should have updated app time.Sleep(ttl) three, err := cache.GetAppByGuid(guid) assert.NoError(t, err) assert.Equal(t, updatedApp, *three) - assert.Equal(t, 2, fakeClient.callCount) + assert.Equal(t, 3, fakeClient.callCount) } type fakeCFClient struct { @@ -63,10 +72,10 @@ type fakeCFClient struct { } func (f *fakeCFClient) GetAppByGuid(guid string) (cfclient.App, error) { + f.callCount++ if f.app.Guid != guid { - return f.app, fmt.Errorf("no app with guid") + return cfclient.App{}, notFoundError() } - f.callCount++ return f.app, nil } @@ -77,3 +86,8 @@ func mustCreateFakeGuid() string { } return uuid.String() } + +// notFoundError returns a cloud foundry error that satisfies cfclient.IsAppNotFoundError(err) +func notFoundError() error { + return cfclient.CloudFoundryError{Code: 100004} +} diff --git a/x-pack/libbeat/common/cloudfoundry/config.go b/x-pack/libbeat/common/cloudfoundry/config.go index 8f1139bd7a22..0724bdc66e1f 100644 --- a/x-pack/libbeat/common/cloudfoundry/config.go +++ b/x-pack/libbeat/common/cloudfoundry/config.go @@ -41,8 +41,11 @@ type Config struct { // multiple filebeats will shard the load of receiving and sending events. ShardID string `config:"shard_id"` - // Maximum amount of time to cache application objects from CF client + // Maximum amount of time to cache application objects from CF client. CacheDuration time.Duration `config:"cache_duration"` + + // Time to wait before retrying to get application info in case of error. + CacheRetryDelay time.Duration `config:"cache_retry_delay"` } // InitDefaults initialize the defaults for the configuration. @@ -55,6 +58,7 @@ func (c *Config) InitDefaults() { } c.ShardID = uuid.String() c.CacheDuration = 120 * time.Second + c.CacheRetryDelay = 20 * time.Second c.Version = ConsumerVersionV1 } diff --git a/x-pack/libbeat/common/cloudfoundry/hub.go b/x-pack/libbeat/common/cloudfoundry/hub.go index 823087ea9591..4bb7fce1eec2 100644 --- a/x-pack/libbeat/common/cloudfoundry/hub.go +++ b/x-pack/libbeat/common/cloudfoundry/hub.go @@ -67,7 +67,7 @@ func (h *Hub) Client() (Client, error) { if h.cfg.UaaAddress != "" { cf.Endpoint.AuthEndpoint = h.cfg.UaaAddress } - return newClientCacheWrap(cf, h.cfg.CacheDuration, h.log), nil + return newClientCacheWrap(cf, h.cfg.CacheDuration, h.cfg.CacheRetryDelay, h.log), nil } // RlpListener returns a listener client that calls the passed callback when the provided events are streamed through diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go index a80cc24700c4..d18a04ca9790 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go @@ -60,7 +60,7 @@ func (d *addCloudFoundryMetadata) Run(event *beat.Event) (*beat.Event, error) { } valI, err := event.GetValue("cloudfoundry.app.id") if err != nil { - // doesn't have the required cf.app.id value to add more information + // doesn't have the required cloudfoundry.app.id value to add more information return event, nil } val, _ := valI.(string) diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go index ca83a3e24c22..1aff4cb2df87 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go @@ -5,7 +5,6 @@ package add_cloudfoundry_metadata import ( - "fmt" "testing" "time" @@ -148,7 +147,7 @@ type fakeClient struct { func (c *fakeClient) GetAppByGuid(guid string) (*cfclient.App, error) { if c.app.Guid != guid { - return nil, fmt.Errorf("unknown app") + return nil, cfclient.CloudFoundryError{Code: 100004} } return &c.app, nil } diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc index 7c5b2daba962..558b5a1031bf 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc @@ -53,6 +53,8 @@ It has the following settings: `client_secret`:: Client Secret to authenticate with Cloud Foundry. -`cache_duration`:: (Optional) Maximum amount of time to cache an application's metadata. +`cache_duration`:: (Optional) Maximum amount of time to cache an application's metadata. Defaults to 120 seconds. + +`cache_retry_delay`:: (Optional) Time to wait before trying to obtain an application's metadata again in case of error. Defaults to 20 seconds. `ssl`:: (Optional) SSL configuration to use when connecting to Cloud Foundry. From 7e53933b4593c2b7653df9f8a4636dcca2ad9b7b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 3 Jul 2020 14:38:53 +0100 Subject: [PATCH 35/35] [ci] fix env variable name for xpack filebeats (#19617) --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 47a29353e0b5..e0f12058de46 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -193,7 +193,7 @@ pipeline { when { beforeAgent true expression { - return env.BUILD_XPACK_FILEBEAT != "false" && params.macosTest + return env.BUILD_FILEBEAT_XPACK != "false" && params.macosTest } } steps {