From e0f7e0e142d3e0258040194e6be753ad5f7c101a Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 27 Oct 2020 14:59:53 -0600 Subject: [PATCH 1/5] Rename google-pubsub Filebeat input to gcp-pubsub --- .../filebeat.inputs.reference.xpack.yml.tmpl | 2 +- ...e-pubsub.asciidoc => input-gcp-pubsub.asciidoc} | 12 ++++++------ x-pack/filebeat/filebeat.reference.yml | 2 +- x-pack/filebeat/include/list.go | 2 +- .../{googlepubsub => gcppubsub}/_meta/Dockerfile | 0 .../_meta/supported-versions.yml | 0 .../input/{googlepubsub => gcppubsub}/config.go | 2 +- .../{googlepubsub => gcppubsub}/config_test.go | 2 +- .../{googlepubsub => gcppubsub}/docker-compose.yml | 0 .../input/{googlepubsub => gcppubsub}/input.go | 14 ++++++++++---- .../{googlepubsub => gcppubsub}/pubsub_test.go | 2 +- .../{googlepubsub => gcppubsub}/testdata/fake.json | 0 .../module/googlecloud/audit/config/input.yml | 4 ++-- .../filebeat/module/googlecloud/audit/manifest.yml | 2 +- .../module/googlecloud/firewall/config/input.yml | 4 ++-- .../module/googlecloud/firewall/manifest.yml | 2 +- .../module/googlecloud/vpcflow/config/input.yml | 4 ++-- 17 files changed, 30 insertions(+), 24 deletions(-) rename x-pack/filebeat/docs/inputs/{input-google-pubsub.asciidoc => input-gcp-pubsub.asciidoc} (90%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/_meta/Dockerfile (100%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/_meta/supported-versions.yml (100%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/config.go (98%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/config_test.go (98%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/docker-compose.yml (100%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/input.go (95%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/pubsub_test.go (99%) rename x-pack/filebeat/input/{googlepubsub => gcppubsub}/testdata/fake.json (100%) diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl index f083b4c814b..c5351be3339 100644 --- a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl +++ b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl @@ -30,7 +30,7 @@ #---------------------------- Google Cloud Pub/Sub Input ----------------------- # Input for reading messages from a Google Cloud Pub/Sub topic subscription. -- type: google-pubsub +- type: gcp-pubsub enabled: false # Google Cloud project ID. Required. diff --git a/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc b/x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc similarity index 90% rename from x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc rename to x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc index c03a3327602..8545065416f 100644 --- a/x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc @@ -1,15 +1,15 @@ [role="xpack"] -:type: google-pubsub +:type: gcp-pubsub [id="{beatname_lc}-input-{type}"] -=== Google Cloud Pub/Sub input +=== GCP Pub/Sub input ++++ -Google Pub/Sub +GCP Pub/Sub ++++ -Use the `google-pubsub` input to read messages from a Google Cloud Pub/Sub topic +Use the `gcp-pubsub` input to read messages from a Google Cloud Pub/Sub topic subscription. This input can, for example, be used to receive Stackdriver logs that have been @@ -23,7 +23,7 @@ Example configuration: ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: -- type: google-pubsub +- type: gcp-pubsub project_id: my-gcp-project-id topic: vpc-firewall-logs-topic subscription.name: filebeat-vpc-firewall-logs-sub @@ -33,7 +33,7 @@ Example configuration: ==== Configuration options -The `google-pubsub` input supports the following configuration options plus the +The `gcp-pubsub` input supports the following configuration options plus the <<{beatname_lc}-input-{type}-common-options>> described later. [float] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 80bfacbf2c3..7b32180921f 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2373,7 +2373,7 @@ filebeat.inputs: #---------------------------- Google Cloud Pub/Sub Input ----------------------- # Input for reading messages from a Google Cloud Pub/Sub topic subscription. -- type: google-pubsub +- type: gcp-pubsub enabled: false # Google Cloud project ID. Required. diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index fe3dd04ad7d..292b9eaf004 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -10,7 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/v7/x-pack/filebeat/input/awscloudwatch" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/azureeventhub" - _ "github.com/elastic/beats/v7/x-pack/filebeat/input/googlepubsub" + _ "github.com/elastic/beats/v7/x-pack/filebeat/input/gcppubsub" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/activemq" diff --git a/x-pack/filebeat/input/googlepubsub/_meta/Dockerfile b/x-pack/filebeat/input/gcppubsub/_meta/Dockerfile similarity index 100% rename from x-pack/filebeat/input/googlepubsub/_meta/Dockerfile rename to x-pack/filebeat/input/gcppubsub/_meta/Dockerfile diff --git a/x-pack/filebeat/input/googlepubsub/_meta/supported-versions.yml b/x-pack/filebeat/input/gcppubsub/_meta/supported-versions.yml similarity index 100% rename from x-pack/filebeat/input/googlepubsub/_meta/supported-versions.yml rename to x-pack/filebeat/input/gcppubsub/_meta/supported-versions.yml diff --git a/x-pack/filebeat/input/googlepubsub/config.go b/x-pack/filebeat/input/gcppubsub/config.go similarity index 98% rename from x-pack/filebeat/input/googlepubsub/config.go rename to x-pack/filebeat/input/gcppubsub/config.go index 56f02f3a471..b9b5ddeedad 100644 --- a/x-pack/filebeat/input/googlepubsub/config.go +++ b/x-pack/filebeat/input/gcppubsub/config.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package googlepubsub +package gcppubsub import ( "context" diff --git a/x-pack/filebeat/input/googlepubsub/config_test.go b/x-pack/filebeat/input/gcppubsub/config_test.go similarity index 98% rename from x-pack/filebeat/input/googlepubsub/config_test.go rename to x-pack/filebeat/input/gcppubsub/config_test.go index 6e949326ca3..9897867507c 100644 --- a/x-pack/filebeat/input/googlepubsub/config_test.go +++ b/x-pack/filebeat/input/gcppubsub/config_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package googlepubsub +package gcppubsub import ( "os" diff --git a/x-pack/filebeat/input/googlepubsub/docker-compose.yml b/x-pack/filebeat/input/gcppubsub/docker-compose.yml similarity index 100% rename from x-pack/filebeat/input/googlepubsub/docker-compose.yml rename to x-pack/filebeat/input/gcppubsub/docker-compose.yml diff --git a/x-pack/filebeat/input/googlepubsub/input.go b/x-pack/filebeat/input/gcppubsub/input.go similarity index 95% rename from x-pack/filebeat/input/googlepubsub/input.go rename to x-pack/filebeat/input/gcppubsub/input.go index a9e1b9f7873..2bc27f83e68 100644 --- a/x-pack/filebeat/input/googlepubsub/input.go +++ b/x-pack/filebeat/input/gcppubsub/input.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package googlepubsub +package gcppubsub import ( "context" @@ -26,7 +26,8 @@ import ( ) const ( - inputName = "google-pubsub" + inputName = "gcp-pubsub" + oldInputName = "google-pubsub" ) func init() { @@ -34,6 +35,11 @@ func init() { if err != nil { panic(errors.Wrapf(err, "failed to register %v input", inputName)) } + + err = input.Register(oldInputName, NewInput) + if err != nil { + panic(errors.Wrapf(err, "failed to register %v input", oldInputName)) + } } type pubsubInput struct { @@ -81,7 +87,7 @@ func NewInput( in := &pubsubInput{ config: conf, - log: logp.NewLogger("google.pubsub").With( + log: logp.NewLogger("gcp.pubsub").With( "pubsub_project", conf.ProjectID, "pubsub_topic", conf.Topic, "pubsub_subscription", conf.Subscription), @@ -109,7 +115,7 @@ func NewInput( if err != nil { return nil, err } - in.log.Info("Initialized Google Pub/Sub input.") + in.log.Info("Initialized GCP Pub/Sub input.") return in, nil } diff --git a/x-pack/filebeat/input/googlepubsub/pubsub_test.go b/x-pack/filebeat/input/gcppubsub/pubsub_test.go similarity index 99% rename from x-pack/filebeat/input/googlepubsub/pubsub_test.go rename to x-pack/filebeat/input/gcppubsub/pubsub_test.go index caf4e82e698..216bd3cb0b0 100644 --- a/x-pack/filebeat/input/googlepubsub/pubsub_test.go +++ b/x-pack/filebeat/input/gcppubsub/pubsub_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package googlepubsub +package gcppubsub import ( "context" diff --git a/x-pack/filebeat/input/googlepubsub/testdata/fake.json b/x-pack/filebeat/input/gcppubsub/testdata/fake.json similarity index 100% rename from x-pack/filebeat/input/googlepubsub/testdata/fake.json rename to x-pack/filebeat/input/gcppubsub/testdata/fake.json diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/googlecloud/audit/config/input.yml index b5e392ee0b6..f1c71d4b84f 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/audit/config/input.yml @@ -1,6 +1,6 @@ -{{ if eq .input "google-pubsub" }} +{{ if eq .input "gcp-pubsub" }} -type: google-pubsub +type: gcp-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} diff --git a/x-pack/filebeat/module/googlecloud/audit/manifest.yml b/x-pack/filebeat/module/googlecloud/audit/manifest.yml index cacba81ad71..42b5c4880d6 100644 --- a/x-pack/filebeat/module/googlecloud/audit/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/audit/manifest.yml @@ -2,7 +2,7 @@ module_version: "1.0" var: - name: input - default: google-pubsub + default: gcp-pubsub - name: project_id default: SET_PROJECT_NAME - name: topic diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml index 39648636c59..1ddda931c49 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml @@ -1,6 +1,6 @@ -{{ if eq .input "google-pubsub" }} +{{ if eq .input "gcp-pubsub" }} -type: google-pubsub +type: gcp-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} diff --git a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml index 6563173197f..009ace59c23 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml @@ -2,7 +2,7 @@ module_version: "1.0" var: - name: input - default: google-pubsub + default: gcp-pubsub - name: project_id default: SET_PROJECT_NAME - name: topic diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml index f1976195687..2854b8ed332 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml @@ -1,6 +1,6 @@ -{{ if eq .input "google-pubsub" }} +{{ if eq .input "gcp-pubsub" }} -type: google-pubsub +type: gcp-pubsub project_id: {{ .project_id }} topic: {{ .topic }} subscription.name: {{ .subscription_name }} From 2a6d72226d6c22649d194ea2646e68091b92bee8 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 27 Oct 2020 15:15:14 -0600 Subject: [PATCH 2/5] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 05e9a8e23de..87ab3716b1d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -643,6 +643,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add SSL option to checkpoint module {pull}19560[19560] - Add max_number_of_messages config into s3 input. {pull}21993[21993] - Update Okta documentation for new stateful restarts. {pull}22091[22091] +- Rename google-pubsub input to gcp-pubsub. {pull}22213[22213] *Heartbeat* From 010845193103eab36d6badfcde98467e769917d2 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 27 Oct 2020 18:50:21 -0600 Subject: [PATCH 3/5] change to gcp-pubsub input name in elastic-agent --- filebeat/docs/filebeat-options.asciidoc | 2 +- x-pack/elastic-agent/spec/filebeat.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index 4cfa0961a20..454e48664ed 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -66,7 +66,7 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-cloudfoundry>> * <<{beatname_lc}-input-container>> * <<{beatname_lc}-input-docker>> -* <<{beatname_lc}-input-google-pubsub>> +* <<{beatname_lc}-input-gcp-pubsub>> * <<{beatname_lc}-input-http_endpoint>> * <<{beatname_lc}-input-httpjson>> * <<{beatname_lc}-input-kafka>> diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index aa09b4f9121..2472f2b5220 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -64,7 +64,7 @@ rules: - cloudfoundry - container - docker - - google-pubsub + - gcp-pubsub - http_endpoint - httpjson - kafka From cb6084c3c5f359a3a893571594329e0119337ce2 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 28 Oct 2020 14:23:14 -0600 Subject: [PATCH 4/5] add warning log message for deprecated old input name --- .../pkg/agent/program/supported.go | 2 +- x-pack/filebeat/input/gcppubsub/config.go | 6 ++++++ x-pack/filebeat/input/gcppubsub/input.go | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index adc13938ae2..67e094a2074 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -21,7 +21,7 @@ func init() { // spec/filebeat.yml // spec/heartbeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJy8WF1zozoSfd+fMa93axdEnF226j4YUnzFJmOSSEJvSLIBW2DfALbx1v73LQHmw0lm7uxszcOUByKk7lb36XP631+Kw5r9fZ3zwz7Ny7/Vmfjyry80s0ryso8DNNsxWz/QfBW/Arjl2DtwZ/cYAnX3lBqCZsGJAlFxU70Q5KssE8p6dUhYHhxIZm35wz4mwx4lsSEwc1+wnBxC8HrvPoTa00P8GIJEhKDcRGh24bZV0If94+LZEGsbbjEgB2q/3pvpPHZN4xTiYP+UztPxvmywLe3WJSzjl6d4H7vmPF48z1OewTpCZOZ277gtSoJ0Vdq4vMwfma1fuCX385UQnYuneF+6NrwjyN+QTBTkZf8ov3MdI+F2fO+a3sf+P7vtOtuqibbs7J6Xrun1e7sjuxbPqspsXocoEDfva4L9I8feluBlOtrnk3Mn66t1Jk4f+epv5yczN2oCdZVmomJakFD7dG+mSkxwIkJVzyJ0FtfYMdtSood97GawIo5xjNBMWWBfhBqsIxz08Qyxl7NLF6NrzNHsnc/vbfFUasNLG29yWFv6hTueCJFy7zqlbnbvqRMIJnQQorNK8DWuxoWgswi14Mi2+zhCsxPHwaX72xvBu3vXCWbMfu3ujiTUgWKwUxnn52MTg0wU3IY11m7WOr6gNtxyW6+fUuNAc0PlzrK761KsX5pcT8LsLMi88zWzCo7gKA8NheVQND5d92tyLjj28QawIMhXqOZdnlKDErkfXlUh8rcE+xcMrFMEdelb4dqkIAgqi6w8hJlVhVCZ5mj/d+sUrZqaKkM8v6klI6M2FLyzmeWwGOI7L13HExTpgLRnXt83/yIAZ0+pkYTAF0zzNyE2DhiUYr3q/a0JUo88g5tmbefjOGYREGmIZsnknrfvYz65szYm/fP03uela+sqdwz16lNjByYHBsSRxvtHDhJBt/uY2rAiWrB/NIN/tHsG+uPz/Df3YR6HaLZz7bOgGVciM96tgaiYAxWmKQf34S5emkZCs1Uc2dblGcCZ3INqUJFrNs+n2AOwCLGvRMi/EGTVIYjzx9X+9y9/bSF3k4o1XUfvIFdCDfJEiFdXmG3KMcxgwueHFtZSg7qparnpKXZzX3AHnhaZKOjzTNDMSqkNd1+RTF9fNGtu1+aBoNgoQhyIRQarEHkFQSudZFbBwGu6MOfp4rX9pciqQsQFRbDi5qykIBBfcVwy29pGtdqmjukWrumWwbP89Up5nQTAkkioGO3PHU8lz5O1BQU8j9AsX2RnwTNYfEWBCHOYu0J5DLGnRIgkoba6d20Zk+CyaNoBTAmylO9CR9qkxh+ynDAQFbHh3TUFuSNOMt7U1nN2akrjQLODhJIN04KaIKvEmlHTNrWPfUraeoWBf6QZKSLkKy0UyJYWbEJEFII7+G9h5961z0eiLRtoocg63cLqDWTVHJ0n8BQC/bSGekLt84bb+oba4sIfBph1TUOhl318tZmdxiX2ztaKAv00LmGCky3BhtLkVO4rLIMJxcvm7iO0an57WGvu2TuxTG+gSEKUvKcbWxWq6kWEfWVa7kIhzb2MYpov/1c/hphnMKOa10GqbI1NHXV3RWoKlHvX7kr3dG1B/xzeab3Pj10LVJikLlbrAwbSbvXy2b3d2hvhQNCX935Mzjx9CsXTtuL0+T20j8yqGDgnvKdE84ldTV6vxrFTE+YYA6T2789H0lGr5v/jeDd5QQTNV8eO+jR1Mj7PNQ1ZrxU39Qu3g4OEU6YFuwjd3ZwDQYMDWrBl0j7bP32yj0qc+b3rwB2bT22RZy9AcAxBKf2Iia1vIwDrm30KCtiRZXAXYX/DwPnIwflIZE4175bv/a/1yxr78rt71/Fn8ptrHP5M6+LYFxh80Gq+8x2xLSWEPVb19cMyWFKNiKaFvkxqvKUydpBw2+rxaZHNEorgRWIx+YGWe3N+1TxjX1IAmZey3ygEe5tb+jJQE/ddTXVUQFljQ3Q5fUPVZFu2NAb8gmpwx4GlhCAesAMfVJa9lm3eBXuO3BGunI8cBRnVJAX1ZsN+/pHmQRKhmWBDjewo8N96HJZ0AehHAs5igQ01zH01HNbuuROcMBio7bB3onDH+IMBvRrelQnJymR4nuRLyXAw+n4muE0KqrHeDnpZAh9ZKrGFMqZXxBYDdXGWo//7CrFFNXq+zVEl1Ob9/hwFp2EtrCI8xJcDUTU1/LN02u77+aeUuunzq2lPbThEHkiJl0mcX2tKg7dNz/hTeD3q+z9EFQc+1d9DR9NG9yrWuKHcwsz5nqC7+yn1G85eZD9PAxfmPO8wKV80uMHfQkTewmdWuCaXnEjK4EtksoMZ/95TxmQdvZUfcMZnG0rJ3cYm80si+9PkXStFXavoeR0DUOF4XkXoXH6PA17XchuWzG56T9Vzggc1C9H5csPrbjigeiT2q7421VOI/LcFamXJhJtmakIzKydIlX1nvH8jmaZrZe/iB5qxijb95aQTG6YcsRTfjB8a3uwsj5N45OO+2uTIHb7ygJc2N9u8uK0Tf8OBUCJLrwniYu3Mhx587QmT/mvU8n5w7s8kphEcHEJteVykRY/tn2PqN6Tod7D401r9QJLe1Ozk3GHNCBfy5YR3NDU+5izmxxKsl2oAzpjcJ3v9UIZdMWfzvIu/pvOTa1sVMY19iP0Fwbu955RHjgO5Rpd1RNA5YZqMqy9C7G0js6mhmqDgwGpWSFs90OK2V8t6lHzcl/1u79W7x2ttZevyLWUfFNcLggrLxLYTYFuKpBhRBXe8Qwg6odbOPmJU9wVwIThQmTk7UFv5XrFc10qieaK2pZDvCbmbYqFI35EX9W6BpRYvyk4ff0vIDfvjoOboRvTZek6kYKpnRdOQH9QdQZ5Kao9LMOG2yMKWiDcFxWq9JDioI+R3BWYcmRZM5nltUrTEZDJPm8yY1CNxmvlDRcyGnEnRUK2R2s+PpFCQ8SZ4dS+BhoKgKeZFtjoyTVwkSC1yUVJzJongVbg8DnONjwt+3KwiNNsRHF+bYkNqnlLj6uOlbUCiirJmftMRKXXDHO8YAnhhQO+Lh4LZJgR6RbLzoRWyomIA1tzSE5IHPXHpBWmXb51IqGXuUNTPPzOW6eV7oRAch3f+1Z7OTjVhDzezyw/EzyeCY0s1Y4aBVVDrE2HXnj2cOQKH977PjrQnLC35X9u+YM6qaU69aKqbujh0grPP1XaIMRGQKV7d2KoFRwzOB6atprOqqzAb3dFEZP6QH/0dpgSRBsx+sXh8R9Kxxg/cTjYsgznBST9o+ICYt00pvXtbgA7HtOXumwTw15LGnxTG8PMm/S2h7HiyxtePD/rqazu8+W2RFof3MeoaqTzjYR974/lyK+KqEKliKrw6YjtZOww9JH5zdBYDIVWTCMBNiL06vJ25djnS4wToiewoV642+2JozH9GYI6++xFBezP3/rUiuHm+jOfBv0pI3wwA/s+CZ8ox5Le3uUVy7zjOhYajSM7xvjf0ffJHxNNk3+FuOwI2mr+PSN3HAmriS3XN958SUa1w6oned0XUf/7y3wAAAP//NCXBlQ==") + unpacked := packer.MustUnpack("eJy8WF1zozoSfd+fMa93axdEnF226j4YUnzFJmOSSEJvSLIBW2DfALbx1v73LQHmw0lm7uxszcOUByKk7lb36XP631+Kw5r9fZ3zwz7Ny7/Vmfjyry80s0ryso8DNNsxWz/QfBW/Arjl2DtwZ/cYAnX3lBqCZsGJAlFxU70Q5KssE8p6dUhYHhxIZm35wz4mwx4lsSEwc1+wnBxC8HrvPoTa00P8GIJEhKDcRGh24bZV0If94+LZEGsbbjEgB2q/3pvpPHZN4xTiYP+UztPxvmywLe3WJSzjl6d4H7vmPF48z1OewTpCZOZ277gtSoJ0Vdq4vMwfma1fuCX385UQnYuneF+6NrwjyN+QTBTkZf8ov3MdI+F2fO+a3sf+P7vtOtuqibbs7J6Xrun1e7sjuxbPqspsXocoEDfva4L9I8feluBlOtrnk3Mn66t1Jk4f+epv5yczN2oCdZVmomJakFD7dG+mSkxwIkJVzyJ0FtfYMdtSood97GawIo5xjNBMWWBfhBqsIxz08Qyxl7NLF6NrzNHsnc/vbfFUasNLG29yWFv6hTueCJFy7zqlbnbvqRMIJnQQorNK8DWuxoWgswi14Mi2+zhCsxPHwaX72xvBu3vXCWbMfu3ujiTUgWKwUxnn52MTg0wU3IY11m7WOr6gNtxyW6+fUuNAc0PlzrK761KsX5pcT8LsLMi88zWzCo7gKA8NheVQND5d92tyLjj28QawIMhXqOZdnlKDErkfXlUh8rcE+xcMrFMEdelb4dqkIAgqi6w8hJlVhVCZ5mj/d+sUrZqaKkM8v6klI6M2FLyzmeWwGOI7L13HExTpgLRnXt83/yIAZ0+pkYTAF0zzNyE2DhiUYr3q/a0JUo88g5tmbefjOGYREGmIZsnknrfvYz65szYm/fP03uela+sqdwz16lNjByYHBsSRxvtHDhJBt/uY2rAiWrB/NIN/tHsG+uPz/Df3YR6HaLZz7bOgGVciM96tgaiYAxWmKQf34S5emkZCs1Uc2dblGcCZ3INqUJFrNs+n2AOwCLGvRMi/EGTVIYjzx9X+9y9/bSF3k4o1XUfvIFdCDfJEiFdXmG3KMcxgwueHFtZSg7qparnpKXZzX3AHnhaZKOjzTNDMSqkNd1+RTF9fNGtu1+aBoNgoQhyIRQarEHkFQSudZFbBwGu6MOfp4rX9pciqQsQFRbDi5qykIBBfcVwy29pGtdqmjukWrumWwbP89Up5nQTAkkioGO3PHU8lz5O1BQU8j9AsX2RnwTNYfEWBCHOYu0J5DLGnRIgkoba6d20Zk+CyaNoBTAmylO9CR9qkxh+ynDAQFbHh3TUFuSNOMt7U1nN2akrjQLODhJIN04KaIKvEmlHTNrWPfUraeoWBf6QZKSLkKy0UyJYWbEJEFII7+G9h5961z0eiLRtoocg63cLqDWTVHJ0n8BQC/bSGekLt84bb+oba4sIfBph1TUOhl318tZmdxiX2ztaKAv00LmGCky3BhtLkVO4rLIMJxcvm7iO0an57WGvu2TuxTG+gSEKUvKcbWxWq6kWEfWVa7kIhzb2MYpov/1c/hphnMKOa10GqbI1NHXV3RWoKlHvX7kr3dG1B/xzeab3Pj10LVJikLlbrAwbSbvXy2b3d2hvhQNCX935Mzjx9CsXTtuL0+T20j8yqGDgnvKdE84ldTV6vxrFTE+YYA6T2789H0lGr5v/jeDd5QQTNV8eO+jR1Mj7PNQ1ZrxU39Qu3g4OEU6YFuwjd3ZwDQYMDWrBl0j7bP32yj0qc+b3rwB2bT22RZy9AcAxBKf2Iia1vIwDrm30KCtiRZXAXYX/DwPnIwflIZE4175bv/a/1yxr78rt71/Fn8ptrHP5M6+LYFxh80Gq+8x2xLSWEPVb19cMyWFKNiKaFvkxqvKUydpBw2+rxaZHNEorgRWIx+YGWe3N+1TxjX1IAmZey3ygEe5tb+jJQE/ddTXVUQFljQ3Q5fUPVZFu2NAb8gmpwx4GlhCAesAMfVJa9lm3eBXuO3BGunI8cBRnVJAX1ZsN+/pHmQRKhmWBDjewo8N96HJZ0AfinBTbUMPfVcFi3505wwmCgtcO+icId4w8G9Gp4VyYkK5PheZIrJcPB6PuZ4DYpqMZ6G+hlCXxkqcQWyphaEVsMtMVZjv7vK8QW1ej5Nj+VUJv3+3MUnIa1sIrwEFsORNXU789Sabvv5Z/S6abHr6b9tOEPeSDlXSYxfq0pDdY2/eJPYfWo5/8QTRy4VH8PHUUb3atY44ZuCzPne4Lu7qe0bzh7kf08BVyY87zDo3zRYAZ/CxF5C59Z4Zpc8iEpgS+RyQ5m/HtPF5N19FZ+wBefbSjldhubzC+J7E2Td60Mda2i53QMQIXjeRWhc/k9/nddy21YMrvpO1XPBx7ULETnyw2nu+F/6pHYr/raVE8h8t8WqJUkE16aqQnNrJwgVfac8f6NXJqulX2LH2jGKtr0lpNObJhyxFJ8M3poOLOzPE7ikY97apMjd/jKAV7a3Gzz4rZO/A0HQoksvSaIi7UzH/rvtR9Meq9Ry/vBuT+TeEZwcAi15XGRFj2uf46n35Ch38HhT2v1Azl6U7OTc4c1I1zIlxPO0dT4mK+YH8uvXqYBOGNyn+z1Qwl2xZzN8y7+ms5Prm1VxDT2IfYXBO/2nlMeOQ7kGl3WEUHnhGkyrr4IsbeNzKaGaoKCA6tZIW31QIvbXi3rUXJxX/a6vVfvHq+1la3Lt5R9UFwvCCosE9tOfG0pkkJEFdzxDiHoRFo794hR3RfAheBAZebsQG3le8VyXStJ5onalkK+J+JuioUifUde1LsFljq8KDtt/C0RN+yPg5qjG8Fn6zmRYqmeFU0zflB3BHkqqT0uwYTbIgtbEt4UFKv1kuCgjpDfFZhxZFowmeW1SdGSksksbTJfUo/EaWYPFTEbYiYFQ7VGaj87kiJBxpvg1b0EGgqCppgX2erINHGRILXIRUnNmSSBV9HyOMw0Pi74cbOK0GxHcHxtig2heUqNq4+XtgGJKsqa2U1HotQNc7xjCOCFAb0vHgpmmxDoFcnOh1bEiooBWHNLT0ge9KSlF6NdvnUCoZa5Q1E/+8xYppfvRUJwHN75V3s6O9WEPdzMLT8QPp+IjS3VjBkGVkGtT0Rde/Zw5ggc3vs+O9KesLTEf237gjmrpjn1gqlu6uLQic0+V9sBxkQ8pnh1Y6sWHDE4H5i2ms6prqJsdEcTgflDfvR3mBJEGjD7xcLxHUHHGj9wO9mwDOYEJ/2Q4QNS3jal9O5tAToc05a7bxLAX0saf1IUw8+b9LdEsuPJGl8/Puirr+3g5rdFWhzex6hrpPKMh33sjWfLrYCrQqSKqejqiO1k7TDwkPjN0VkMhFRNIgA3Ifbq8Hbe2uVIjxOgJ7KjXLna7IuhMf8ZcTn67kfE7M3M+9cK4Ob5Mp4F/yoRfSP+/8+CZ8ox5Le3uUVy7zjOhYajSM7xvjf0ffJHxNNk3+FuOwI2mr2PSN3HAmriS3XN958SUa1w6oned0XUf/7y3wAAAP//HFXAOw==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/filebeat/input/gcppubsub/config.go b/x-pack/filebeat/input/gcppubsub/config.go index b9b5ddeedad..fdb4c30dfee 100644 --- a/x-pack/filebeat/input/gcppubsub/config.go +++ b/x-pack/filebeat/input/gcppubsub/config.go @@ -9,11 +9,14 @@ import ( "fmt" "os" + "github.com/elastic/beats/v7/filebeat/harvester" + "cloud.google.com/go/pubsub" "golang.org/x/oauth2/google" ) type config struct { + harvester.ForwarderConfig `config:",inline"` // Google Cloud project name. ProjectID string `config:"project_id" validate:"required"` @@ -62,6 +65,9 @@ func (c *config) Validate() error { func defaultConfig() config { var c config + c.ForwarderConfig = harvester.ForwarderConfig{ + Type: "gcp-pubsub", + } c.Subscription.NumGoroutines = 1 c.Subscription.MaxOutstandingMessages = 1000 c.Subscription.Create = true diff --git a/x-pack/filebeat/input/gcppubsub/input.go b/x-pack/filebeat/input/gcppubsub/input.go index 2bc27f83e68..69e8bb64cc2 100644 --- a/x-pack/filebeat/input/gcppubsub/input.go +++ b/x-pack/filebeat/input/gcppubsub/input.go @@ -26,7 +26,7 @@ import ( ) const ( - inputName = "gcp-pubsub" + inputName = "gcp-pubsub" oldInputName = "google-pubsub" ) @@ -70,6 +70,15 @@ func NewInput( return nil, err } + logger := logp.NewLogger("gcp.pubsub").With( + "pubsub_project", conf.ProjectID, + "pubsub_topic", conf.Topic, + "pubsub_subscription", conf.Subscription) + + if conf.Type == oldInputName { + logger.Warnf("%s input name is deprecated, please use %s instead", oldInputName, inputName) + } + // Wrap input.Context's Done channel with a context.Context. This goroutine // stops with the parent closes the Done channel. inputCtx, cancelInputCtx := context.WithCancel(context.Background()) @@ -86,11 +95,8 @@ func NewInput( workerCtx, workerCancel := context.WithCancel(inputCtx) in := &pubsubInput{ - config: conf, - log: logp.NewLogger("gcp.pubsub").With( - "pubsub_project", conf.ProjectID, - "pubsub_topic", conf.Topic, - "pubsub_subscription", conf.Subscription), + config: conf, + log: logger, inputCtx: inputCtx, workerCtx: workerCtx, workerCancel: workerCancel, From 1bd3b6f87fd42778b24200a611d35ee2f07b44dc Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 28 Oct 2020 14:53:16 -0600 Subject: [PATCH 5/5] fix doc name --- filebeat/docs/filebeat-options.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index 454e48664ed..edafda17bb4 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -96,7 +96,7 @@ include::inputs/input-docker.asciidoc[] include::inputs/input-filestream.asciidoc[] -include::../../x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc[]