From 67ee43c4631f71c5b7c4d1105d953a933dae68b3 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 25 May 2022 12:07:05 +0200 Subject: [PATCH 1/5] Convert aggregator plugins to the new sample-config style. --- plugins/aggregators/basicstats/README.md | 2 +- plugins/aggregators/basicstats/basicstats.go | 10 ++++++++++ .../basicstats/basicstats_sample_config.go | 8 -------- plugins/aggregators/derivative/README.md | 2 +- plugins/aggregators/derivative/derivative.go | 10 ++++++++++ .../derivative/derivative_sample_config.go | 8 -------- plugins/aggregators/final/README.md | 2 +- plugins/aggregators/final/final.go | 10 ++++++++++ plugins/aggregators/final/final_sample_config.go | 8 -------- plugins/aggregators/histogram/README.md | 2 +- plugins/aggregators/histogram/histogram.go | 10 ++++++++++ .../histogram/histogram_sample_config.go | 8 -------- plugins/aggregators/merge/README.md | 2 +- plugins/aggregators/merge/merge.go | 10 ++++++++++ plugins/aggregators/merge/merge_sample_config.go | 8 -------- plugins/aggregators/minmax/README.md | 2 +- plugins/aggregators/minmax/minmax_sample_config.go | 8 -------- plugins/aggregators/quantile/README.md | 2 +- plugins/aggregators/quantile/quantile.go | 10 ++++++++++ .../aggregators/quantile/quantile_sample_config.go | 8 -------- plugins/aggregators/starlark/README.md | 2 +- plugins/aggregators/starlark/starlark.go | 14 +++++++++++++- .../aggregators/starlark/starlark_sample_config.go | 8 -------- plugins/aggregators/valuecounter/README.md | 2 +- .../valuecounter/valuecounter_sample_config.go | 8 -------- 25 files changed, 82 insertions(+), 82 deletions(-) delete mode 100644 plugins/aggregators/basicstats/basicstats_sample_config.go delete mode 100644 plugins/aggregators/derivative/derivative_sample_config.go delete mode 100644 plugins/aggregators/final/final_sample_config.go delete mode 100644 plugins/aggregators/histogram/histogram_sample_config.go delete mode 100644 plugins/aggregators/merge/merge_sample_config.go delete mode 100644 plugins/aggregators/minmax/minmax_sample_config.go delete mode 100644 plugins/aggregators/quantile/quantile_sample_config.go delete mode 100644 plugins/aggregators/starlark/starlark_sample_config.go delete mode 100644 plugins/aggregators/valuecounter/valuecounter_sample_config.go diff --git a/plugins/aggregators/basicstats/README.md b/plugins/aggregators/basicstats/README.md index ede108ec57d90..f71210b8750ae 100644 --- a/plugins/aggregators/basicstats/README.md +++ b/plugins/aggregators/basicstats/README.md @@ -5,7 +5,7 @@ emitting the aggregate every `period` seconds. ## Configuration -```toml +```toml @sample.conf # Keep the aggregate basicstats of each metric passing through. [[aggregators.basicstats]] ## The period on which to flush & clear the aggregator. diff --git a/plugins/aggregators/basicstats/basicstats.go b/plugins/aggregators/basicstats/basicstats.go index 6f4afabd12b98..2ad8d2720ad2a 100644 --- a/plugins/aggregators/basicstats/basicstats.go +++ b/plugins/aggregators/basicstats/basicstats.go @@ -1,6 +1,8 @@ +//go:generate ../../../tools/readme_config_includer/generator package basicstats import ( + _ "embed" "math" "time" @@ -8,6 +10,10 @@ import ( "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type BasicStats struct { Stats []string `toml:"stats"` Log telegraf.Logger @@ -57,6 +63,10 @@ type basicstats struct { TIME time.Time //intermediate value for rate } +func (*BasicStats) SampleConfig() string { + return sampleConfig +} + func (b *BasicStats) Add(in telegraf.Metric) { id := in.HashID() if _, ok := b.cache[id]; !ok { diff --git a/plugins/aggregators/basicstats/basicstats_sample_config.go b/plugins/aggregators/basicstats/basicstats_sample_config.go deleted file mode 100644 index c1c391e18afc3..0000000000000 --- a/plugins/aggregators/basicstats/basicstats_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package basicstats - -func (*BasicStats) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/derivative/README.md b/plugins/aggregators/derivative/README.md index 124b96b2444c0..5d308599edb04 100644 --- a/plugins/aggregators/derivative/README.md +++ b/plugins/aggregators/derivative/README.md @@ -137,7 +137,7 @@ Using `max_roll_over` with a value greater 0 may be important, if you need to de ## Configuration -```toml +```toml @sample.conf # Calculates a derivative for every field. [[aggregators.derivative]] ## Specific Derivative Aggregator Arguments: diff --git a/plugins/aggregators/derivative/derivative.go b/plugins/aggregators/derivative/derivative.go index 0b45ad30cb3ed..840505ae7cc84 100644 --- a/plugins/aggregators/derivative/derivative.go +++ b/plugins/aggregators/derivative/derivative.go @@ -1,6 +1,8 @@ +//go:generate ../../../tools/readme_config_includer/generator package derivative import ( + _ "embed" "strings" "time" @@ -8,6 +10,10 @@ import ( "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Derivative struct { Variable string `toml:"variable"` Suffix string `toml:"suffix"` @@ -38,6 +44,10 @@ func NewDerivative() *Derivative { return derivative } +func (*Derivative) SampleConfig() string { + return sampleConfig +} + func (d *Derivative) Add(in telegraf.Metric) { id := in.HashID() current, ok := d.cache[id] diff --git a/plugins/aggregators/derivative/derivative_sample_config.go b/plugins/aggregators/derivative/derivative_sample_config.go deleted file mode 100644 index cb88dfa07d4c2..0000000000000 --- a/plugins/aggregators/derivative/derivative_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package derivative - -func (d *Derivative) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/final/README.md b/plugins/aggregators/final/README.md index c7e401d75bf8c..4bf6800ff0b77 100644 --- a/plugins/aggregators/final/README.md +++ b/plugins/aggregators/final/README.md @@ -13,7 +13,7 @@ When a series has not been updated within the time defined in ## Configuration -```toml +```toml @sample.conf # Report the final metric of a series [[aggregators.final]] ## The period on which to flush & clear the aggregator. diff --git a/plugins/aggregators/final/final.go b/plugins/aggregators/final/final.go index 09b244f17b49a..a8699eadbb363 100644 --- a/plugins/aggregators/final/final.go +++ b/plugins/aggregators/final/final.go @@ -1,6 +1,8 @@ +//go:generate ../../../tools/readme_config_includer/generator package final import ( + _ "embed" "time" "github.com/influxdata/telegraf" @@ -8,6 +10,10 @@ import ( "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Final struct { SeriesTimeout config.Duration `toml:"series_timeout"` @@ -22,6 +28,10 @@ func NewFinal() *Final { } } +func (*Final) SampleConfig() string { + return sampleConfig +} + func (m *Final) Add(in telegraf.Metric) { id := in.HashID() m.metricCache[id] = in diff --git a/plugins/aggregators/final/final_sample_config.go b/plugins/aggregators/final/final_sample_config.go deleted file mode 100644 index 181fdfd4c0c06..0000000000000 --- a/plugins/aggregators/final/final_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package final - -func (m *Final) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/histogram/README.md b/plugins/aggregators/histogram/README.md index 2eeb33fc6dc9a..b5bc0d387f2b5 100644 --- a/plugins/aggregators/histogram/README.md +++ b/plugins/aggregators/histogram/README.md @@ -26,7 +26,7 @@ of the algorithm which is implemented in the Prometheus ## Configuration -```toml +```toml @sample.conf # Configuration for aggregate histogram metrics [[aggregators.histogram]] ## The period in which to flush the aggregator. diff --git a/plugins/aggregators/histogram/histogram.go b/plugins/aggregators/histogram/histogram.go index 7c0e39e260777..29ef7c8cebae7 100644 --- a/plugins/aggregators/histogram/histogram.go +++ b/plugins/aggregators/histogram/histogram.go @@ -1,6 +1,8 @@ +//go:generate ../../../tools/readme_config_includer/generator package histogram import ( + _ "embed" "sort" "strconv" "time" @@ -10,6 +12,10 @@ import ( "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + // bucketRightTag is the tag, which contains right bucket border const bucketRightTag = "le" @@ -82,6 +88,10 @@ func NewHistogramAggregator() *HistogramAggregator { return h } +func (*HistogramAggregator) SampleConfig() string { + return sampleConfig +} + // Add adds new hit to the buckets func (h *HistogramAggregator) Add(in telegraf.Metric) { addTime := timeNow() diff --git a/plugins/aggregators/histogram/histogram_sample_config.go b/plugins/aggregators/histogram/histogram_sample_config.go deleted file mode 100644 index 34ef6869dd348..0000000000000 --- a/plugins/aggregators/histogram/histogram_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package histogram - -func (h *HistogramAggregator) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/merge/README.md b/plugins/aggregators/merge/README.md index ca0c03c969dec..362a86673023a 100644 --- a/plugins/aggregators/merge/README.md +++ b/plugins/aggregators/merge/README.md @@ -9,7 +9,7 @@ be handled more efficiently by the output. ## Configuration -```toml +```toml @sample.conf # Merge metrics into multifield metrics by series key [[aggregators.merge]] ## If true, the original metric will be dropped by the diff --git a/plugins/aggregators/merge/merge.go b/plugins/aggregators/merge/merge.go index e1c06bec0fe4b..5038020d9e7f4 100644 --- a/plugins/aggregators/merge/merge.go +++ b/plugins/aggregators/merge/merge.go @@ -1,6 +1,8 @@ +//go:generate ../../../tools/readme_config_includer/generator package merge import ( + _ "embed" "time" "github.com/influxdata/telegraf" @@ -8,10 +10,18 @@ import ( "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Merge struct { grouper *metric.SeriesGrouper } +func (*Merge) SampleConfig() string { + return sampleConfig +} + func (a *Merge) Init() error { a.grouper = metric.NewSeriesGrouper() return nil diff --git a/plugins/aggregators/merge/merge_sample_config.go b/plugins/aggregators/merge/merge_sample_config.go deleted file mode 100644 index 58a87a491603c..0000000000000 --- a/plugins/aggregators/merge/merge_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package merge - -func (a *Merge) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/minmax/README.md b/plugins/aggregators/minmax/README.md index fefd2f2e2e165..84cc1c0bb324f 100644 --- a/plugins/aggregators/minmax/README.md +++ b/plugins/aggregators/minmax/README.md @@ -5,7 +5,7 @@ emitting the aggrate every `period` seconds. ## Configuration -```toml +```toml @sample.conf # Keep the aggregate min/max of each metric passing through. [[aggregators.minmax]] ## General Aggregator Arguments: diff --git a/plugins/aggregators/minmax/minmax_sample_config.go b/plugins/aggregators/minmax/minmax_sample_config.go deleted file mode 100644 index 4de3ba2ac2891..0000000000000 --- a/plugins/aggregators/minmax/minmax_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package minmax - -func (m *MinMax) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/quantile/README.md b/plugins/aggregators/quantile/README.md index f6d44ea09044b..87ab3ca0cce66 100644 --- a/plugins/aggregators/quantile/README.md +++ b/plugins/aggregators/quantile/README.md @@ -5,7 +5,7 @@ per metric it sees and emits the quantiles every `period`. ## Configuration -```toml +```toml @sample.conf # Keep the aggregate quantiles of each metric passing through. [[aggregators.quantile]] ## General Aggregator Arguments: diff --git a/plugins/aggregators/quantile/quantile.go b/plugins/aggregators/quantile/quantile.go index d724c61a4e11e..739b83ab79d56 100644 --- a/plugins/aggregators/quantile/quantile.go +++ b/plugins/aggregators/quantile/quantile.go @@ -1,12 +1,18 @@ +//go:generate ../../../tools/readme_config_includer/generator package quantile import ( + _ "embed" "fmt" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Quantile struct { Quantiles []float64 `toml:"quantiles"` Compression float64 `toml:"compression"` @@ -26,6 +32,10 @@ type aggregate struct { type newAlgorithmFunc func(compression float64) (algorithm, error) +func (*Quantile) SampleConfig() string { + return sampleConfig +} + func (q *Quantile) Add(in telegraf.Metric) { id := in.HashID() if cached, ok := q.cache[id]; ok { diff --git a/plugins/aggregators/quantile/quantile_sample_config.go b/plugins/aggregators/quantile/quantile_sample_config.go deleted file mode 100644 index 2329d264389b0..0000000000000 --- a/plugins/aggregators/quantile/quantile_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package quantile - -func (q *Quantile) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/starlark/README.md b/plugins/aggregators/starlark/README.md index 87fbaa08b23b3..87a8ed8e57fbd 100644 --- a/plugins/aggregators/starlark/README.md +++ b/plugins/aggregators/starlark/README.md @@ -18,7 +18,7 @@ functions. ## Configuration -```toml +```toml @sample.conf # Aggregate metrics using a Starlark script [[aggregators.starlark]] ## The Starlark source can be set as a string in this configuration file, or diff --git a/plugins/aggregators/starlark/starlark.go b/plugins/aggregators/starlark/starlark.go index 8c58b5afa0e26..ade07789396c4 100644 --- a/plugins/aggregators/starlark/starlark.go +++ b/plugins/aggregators/starlark/starlark.go @@ -1,16 +1,28 @@ +//go:generate ../../../tools/readme_config_includer/generator package starlark import ( + _ "embed" + + "go.starlark.net/starlark" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/aggregators" common "github.com/influxdata/telegraf/plugins/common/starlark" - "go.starlark.net/starlark" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Starlark struct { common.StarlarkCommon } +func (*Starlark) SampleConfig() string { + return sampleConfig +} + func (s *Starlark) Init() error { // Execute source err := s.StarlarkCommon.Init() diff --git a/plugins/aggregators/starlark/starlark_sample_config.go b/plugins/aggregators/starlark/starlark_sample_config.go deleted file mode 100644 index 67d3483d7d0d3..0000000000000 --- a/plugins/aggregators/starlark/starlark_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package starlark - -func (s *Starlark) SampleConfig() string { - return `{{ .SampleConfig }}` -} diff --git a/plugins/aggregators/valuecounter/README.md b/plugins/aggregators/valuecounter/README.md index 2d26bc1b6a764..fbb278e669fd3 100644 --- a/plugins/aggregators/valuecounter/README.md +++ b/plugins/aggregators/valuecounter/README.md @@ -17,7 +17,7 @@ limited set of values. ## Configuration -```toml +```toml @sample.conf # Count the occurrence of values in fields. [[aggregators.valuecounter]] ## General Aggregator Arguments: diff --git a/plugins/aggregators/valuecounter/valuecounter_sample_config.go b/plugins/aggregators/valuecounter/valuecounter_sample_config.go deleted file mode 100644 index 8bc4a49caae8a..0000000000000 --- a/plugins/aggregators/valuecounter/valuecounter_sample_config.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package valuecounter - -func (vc *ValueCounter) SampleConfig() string { - return `{{ .SampleConfig }}` -} From b4258975baed50a256546d13701df8e50dac417f Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 25 May 2022 12:10:06 +0200 Subject: [PATCH 2/5] Convert remaining aggregators manually. --- plugins/aggregators/minmax/minmax.go | 11 +++++++++++ plugins/aggregators/valuecounter/valuecounter.go | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/plugins/aggregators/minmax/minmax.go b/plugins/aggregators/minmax/minmax.go index 5c3d70828dbb9..7ff0b284de280 100644 --- a/plugins/aggregators/minmax/minmax.go +++ b/plugins/aggregators/minmax/minmax.go @@ -1,10 +1,17 @@ +//go:generate ../../../tools/readme_config_includer/generator package minmax import ( + _ "embed" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type MinMax struct { cache map[uint64]aggregate } @@ -26,6 +33,10 @@ type minmax struct { max float64 } +func (*MinMax) SampleConfig() string { + return sampleConfig +} + func (m *MinMax) Add(in telegraf.Metric) { id := in.HashID() if _, ok := m.cache[id]; !ok { diff --git a/plugins/aggregators/valuecounter/valuecounter.go b/plugins/aggregators/valuecounter/valuecounter.go index 5701b34352262..391d56f9c1358 100644 --- a/plugins/aggregators/valuecounter/valuecounter.go +++ b/plugins/aggregators/valuecounter/valuecounter.go @@ -1,12 +1,18 @@ +//go:generate ../../../tools/readme_config_includer/generator package valuecounter import ( + _ "embed" "fmt" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type aggregate struct { name string tags map[string]string @@ -27,6 +33,10 @@ func NewValueCounter() telegraf.Aggregator { return vc } +func (*ValueCounter) SampleConfig() string { + return sampleConfig +} + // Add is run on every metric which passes the plugin func (vc *ValueCounter) Add(in telegraf.Metric) { id := in.HashID() From 746e944b010a7fb6e89bd70220c6bc8b1b7608a7 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 25 May 2022 12:11:23 +0200 Subject: [PATCH 3/5] Update Makefile. --- Makefile | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index ace542df4997b..92b529819c36e 100644 --- a/Makefile +++ b/Makefile @@ -119,25 +119,18 @@ versioninfo: build_generator: go build -o ./tools/readme_config_includer/generator ./tools/readme_config_includer/generator.go -insert_config_to_readme_%: build_generator +embed_readme_%: build_generator go generate -run="readme_config_includer/generator$$" ./plugins/$*/... -generate_plugins_%: build_generator - go generate -run="plugindata/main.go$$" ./plugins/$*/... - .PHONY: generate -generate: insert_config_to_readme_inputs insert_config_to_readme_outputs insert_config_to_readme_processors generate_plugins_aggregators - -.PHONY: generate-clean -generate-clean: - go generate -run="plugindata/main.go --clean" ./plugins/aggregators/... +generate: embed_readme_inputs embed_readme_outputs embed_readme_processors embed_readme_aggregators .PHONY: build build: go build -ldflags "$(LDFLAGS)" ./cmd/telegraf .PHONY: telegraf -telegraf: generate build generate-clean +telegraf: generate build # Used by dockerfile builds .PHONY: go-install @@ -340,7 +333,7 @@ darwin-arm64: include_packages := $(mips) $(mipsel) $(arm64) $(amd64) $(static) $(armel) $(armhf) $(riscv64) $(s390x) $(ppc64le) $(i386) $(windows) $(darwin-amd64) $(darwin-arm64) .PHONY: package -package: generate $(include_packages) generate-clean +package: generate $(include_packages) .PHONY: $(include_packages) $(include_packages): From ddd523e088534fb4ffd64f7ff8bb799945615d28 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 25 May 2022 17:52:59 +0200 Subject: [PATCH 4/5] Fix documentation. --- docs/AGGREGATORS.md | 37 +++++++++++++++---------------------- docs/INPUTS.md | 31 +++++++++++++++---------------- docs/OUTPUTS.md | 33 +++++++++++++++------------------ docs/PROCESSORS.md | 42 ++++++++++++++++++++++++------------------ 4 files changed, 69 insertions(+), 74 deletions(-) diff --git a/docs/AGGREGATORS.md b/docs/AGGREGATORS.md index d0604bc02ee02..2d1ddaa5edfca 100644 --- a/docs/AGGREGATORS.md +++ b/docs/AGGREGATORS.md @@ -9,10 +9,11 @@ This section is for developers who want to create a new aggregator plugin. register themselves. See below for a quick example. * To be available within Telegraf itself, plugins must add themselves to the `github.com/influxdata/telegraf/plugins/aggregators/all/all.go` file. -* Each plugin requires a file called `_sample_config.go`, where `` is replaced with the actual plugin name. - Copy the [example template](#sample-configuration-template) into this file, also updating `` were appropriate. - This file is automatically updated during the build process to include the sample configuration from the `README.md`. +* Each plugin requires a file called `sample.conf` containing the sample configuration + for the plugin in TOML format. Please consult the [Sample Config][] page for the latest style guidelines. +* Each plugin `README.md` file should include the `sample.conf` file in a section + describing the configuration by specifying a `toml` section in the form `toml @sample.conf`. The specified file(s) are then injected automatically into the Readme. * The Aggregator plugin will need to keep caches of metrics that have passed through it. This should be done using the builtin `HashID()` function of each metric. @@ -22,17 +23,22 @@ This section is for developers who want to create a new aggregator plugin. ### Aggregator Plugin Example ```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean +//go:generate ../../../tools/readme_config_includer/generator package min // min.go import ( + _ "embed" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/aggregators" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Min struct { // caches for metric fields, names, and tags fieldCache map[uint64]map[string]float64 @@ -46,6 +52,10 @@ func NewMin() telegraf.Aggregator { return m } +func (*Min) SampleConfig() string { + return sampleConfig +} + func (m *Min) Init() error { return nil } @@ -112,20 +122,3 @@ func init() { }) } ``` - -### Sample Configuration Template - -```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package - -func (k *) SampleConfig() string { - return `{{ .SampleConfig }}` -} -``` - -[telegraf.Aggregator]: https://godoc.org/github.com/influxdata/telegraf#Aggregator -[Sample Config]: https://github.com/influxdata/telegraf/blob/master/docs/developers/SAMPLE_CONFIG.md -[Code Style]: https://github.com/influxdata/telegraf/blob/master/docs/developers/CODE_STYLE.md diff --git a/docs/INPUTS.md b/docs/INPUTS.md index e910578a36b2f..97ddf9b537344 100644 --- a/docs/INPUTS.md +++ b/docs/INPUTS.md @@ -15,10 +15,11 @@ and submit new inputs. themselves. See below for a quick example. - Input Plugins must be added to the `github.com/influxdata/telegraf/plugins/inputs/all/all.go` file. -- Each plugin requires a file called `_sample_config.go`, where `` is replaced with the actual plugin name. - Copy the [example template](#sample-configuration-template) into this file, also updating `` were appropriate. - This file is automatically updated during the build process to include the sample configuration from the `README.md`. +- Each plugin requires a file called `sample.conf` containing the sample + configuration for the plugin in TOML format. Please consult the [Sample Config][] page for the latest style guidelines. +- Each plugin `README.md` file should include the `sample.conf` file in a section + describing the configuration by specifying a `toml` section in the form `toml @sample.conf`. The specified file(s) are then injected automatically into the Readme. - Follow the recommended [Code Style][]. Let's say you've written a plugin that emits metrics about processes on the @@ -27,20 +28,29 @@ current host. ## Input Plugin Example ```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean +//go:generate ../../../tools/readme_config_includer/generator package simple import ( + _ "embed" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Simple struct { Ok bool `toml:"ok"` Log telegraf.Logger `toml:"-"` } +func (*Simple) SampleConfig() string { + return sampleConfig +} + // Init is for setup, and validating config. func (s *Simple) Init() error { return nil @@ -61,17 +71,6 @@ func init() { } ``` -```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package - -func (k *) SampleConfig() string { - return `{{ .SampleConfig }}` -} -``` - ### Development - Run `make static` followed by `make plugin-[pluginName]` to spin up a docker diff --git a/docs/OUTPUTS.md b/docs/OUTPUTS.md index fd84b070b7d7f..0ef5a69a17881 100644 --- a/docs/OUTPUTS.md +++ b/docs/OUTPUTS.md @@ -11,31 +11,41 @@ similar constructs. themselves. See below for a quick example. - To be available within Telegraf itself, plugins must add themselves to the `github.com/influxdata/telegraf/plugins/outputs/all/all.go` file. -- Each plugin requires a file called `_sample_config.go`, where `` is replaced with the actual plugin name. - Copy the [example template](#sample-configuration-template) into this file, also updating `` were appropriate. - This file is automatically updated during the build process to include the sample configuration from the `README.md`. +- Each plugin requires a file called `sample.conf` containing the sample + configuration for the plugin in TOML format. Please consult the [Sample Config][] page for the latest style guidelines. +- Each plugin `README.md` file should include the `sample.conf` file in a section + describing the configuration by specifying a `toml` section in the form `toml @sample.conf`. The specified file(s) are then injected automatically into the Readme. - Follow the recommended [Code Style][]. ## Output Plugin Example ```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean +//go:generate ../../../tools/readme_config_includer/generator package simpleoutput // simpleoutput.go import ( + _ "embed" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/outputs" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Simple struct { Ok bool `toml:"ok"` Log telegraf.Logger `toml:"-"` } +func (*Simple) SampleConfig() string { + return sampleConfig +} + // Init is for setup, and validating config. func (s *Simple) Init() error { return nil @@ -68,19 +78,6 @@ func init() { ``` -### Sample Configuration Template - -```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package - -func (k *) SampleConfig() string { - return `{{ .SampleConfig }}` -} -``` - ## Data Formats Some output plugins, such as the [file][] plugin, can write in any supported diff --git a/docs/PROCESSORS.md b/docs/PROCESSORS.md index aeb40cf173cda..ba3ea2d68e56e 100644 --- a/docs/PROCESSORS.md +++ b/docs/PROCESSORS.md @@ -9,32 +9,41 @@ This section is for developers who want to create a new processor plugin. themselves. See below for a quick example. * To be available within Telegraf itself, plugins must add themselves to the `github.com/influxdata/telegraf/plugins/processors/all/all.go` file. -* Each plugin requires a file called `_sample_config.go`, where `` is replaced with the actual plugin name. - Copy the [example template](#sample-configuration-template) into this file, also updating `` were appropriate. - This file is automatically updated during the build process to include the sample configuration from the `README.md`. +* Each plugin requires a file called `sample.conf` containing the sample + configuration for the plugin in TOML format. Please consult the [Sample Config][] page for the latest style guidelines. +* Each plugin `README.md` file should include the `sample.conf` file in a section + describing the configuration by specifying a `toml` section in the form `toml @sample.conf`. The specified file(s) are then injected automatically into the Readme. * Follow the recommended [Code Style][]. ## Processor Plugin Example ```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean +//go:generate ../../../tools/readme_config_includer/generator package printer // printer.go import ( + _ "embed" "fmt" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/processors" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Printer struct { Log telegraf.Logger `toml:"-"` } +func (*Printer) SampleConfig() string { + return sampleConfig +} + // Init is for setup, and validating config. func (p *Printer) Init() error { return nil @@ -54,19 +63,6 @@ func init() { } ``` -### Sample Configuration Template - -```go -//go:generate go run ../../../tools/generate_plugindata/main.go -//go:generate go run ../../../tools/generate_plugindata/main.go --clean -// DON'T EDIT; This file is used as a template by tools/generate_plugindata -package - -func (k *) SampleConfig() string { - return `{{ .SampleConfig }}` -} -``` - ## Streaming Processors Streaming processors are a new processor type available to you. They are @@ -88,21 +84,31 @@ Some differences from classic Processors: ## Streaming Processor Example ```go +//go:generate ../../../tools/readme_config_includer/generator package printer // printer.go import ( + _ "embed" "fmt" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/processors" ) +// DO NOT REMOVE THE NEXT TWO LINES! This is required to embedd the sampleConfig data. +//go:embed sample.conf +var sampleConfig string + type Printer struct { Log telegraf.Logger `toml:"-"` } +func (*Printer) SampleConfig() string { + return sampleConfig +} + // Init is for setup, and validating config. func (p *Printer) Init() error { return nil From 88dd05bc73ad05084df4c7cc8be6f7395fabbfbf Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 25 May 2022 17:53:29 +0200 Subject: [PATCH 5/5] Remove old sample-config generator. --- tools/generate_plugindata/main.go | 157 ------------------------- tools/generate_plugindata/main_test.go | 133 --------------------- 2 files changed, 290 deletions(-) delete mode 100644 tools/generate_plugindata/main.go delete mode 100644 tools/generate_plugindata/main_test.go diff --git a/tools/generate_plugindata/main.go b/tools/generate_plugindata/main.go deleted file mode 100644 index bc096ea74527a..0000000000000 --- a/tools/generate_plugindata/main.go +++ /dev/null @@ -1,157 +0,0 @@ -// generate_plugindata is a tool used to inject the sample configuration into all the plugins -// It extracts the sample configuration from the plugins README.md -// Then using the file plugin_name_sample_config.go as a template, and will be updated with the sample configuration -// This tool is then also used to revert these changes with the `--clean` flag -package main - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "log" //nolint:revive - "os" - "strings" - "text/template" - - "github.com/yuin/goldmark" - gast "github.com/yuin/goldmark/ast" - "github.com/yuin/goldmark/text" -) - -func createSourceName(packageName string) string { - return fmt.Sprintf("%s_sample_config.go", packageName) -} - -// extractPluginData reads the README.md to get the sample configuration -func extractPluginData() (string, error) { - readMe, err := os.ReadFile("README.md") - if err != nil { - return "", err - } - p := goldmark.DefaultParser() - r := text.NewReader(readMe) - root := p.Parse(r) - - var currentSection string - for n := root.FirstChild(); n != nil; n = n.NextSibling() { - switch tok := n.(type) { - case *gast.Heading: - if tok.FirstChild() != nil { - currentSection = string(tok.FirstChild().Text(readMe)) - } - case *gast.FencedCodeBlock: - if currentSection == "Configuration" && string(tok.Language(readMe)) == "toml" { - var config []byte - for i := 0; i < tok.Lines().Len(); i++ { - line := tok.Lines().At(i) - config = append(config, line.Value(readMe)...) - } - return string(config), nil - } - } - } - - fmt.Printf("No configuration found for plugin: %s\n", os.Getenv("GOPACKAGE")) - - return "", nil -} - -// generatePluginData parses the main source file of the plugin as a template and updates it with the sample configuration -// The original source file is saved so that these changes can be reverted -func generatePluginData(packageName string, sampleConfig string) error { - sourceName := createSourceName(packageName) - - plugin, err := os.ReadFile(sourceName) - if err != nil { - return err - } - - generatedTemplate := template.Must(template.New("").Parse(string(plugin))) - - f, err := os.Create(sourceName) - if err != nil { - return err - } - defer f.Close() - - err = generatedTemplate.Execute(f, struct { - SampleConfig string - }{ - SampleConfig: sampleConfig, - }) - if err != nil { - return err - } - - return nil -} - -var newSampleConfigFunc = ` return ` + "`{{ .SampleConfig }}`\n" - -// cleanGeneratedFiles will revert the changes made by generatePluginData -func cleanGeneratedFiles(packageName string) error { - sourceName := createSourceName(packageName) - sourcefile, err := os.Open(sourceName) - if err != nil { - return err - } - defer sourcefile.Close() - - var c []byte - buf := bytes.NewBuffer(c) - - scanner := bufio.NewScanner(sourcefile) - - var sampleconfigSection bool - for scanner.Scan() { - if sampleconfigSection && strings.TrimSpace(scanner.Text()) == "}" { - sampleconfigSection = false - if _, err := buf.Write([]byte(newSampleConfigFunc)); err != nil { - return err - } - } - - if !sampleconfigSection { - if _, err := buf.Write(scanner.Bytes()); err != nil { - return err - } - if _, err = buf.WriteString("\n"); err != nil { - return err - } - } - if !sampleconfigSection && strings.Contains(scanner.Text(), "SampleConfig() string") { - sampleconfigSection = true - } - } - - err = os.WriteFile(sourceName, buf.Bytes(), 0664) - if err != nil { - return err - } - return nil -} - -func main() { - clean := flag.Bool("clean", false, "Remove generated files") - flag.Parse() - - goPackage := os.Getenv("GOPACKAGE") - - if *clean { - err := cleanGeneratedFiles(goPackage) - if err != nil { - log.Fatal(err) - } - } else { - s, err := extractPluginData() - if err != nil { - log.Fatal(err) - } - - err = generatePluginData(goPackage, s) - if err != nil { - log.Fatal(err) - } - } -} diff --git a/tools/generate_plugindata/main_test.go b/tools/generate_plugindata/main_test.go deleted file mode 100644 index 24b34e87e542c..0000000000000 --- a/tools/generate_plugindata/main_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package main - -import ( - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -var originalPlugin = `package main -func (*Plugin) SampleConfig() string { - return ` + "`{{ .SampleConfig }}`" + ` -} - -` - -func TestGeneratePluginData(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - readme := `# plugin - -## Configuration - -` + "```" + `toml -# test plugin -[[input.plugin]] - # No configuration -` + "```" - r, err := os.Create("README.md") - require.NoError(t, err) - _, err = r.Write([]byte(readme)) - require.NoError(t, err) - err = r.Close() - require.NoError(t, err) - - sourceFile, err := os.Create("test_sample_config.go") - require.NoError(t, err) - _, err = sourceFile.Write([]byte(originalPlugin)) - require.NoError(t, err) - err = sourceFile.Close() - require.NoError(t, err) - - defer func() { - err = os.Remove("test_sample_config.go") - require.NoError(t, err) - err = os.Remove("README.md") - require.NoError(t, err) - }() - - s, err := extractPluginData() - require.NoError(t, err) - - err = generatePluginData("test", s) - require.NoError(t, err) - - expected := `package main -func (*Plugin) SampleConfig() string { - return ` + "`" + `# test plugin -[[input.plugin]] - # No configuration -` + "`" + ` -} - -` - - newSourceFile, err := os.ReadFile("test_sample_config.go") - require.NoError(t, err) - - require.Equal(t, expected, string(newSourceFile)) -} - -func TestGeneratePluginDataNoConfig(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - readme := `# plugin` - - r, err := os.Create("README.md") - require.NoError(t, err) - _, err = r.Write([]byte(readme)) - require.NoError(t, err) - err = r.Close() - require.NoError(t, err) - - defer func() { - err = os.Remove("README.md") - require.NoError(t, err) - }() - - s, err := extractPluginData() - require.NoError(t, err) - require.Empty(t, s) -} - -func setupGeneratedPluginFile(t *testing.T, fileName string) { - // Create files that will be cleaned up - r, err := os.Create(fileName) - require.NoError(t, err) - defer r.Close() - - updatePlugin := `package main -func (*Plugin) SampleConfig() string { - return "I am a sample config" -} - -` - _, err = r.Write([]byte(updatePlugin)) - require.NoError(t, err) -} - -func TestCleanGeneratedFiles(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - filename := "testClean_sample_config.go" - - setupGeneratedPluginFile(t, filename) - - err := cleanGeneratedFiles("testClean") - require.NoError(t, err) - - b, err := os.ReadFile(filename) - require.NoError(t, err) - - require.Equal(t, originalPlugin, string(b)) - - err = os.Remove(filename) - require.NoError(t, err) -}