diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1de393a5c..46ab69b56b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ - Rename `NewComponent()` to `New()` - obsReport.NewExporter accepts a settings struct (#2668) - Remove ErrorWaitingHost from `componenttest` (#2582) +- Move `config.Load` to use `configparser.Load` (#2796) +- Remove `configtest.NewViperFromYamlFile()`, use `config.Parser.NewParserFromFile()` (#2806) +- Move `config.ViperSubExact()` to use `config.Parser.Sub()` (#2806) ## 💡 Enhancements 💡 - `batch` processor: - Support max batch size for logs (#2736) diff --git a/config/configparser/config.go b/config/configparser/config.go index 33874784d4c..4778c2cd664 100644 --- a/config/configparser/config.go +++ b/config/configparser/config.go @@ -25,7 +25,6 @@ import ( "strings" "github.com/spf13/cast" - "github.com/spf13/viper" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config" @@ -225,7 +224,7 @@ func loadExtensions(exts map[string]interface{}, factories map[configmodels.Type // Iterate over extensions and create a config for each. for key, value := range exts { - componentConfig := viperFromStringMap(cast.ToStringMap(value)) + componentConfig := config.NewParserFromStringMap(cast.ToStringMap(value)) expandEnvConfig(componentConfig) // Decode the key into type and fullName components. @@ -275,7 +274,7 @@ func loadService(rawService serviceSettings) (configmodels.Service, error) { } // LoadReceiver loads a receiver config from componentConfig using the provided factories. -func LoadReceiver(componentConfig *viper.Viper, typeStr configmodels.Type, fullName string, factory component.ReceiverFactory) (configmodels.Receiver, error) { +func LoadReceiver(componentConfig *config.Parser, typeStr configmodels.Type, fullName string, factory component.ReceiverFactory) (configmodels.Receiver, error) { // Create the default config for this receiver. receiverCfg := factory.CreateDefaultConfig() receiverCfg.SetName(fullName) @@ -297,7 +296,7 @@ func loadReceivers(recvs map[string]interface{}, factories map[configmodels.Type // Iterate over input map and create a config for each. for key, value := range recvs { - componentConfig := viperFromStringMap(cast.ToStringMap(value)) + componentConfig := config.NewParserFromStringMap(cast.ToStringMap(value)) expandEnvConfig(componentConfig) // Decode the key into type and fullName components. @@ -334,7 +333,7 @@ func loadExporters(exps map[string]interface{}, factories map[configmodels.Type] // Iterate over Exporters and create a config for each. for key, value := range exps { - componentConfig := viperFromStringMap(cast.ToStringMap(value)) + componentConfig := config.NewParserFromStringMap(cast.ToStringMap(value)) expandEnvConfig(componentConfig) // Decode the key into type and fullName components. @@ -377,7 +376,7 @@ func loadProcessors(procs map[string]interface{}, factories map[configmodels.Typ // Iterate over processors and create a config for each. for key, value := range procs { - componentConfig := viperFromStringMap(cast.ToStringMap(value)) + componentConfig := config.NewParserFromStringMap(cast.ToStringMap(value)) expandEnvConfig(componentConfig) // Decode the key into type and fullName components. @@ -456,7 +455,7 @@ func loadPipelines(pipelinesConfig map[string]pipelineSettings) (configmodels.Pi // expandEnvConfig creates a new viper config with expanded values for all the values (simple, list or map value). // It does not expand the keys. -func expandEnvConfig(v *viper.Viper) { +func expandEnvConfig(v *config.Parser) { for _, k := range v.AllKeys() { v.Set(k, expandStringValues(v.Get(k))) } @@ -535,20 +534,15 @@ func expandEnv(s string) string { }) } -func unmarshaler(factory component.Factory) component.CustomUnmarshaler { +func unmarshaler(factory component.Factory) func(componentViperSection *config.Parser, intoCfg interface{}) error { if fu, ok := factory.(component.ConfigUnmarshaler); ok { - return fu.Unmarshal + return func(componentParser *config.Parser, intoCfg interface{}) error { + return fu.Unmarshal(componentParser.Viper(), intoCfg) + } } return defaultUnmarshaler } -func defaultUnmarshaler(componentViperSection *viper.Viper, intoCfg interface{}) error { - return componentViperSection.UnmarshalExact(intoCfg) -} - -func viperFromStringMap(data map[string]interface{}) *viper.Viper { - v := config.NewViper() - // Cannot return error because the subv is empty. - _ = v.MergeConfigMap(cast.ToStringMap(data)) - return v +func defaultUnmarshaler(componentParser *config.Parser, intoCfg interface{}) error { + return componentParser.UnmarshalExact(intoCfg) } diff --git a/config/parser.go b/config/parser.go index cd2d88b53d0..24f08fcf964 100644 --- a/config/parser.go +++ b/config/parser.go @@ -58,6 +58,13 @@ func ParserFromViper(v *viper.Viper) *Parser { return &Parser{v: v} } +func NewParserFromStringMap(data map[string]interface{}) *Parser { + v := NewViper() + // Cannot return error because the subv is empty. + _ = v.MergeConfigMap(data) + return ParserFromViper(v) +} + // Parser loads configuration. type Parser struct { v *viper.Viper @@ -111,6 +118,11 @@ func (l *Parser) Sub(key string) (*Parser, error) { return nil, fmt.Errorf("unexpected sub-config value kind for key:%s value:%v kind:%v)", key, data, reflect.TypeOf(data).Kind()) } +// Viper returns the viper.Viper implementation of this Parser. +func (l *Parser) Viper() *viper.Viper { + return l.v +} + // deepSearch scans deep maps, following the key indexes listed in the // sequence "path". // The last value is expected to be another map, and is returned. diff --git a/receiver/hostmetricsreceiver/factory.go b/receiver/hostmetricsreceiver/factory.go index b3aa3d428b4..d68d1e41378 100644 --- a/receiver/hostmetricsreceiver/factory.go +++ b/receiver/hostmetricsreceiver/factory.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" + "github.com/spf13/cast" "github.com/spf13/viper" "go.uber.org/zap" @@ -76,11 +77,9 @@ func NewFactory() component.ReceiverFactory { // customUnmarshaler returns custom unmarshaler for this config. func customUnmarshaler(componentViperSection *viper.Viper, intoCfg interface{}) error { - + componentParser := config.ParserFromViper(componentViperSection) // load the non-dynamic config normally - cp := config.ParserFromViper(componentViperSection) - - err := cp.Unmarshal(intoCfg) + err := componentParser.Unmarshal(intoCfg) if err != nil { return err } @@ -94,7 +93,7 @@ func customUnmarshaler(componentViperSection *viper.Viper, intoCfg interface{}) cfg.Scrapers = map[string]internal.Config{} - scrapersSection, err := cp.Sub(scrapersKey) + scrapersSection, err := componentParser.Sub(scrapersKey) if err != nil { return err } @@ -102,7 +101,7 @@ func customUnmarshaler(componentViperSection *viper.Viper, intoCfg interface{}) return errors.New("must specify at least one scraper when using hostmetrics receiver") } - for key := range componentViperSection.GetStringMap(scrapersKey) { + for key := range cast.ToStringMap(componentParser.Get(scrapersKey)) { factory, ok := getScraperFactory(key) if !ok { return fmt.Errorf("invalid scraper key: %s", key) diff --git a/receiver/jaegerreceiver/factory.go b/receiver/jaegerreceiver/factory.go index 9bd940384f2..ca66ff52b5c 100644 --- a/receiver/jaegerreceiver/factory.go +++ b/receiver/jaegerreceiver/factory.go @@ -22,9 +22,11 @@ import ( "net" "strconv" + "github.com/spf13/cast" "github.com/spf13/viper" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configmodels" @@ -61,22 +63,21 @@ func NewFactory() component.ReceiverFactory { // customUnmarshaler is used to add defaults for named but empty protocols func customUnmarshaler(componentViperSection *viper.Viper, intoCfg interface{}) error { - if componentViperSection == nil || len(componentViperSection.AllKeys()) == 0 { + componentParser := config.ParserFromViper(componentViperSection) + if componentParser == nil || len(componentParser.AllKeys()) == 0 { return fmt.Errorf("empty config for Jaeger receiver") } - componentViperSection.SetConfigType("yaml") - // UnmarshalExact will not set struct properties to nil even if no key is provided, // so set the protocol structs to nil where the keys were omitted. - err := componentViperSection.UnmarshalExact(intoCfg) + err := componentParser.UnmarshalExact(intoCfg) if err != nil { return err } receiverCfg := intoCfg.(*Config) - protocols := componentViperSection.GetStringMap(protocolsFieldName) + protocols := cast.ToStringMap(componentParser.Get(protocolsFieldName)) if len(protocols) == 0 { return fmt.Errorf("must specify at least one protocol when using the Jaeger receiver") } diff --git a/receiver/otlpreceiver/factory.go b/receiver/otlpreceiver/factory.go index 70d504ebec7..9b0afca2f35 100644 --- a/receiver/otlpreceiver/factory.go +++ b/receiver/otlpreceiver/factory.go @@ -18,10 +18,12 @@ import ( "context" "fmt" + "github.com/spf13/cast" "github.com/spf13/viper" "go.uber.org/zap" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/configgrpc" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configmodels" @@ -79,18 +81,19 @@ func createDefaultConfig() configmodels.Receiver { // customUnmarshaler is used to add defaults for named but empty protocols func customUnmarshaler(componentViperSection *viper.Viper, intoCfg interface{}) error { - if componentViperSection == nil || len(componentViperSection.AllKeys()) == 0 { + componentParser := config.ParserFromViper(componentViperSection) + if componentParser == nil || len(componentParser.AllKeys()) == 0 { return fmt.Errorf("empty config for OTLP receiver") } // first load the config normally - err := componentViperSection.UnmarshalExact(intoCfg) + err := componentParser.UnmarshalExact(intoCfg) if err != nil { return err } receiverCfg := intoCfg.(*Config) // next manually search for protocols in viper, if a protocol is not present it means it is disable. - protocols := componentViperSection.GetStringMap(protocolsFieldName) + protocols := cast.ToStringMap(componentParser.Get(protocolsFieldName)) // UnmarshalExact will ignore empty entries like a protocol with no values, so if a typo happened // in the protocol that is intended to be enabled will not be enabled. So check if the protocols diff --git a/receiver/prometheusreceiver/config_test.go b/receiver/prometheusreceiver/config_test.go index b125911eb3f..4a8d73c4d95 100644 --- a/receiver/prometheusreceiver/config_test.go +++ b/receiver/prometheusreceiver/config_test.go @@ -37,7 +37,6 @@ func TestLoadConfig(t *testing.T) { factory := NewFactory() factories.Receivers[typeStr] = factory cfg, err := configtest.LoadConfigFile(t, path.Join(".", "testdata", "config.yaml"), factories) - require.NoError(t, err) require.NotNil(t, cfg) diff --git a/receiver/prometheusreceiver/factory.go b/receiver/prometheusreceiver/factory.go index b0e78fc2b60..3e3f58d5157 100644 --- a/receiver/prometheusreceiver/factory.go +++ b/receiver/prometheusreceiver/factory.go @@ -20,10 +20,12 @@ import ( "fmt" _ "github.com/prometheus/prometheus/discovery/install" // init() of this package registers service discovery impl. + "github.com/spf13/cast" "github.com/spf13/viper" "gopkg.in/yaml.v2" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/config/configmodels" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/receiver/receiverhelper" @@ -52,22 +54,23 @@ func NewFactory() component.ReceiverFactory { } func customUnmarshaler(componentViperSection *viper.Viper, intoCfg interface{}) error { - if componentViperSection == nil { + componentParser := config.ParserFromViper(componentViperSection) + if componentParser == nil { return nil } // We need custom unmarshaling because prometheus "config" subkey defines its own // YAML unmarshaling routines so we need to do it explicitly. - err := componentViperSection.UnmarshalExact(intoCfg) + err := componentParser.UnmarshalExact(intoCfg) if err != nil { return fmt.Errorf("prometheus receiver failed to parse config: %s", err) } // Unmarshal prometheus's config values. Since prometheus uses `yaml` tags, so use `yaml`. - if !componentViperSection.IsSet(prometheusConfigKey) { + promCfgMap := cast.ToStringMap(componentParser.Get(prometheusConfigKey)) + if len(promCfgMap) == 0 { return nil } - promCfgMap := componentViperSection.Sub(prometheusConfigKey).AllSettings() out, err := yaml.Marshal(promCfgMap) if err != nil { return fmt.Errorf("prometheus receiver failed to marshal config to yaml: %s", err)