diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ce37cb3e..0e370a720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Add an option `enable_fetch_replicaset` to control whether to fetch ReplicaSet metadata. The default value is false which aims to release pressure on Kubernetes API server. ([#492](https://github.com/KindlingProject/kindling/pull/492)) ### Bug fixes - +- Fix the bug that the default configs of slice/map are not overridden. ([#497](https://github.com/KindlingProject/kindling/pull/497)) ## v0.7.1 - 2023-03-01 ### New features diff --git a/collector/internal/application/factory.go b/collector/internal/application/factory.go index e21ec8ffe..aa0c92ae9 100644 --- a/collector/internal/application/factory.go +++ b/collector/internal/application/factory.go @@ -1,13 +1,15 @@ package application import ( + "github.com/mitchellh/mapstructure" + "github.com/spf13/viper" + "github.com/Kindling-project/kindling/collector/pkg/component" "github.com/Kindling-project/kindling/collector/pkg/component/analyzer" "github.com/Kindling-project/kindling/collector/pkg/component/consumer" "github.com/Kindling-project/kindling/collector/pkg/component/consumer/exporter" "github.com/Kindling-project/kindling/collector/pkg/component/consumer/processor" "github.com/Kindling-project/kindling/collector/pkg/component/receiver" - "github.com/spf13/viper" ) const ( @@ -103,13 +105,20 @@ func (c *ComponentsFactory) RegisterExporter( } } +// mapStructureDecoderConfigFunc is a function that is used to configure the mapstructure decoder. +// ZeroFields option is set to true to allow the map/slice in the configuration file to override +// the default values in the config struct. +var mapStructureDecoderConfigFunc = func(dc *mapstructure.DecoderConfig) { + dc.ZeroFields = true +} + func (c *ComponentsFactory) ConstructConfig(viper *viper.Viper) error { for _, componentKind := range ComponentsKeyMap { switch componentKind { case ReceiversKey: for k, factory := range c.Receivers { key := ReceiversKey + "." + k - err := viper.UnmarshalKey(key, factory.Config) + err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc) if err != nil { return err } @@ -117,7 +126,7 @@ func (c *ComponentsFactory) ConstructConfig(viper *viper.Viper) error { case AnalyzersKey: for k, factory := range c.Analyzers { key := AnalyzersKey + "." + k - err := viper.UnmarshalKey(key, factory.Config) + err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc) if err != nil { return err } @@ -125,7 +134,7 @@ func (c *ComponentsFactory) ConstructConfig(viper *viper.Viper) error { case ProcessorsKey: for k, factory := range c.Processors { key := ProcessorsKey + "." + k - err := viper.UnmarshalKey(key, factory.Config) + err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc) if err != nil { return err } @@ -133,7 +142,7 @@ func (c *ComponentsFactory) ConstructConfig(viper *viper.Viper) error { case ExportersKey: for k, factory := range c.Exporters { key := ExportersKey + "." + k - err := viper.UnmarshalKey(key, factory.Config) + err := viper.UnmarshalKey(key, factory.Config, mapStructureDecoderConfigFunc) if err != nil { return err } diff --git a/collector/internal/application/factory_test.go b/collector/internal/application/factory_test.go index ebbf57863..a27591046 100644 --- a/collector/internal/application/factory_test.go +++ b/collector/internal/application/factory_test.go @@ -1,28 +1,66 @@ package application import ( - "reflect" "testing" - "github.com/Kindling-project/kindling/collector/pkg/component/consumer/processor/k8sprocessor" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + + "github.com/Kindling-project/kindling/collector/pkg/component/analyzer/network" + "github.com/Kindling-project/kindling/collector/pkg/component/consumer/processor/k8sprocessor" ) func TestConstructConfig(t *testing.T) { factory := NewComponentsFactory() factory.RegisterProcessor(k8sprocessor.K8sMetadata, k8sprocessor.NewKubernetesProcessor, &k8sprocessor.DefaultConfig) + factory.RegisterAnalyzer(network.Network.String(), network.NewNetworkAnalyzer, network.NewDefaultConfig()) + + // Construct the config from the yaml file v := viper.New() - v.SetConfigFile("testdata/kindling-collector-config.yaml") - v.ReadInConfig() - factory.ConstructConfig(v) - k8sprocessorFactory := factory.Processors[k8sprocessor.K8sMetadata] - cfg := k8sprocessorFactory.Config.(*k8sprocessor.Config) + v.SetConfigFile("./testdata/kindling-collector-config.yaml") + err := v.ReadInConfig() + assert.NoError(t, err) + + err = factory.ConstructConfig(v) + assert.NoError(t, err) + + //// Assert the config is as expected + k8sProcessorFactory := factory.Processors[k8sprocessor.K8sMetadata] + k8sCfg := k8sProcessorFactory.Config.(*k8sprocessor.Config) + // The expected config is exactly the opposite of the default config expectedCfg := &k8sprocessor.Config{ - KubeAuthType: "kubeConfig", - KubeConfigDir: "~/.kube/config", - GraceDeletePeriod: 60, + Enable: false, + KubeAuthType: "kubeConfig", + KubeConfigDir: "/opt/.kube/config", + GraceDeletePeriod: 30, + EnableFetchReplicaSet: true, } - if !reflect.DeepEqual(cfg, expectedCfg) { - t.Errorf("Expected %v, but get %v", expectedCfg, cfg) + assert.Equal(t, expectedCfg, k8sCfg) + + networkAnalyzerFactory := factory.Analyzers[network.Network.String()] + networkConfig := networkAnalyzerFactory.Config + expectedNetworkConfig := &network.Config{ + EnableTimeoutCheck: true, + ConnectTimeout: 100, + FdReuseTimeout: 15, + NoResponseThreshold: 120, + ResponseSlowThreshold: 500, + EnableConntrack: true, + ConntrackMaxStateSize: 131072, + ConntrackRateLimit: 500, + ProcRoot: "/proc", + // Case: This slice is from the default config. The config file doesn't have this field. + ProtocolParser: []string{"http", "mysql", "dns", "redis", "kafka", "dubbo"}, + // Case: This slice is overridden by the config file. The default config is different. + ProtocolConfigs: []network.ProtocolConfig{ + { + Key: "http", + Ports: []uint32{80, 8080}, + PayloadLength: 100, + Threshold: 200, + }, + }, + UrlClusteringMethod: "blank", } + assert.Equal(t, expectedNetworkConfig, networkConfig) } diff --git a/collector/internal/application/testdata/kindling-collector-config.yaml b/collector/internal/application/testdata/kindling-collector-config.yaml index a8b951aaa..541535b60 100644 --- a/collector/internal/application/testdata/kindling-collector-config.yaml +++ b/collector/internal/application/testdata/kindling-collector-config.yaml @@ -1,114 +1,31 @@ -receivers: - cgoreceiver: - subscribe: - - name: syscall_exit-writev - category: net - - name: syscall_exit-readv - category: net - - name: syscall_exit-write - category: net - - name: syscall_exit-read - category: net - - name: syscall_exit-sendto - category: net - - name: syscall_exit-recvfrom - category: net - - name: syscall_exit-sendmsg - category: net - - name: syscall_exit-recvmsg - category: net - - name: kprobe-tcp_close - - name: kprobe-tcp_rcv_established - - name: kprobe-tcp_drop - - name: kprobe-tcp_retransmit_skb - - name: syscall_exit-connect - - name: kretprobe-tcp_connect - - name: kprobe-tcp_set_state analyzers: mockanalyzer: num: 10 networkanalyzer: - connect_timeout: 100 - fd_reuse_timeout: 15 - response_slow_threshold: 500 - enable_conntrack: true - conntrack_max_state_size: 131072 - conntrack_rate_limit: 500 - proc_root: /proc - protocol_parser: [ http, mysql, dns, redis, kafka, dubbo ] + # If the destination port of data is one of the followings, the protocol of such network request + # is set to the corresponding one. Note the program will try to identify the protocol automatically + # for the ports that are not in the lists, in which case the cpu usage will be increased much inevitably. protocol_config: - key: "http" - payload_length: 200 - - key: "dubbo" - payload_length: 200 - - key: "mysql" - slow_threshold: 100 - disable_discern: false - - key: "kafka" - slow_threshold: 100 - - key: "cassandra" - ports: [ 9042 ] - slow_threshold: 100 - - key: "s3" - ports: [ 9190 ] - slow_threshold: 100 - - key: "dns" - ports: [ 53 ] - slow_threshold: 100 - + ports: [ 80, 8080 ] + # payload_length indicates the maximum size that payload can be fetched for target protocol + # The trace data sent may contain such payload, so the higher this value, the larger network traffic. + payload_length: 100 + slow_threshold: 200 + url_clustering_method: blank processors: k8smetadataprocessor: + # Set "enable" false if you want to run the agent in the non-Kubernetes environment. + # Otherwise, the agent will panic if it can't connect to the API-server. + enable: false kube_auth_type: kubeConfig - nodemetricprocessor: - -exporters: - otelexporter: - metric_aggregation_map: - kindling_entity_request_total: counter - kindling_entity_request_duration_nanoseconds_total: counter - kindling_entity_request_average_duration_nanoseconds: histogram - kindling_entity_request_send_bytes_total: counter - kindling_entity_request_receive_bytes_total: counter - kindling_topology_request_total: counter - kindling_topology_request_duration_nanoseconds_total: counter - kindling_topology_request_average_duration_nanoseconds: histogram - kindling_topology_request_request_bytes_total: counter - kindling_topology_request_response_bytes_total: counter - kindling_trace_request_duration_nanoseconds: gauge - kindling_tcp_srtt_microseconds: gauge - kindling_tcp_retransmit_total: counter - kindling_tcp_packet_loss_total: counter - export_kind: prometheus - custom_labels: - job: test-hcmine - prometheus: - port: :8080 - otlp: - collect_period: 15s - endpoint: 10.10.10.10:8080 - stdout: - collect_period: 15s - -observability: - logger: - console_level: debug # debug,info,warn,error,none - file_level: debug - file_rotation: - filename: agent.log - maxsize: 512 #MB - maxage: 30 #day - maxbackups: 5 - localtime: true - compress: false - opentelemetry: - # Export data in the following ways: ["prometheus", "otlp", "stdout"] - # Note: configure the corresponding section to make everything ok - export_kind: stdout - prometheus: - port: :9501 - otlp: - collect_period: 15s - # Note: DO NOT add the prefix "http://" - endpoint: 10.10.10.10:8080 - stdout: - collect_period: 15s \ No newline at end of file + kube_config_dir: /opt/.kube/config + # GraceDeletePeriod controls the delay interval after receiving delete event. + # The unit is seconds, and the default value is 60 seconds. + # Should not be lower than 30 seconds. + grace_delete_period: 30 + # enable_fetch_replicaset controls whether to fetch ReplicaSet information. + # The default value is false. It should be enabled if the ReplicaSet + # is used to control pods in the third-party CRD except for Deployment. + enable_fetch_replicaset: true + nodemetricprocessor: \ No newline at end of file diff --git a/collector/pkg/component/controller/controller.go b/collector/pkg/component/controller/controller.go index 7227efde4..33b9f1fa2 100644 --- a/collector/pkg/component/controller/controller.go +++ b/collector/pkg/component/controller/controller.go @@ -4,8 +4,10 @@ import ( "encoding/json" "net/http" - "github.com/Kindling-project/kindling/collector/pkg/component" + "github.com/mitchellh/mapstructure" "github.com/spf13/viper" + + "github.com/Kindling-project/kindling/collector/pkg/component" ) type ControllerAPI interface { @@ -50,7 +52,9 @@ type ControllerConfig struct { func (cf *ControllerFactory) ConstructConfig(viper *viper.Viper, tools *component.TelemetryTools) error { var controllerConfig ControllerConfig key := ControllerComponent - err := viper.UnmarshalKey(key, &controllerConfig) + err := viper.UnmarshalKey(key, &controllerConfig, func(config *mapstructure.DecoderConfig) { + config.ZeroFields = true + }) if err != nil { tools.Logger.Errorf("Error happened when reading controller config, will disable all controller: %v", err) } diff --git a/collector/pkg/component/telemetry.go b/collector/pkg/component/telemetry.go index 39a86acd8..7c3d0da5a 100644 --- a/collector/pkg/component/telemetry.go +++ b/collector/pkg/component/telemetry.go @@ -3,13 +3,15 @@ package component import ( "log" - "github.com/Kindling-project/kindling/collector/pkg/observability" - "github.com/Kindling-project/kindling/collector/pkg/observability/logger" + "github.com/mitchellh/mapstructure" "github.com/spf13/viper" "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric/global" "go.uber.org/zap" "go.uber.org/zap/zapcore" + + "github.com/Kindling-project/kindling/collector/pkg/observability" + "github.com/Kindling-project/kindling/collector/pkg/observability/logger" ) const ( @@ -40,7 +42,9 @@ func (t *TelemetryManager) ConstructConfig(viper *viper.Viper) { func (t *TelemetryManager) initLogger(viper *viper.Viper) { var loggerConfig = logger.Config{} key := ObservabilityConfig + "." + LogKey - err := viper.UnmarshalKey(key, &loggerConfig) + err := viper.UnmarshalKey(key, &loggerConfig, func(config *mapstructure.DecoderConfig) { + config.ZeroFields = true + }) if err != nil { log.Printf("Error happened when reading logger config, and default config will be used: %v", err) } @@ -51,7 +55,9 @@ func (t *TelemetryManager) initLogger(viper *viper.Viper) { func (t *TelemetryManager) initProvider(viper *viper.Viper) { var config = &observability.DefaultConfig key := ObservabilityConfig + "." + MetricKey - err := viper.UnmarshalKey(key, config) + err := viper.UnmarshalKey(key, config, func(config *mapstructure.DecoderConfig) { + config.ZeroFields = true + }) if err != nil { log.Printf("Error happened when reading observability config, and default config will be used: %v", err) }