From f390d7042fdfaeff072a5313242b3b6439632aff Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 27 Dec 2022 11:37:45 -0300 Subject: [PATCH 01/56] feat(maestro): add hc on config. Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder.go | 20 ++++++++++++-------- maestro/redis/consumer/hashset.go | 1 + 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index 3850aed6f..d75c3a124 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -80,7 +80,7 @@ var k8sOtelCollector = ` "containers": [ { "name": "otel-collector", - "image": "otel/opentelemetry-collector-contrib:0.60.0", + "image": "otel/opentelemetry-collector-contrib:0.68.0", "ports": [ { "containerPort": 13133, @@ -210,6 +210,8 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl Extensions: &Extensions{ HealthCheckExtConfig: &HealthCheckExtension{ Endpoint: "0.0.0.0:13133", + CollectorPipeline: &CollectorPipelineExtension{ + Enabled: true, Interval: "5m", FailureThreshold: 5}, }, PProf: &PProfExtension{ Endpoint: "0.0.0.0:1888", // Leaving default for now, will need to change with more processes @@ -244,7 +246,7 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl Exporters []string `json:"exporters" yaml:"exporters"` }{ Receivers: []string{"kafka"}, - Exporters: []string{"prometheusremotewrite"}, + Exporters: []string{"prometheusremotewrite", "logging"}, }, }, }, @@ -291,12 +293,14 @@ type Extensions struct { } type HealthCheckExtension struct { - Endpoint string `json:"endpoint" yaml:"endpoint"` - CollectorPipeline *struct { - Enabled bool `json:"enabled" yaml:"enabled"` - Interval string `json:"interval" yaml:"interval"` - FailureThreshold int32 `json:"exporter_failure_threshold" yaml:"exporter_failure_threshold"` - } `json:"check_collector_pipeline,omitempty" yaml:"check_collector_pipeline,omitempty"` + Endpoint string `json:"endpoint" yaml:"endpoint"` + CollectorPipeline *CollectorPipelineExtension `json:"check_collector_pipeline,omitempty" yaml:"check_collector_pipeline,omitempty"` +} + +type CollectorPipelineExtension struct { + Enabled bool `json:"enabled" yaml:"enabled"` + Interval string `json:"interval" yaml:"interval"` + FailureThreshold int32 `json:"exporter_failure_threshold" yaml:"exporter_failure_threshold"` } type PProfExtension struct { diff --git a/maestro/redis/consumer/hashset.go b/maestro/redis/consumer/hashset.go index fc022afc2..f9cb93179 100644 --- a/maestro/redis/consumer/hashset.go +++ b/maestro/redis/consumer/hashset.go @@ -54,6 +54,7 @@ func (es eventStore) handleSinksCreateCollector(ctx context.Context, event sinks sinkUrl := data.Url sinkUsername := data.User sinkPassword := data.Password + es.logger.Info("DEBUG - ", zap.Any("data", data)) err2 := es.CreateDeploymentEntry(ctx, event.sinkID, sinkUrl, sinkUsername, sinkPassword) if err2 != nil { return err2 From 8d5cbdf57d22074df8a41e603d9890dd10435bca Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 10:04:15 -0300 Subject: [PATCH 02/56] feat(maestro): check true Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index d75c3a124..91d5922c2 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -211,7 +211,7 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl HealthCheckExtConfig: &HealthCheckExtension{ Endpoint: "0.0.0.0:13133", CollectorPipeline: &CollectorPipelineExtension{ - Enabled: true, Interval: "5m", FailureThreshold: 5}, + Enabled: "true", Interval: "5m", FailureThreshold: 5}, }, PProf: &PProfExtension{ Endpoint: "0.0.0.0:1888", // Leaving default for now, will need to change with more processes @@ -298,7 +298,7 @@ type HealthCheckExtension struct { } type CollectorPipelineExtension struct { - Enabled bool `json:"enabled" yaml:"enabled"` + Enabled string `json:"enabled" yaml:"enabled"` Interval string `json:"interval" yaml:"interval"` FailureThreshold int32 `json:"exporter_failure_threshold" yaml:"exporter_failure_threshold"` } From 76f277a55952e1e1daf63c5e5882f0e0800c56a9 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 10:05:44 -0300 Subject: [PATCH 03/56] feat(maestro): remove debug Signed-off-by: Luiz Pegoraro --- maestro/redis/consumer/hashset.go | 1 - 1 file changed, 1 deletion(-) diff --git a/maestro/redis/consumer/hashset.go b/maestro/redis/consumer/hashset.go index f9cb93179..fc022afc2 100644 --- a/maestro/redis/consumer/hashset.go +++ b/maestro/redis/consumer/hashset.go @@ -54,7 +54,6 @@ func (es eventStore) handleSinksCreateCollector(ctx context.Context, event sinks sinkUrl := data.Url sinkUsername := data.User sinkPassword := data.Password - es.logger.Info("DEBUG - ", zap.Any("data", data)) err2 := es.CreateDeploymentEntry(ctx, event.sinkID, sinkUrl, sinkUsername, sinkPassword) if err2 != nil { return err2 From ace20bb4b055d587a88fe9f4265876093311e730 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 12:39:06 -0300 Subject: [PATCH 04/56] feat(maestro): remove logging exporter Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index 91d5922c2..c24e734f7 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -246,7 +246,7 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl Exporters []string `json:"exporters" yaml:"exporters"` }{ Receivers: []string{"kafka"}, - Exporters: []string{"prometheusremotewrite", "logging"}, + Exporters: []string{"prometheusremotewrite"}, }, }, }, From 7a3857dd5c29d8e5917b7e23725fbd44d6b885ba Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 12:39:42 -0300 Subject: [PATCH 05/56] feat(sinker): remove deprecated parameter Signed-off-by: Luiz Pegoraro --- sinker/redis/producer/streams.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/sinker/redis/producer/streams.go b/sinker/redis/producer/streams.go index 3b88e5cc8..2e4af4ae5 100644 --- a/sinker/redis/producer/streams.go +++ b/sinker/redis/producer/streams.go @@ -10,8 +10,7 @@ import ( ) const ( - streamID = "orb.sinker" - streamLen = 1000 + streamID = "orb.sinker" ) var _ config.ConfigRepo = (*eventStore)(nil) @@ -40,9 +39,8 @@ func (e eventStore) Add(config config.SinkConfig) error { Timestamp: time.Now(), } record := &redis.XAddArgs{ - Stream: streamID, - MaxLenApprox: streamLen, - Values: event.Encode(), + Stream: streamID, + Values: event.Encode(), } err = e.client.XAdd(context.Background(), record).Err() if err != nil { @@ -64,9 +62,8 @@ func (e eventStore) Remove(ownerID string, sinkID string) error { Timestamp: time.Now(), } record := &redis.XAddArgs{ - Stream: streamID, - MaxLenApprox: streamLen, - Values: event.Encode(), + Stream: streamID, + Values: event.Encode(), } err = e.client.XAdd(context.Background(), record).Err() if err != nil { @@ -93,9 +90,8 @@ func (e eventStore) Edit(config config.SinkConfig) error { Timestamp: time.Now(), } record := &redis.XAddArgs{ - Stream: streamID, - MaxLenApprox: streamLen, - Values: event.Encode(), + Stream: streamID, + Values: event.Encode(), } err = e.client.XAdd(context.Background(), record).Err() if err != nil { From 183e3cfc86e7dbd482392cdde84bf091b2ad4160 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 12:54:44 -0300 Subject: [PATCH 06/56] feat(maestro): wip monitor of otel collectors Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 maestro/kubecontrol/monitor.go diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go new file mode 100644 index 000000000..fb544313c --- /dev/null +++ b/maestro/kubecontrol/monitor.go @@ -0,0 +1,32 @@ +package kubecontrol + +import ( + "context" + "go.uber.org/zap" + "time" +) + +func (svc *deployService) StartMonitor(ctx context.Context, cancelFunc context.CancelFunc) { + ticker := time.NewTicker(5 * time.Minute) + svc.logger.Debug("start monitor routine", zap.Any("routine", ctx.Value("#routine"))) + defer func() { + cancelFunc() + svc.logger.Debug("stopping monitor routine") + }() + for { + select { + case <-ctx.Done(): + return + case _ = <-ticker.C: + svc.monitorSink() + } + } +} + +func (svc *deployService) monitorSink() { + for sink, up := range svc.deploymentState { + if up { + svc.logger.Info("check sink", zap.Any("sink", sink)) + } + } +} From bf8861ca7160b1c561fdc864ba1b43e24c209abb Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 12:55:32 -0300 Subject: [PATCH 07/56] feat(build): update kind mechanism. Signed-off-by: Luiz Pegoraro --- Makefile | 1 + kind/README.md | 13 +++++++++++++ kind/values.yaml | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b7235982e..341f6eb83 100644 --- a/Makefile +++ b/Makefile @@ -189,6 +189,7 @@ kind-install-orb: kind-upgrade-orb: helm upgrade -n orb kind-orb ./kind + kubectl rollout restart deployment -n orb kind-delete-orb: kubectl delete -f ./kind/nginx.yaml diff --git a/kind/README.md b/kind/README.md index a59c65d72..24c199788 100644 --- a/kind/README.md +++ b/kind/README.md @@ -103,3 +103,16 @@ Have fun! 🎉 When you are done, you can delete the cluster by running: ```shell make kind-delete-cluster ``` + + +## Updating inflight service with recent development + +If you want to change a service, lets say you added some logs to the fleet service, after commiting the changes, add this +```shell +SERVICE=fleet make build_docker +``` +This will build only the docker image of the new service. +After changing you can simply execute +```shell +make kind-upgrade-all +``` diff --git a/kind/values.yaml b/kind/values.yaml index 10ae99083..7b6485b52 100644 --- a/kind/values.yaml +++ b/kind/values.yaml @@ -120,7 +120,7 @@ orb: name: "orb-migrate" pullPolicy: "IfNotPresent" repository: ns1labs - tag: "0.17.0-11889e3a" + tag: "develop" autoMigrate: false logLevel: "debug" From 8fdd42416aeca1bc3e70701ddf358172d7d92d3d Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 15:31:20 -0300 Subject: [PATCH 08/56] feat(maestro): change params --- maestro/config/config_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index c24e734f7..a7477798d 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -211,7 +211,7 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl HealthCheckExtConfig: &HealthCheckExtension{ Endpoint: "0.0.0.0:13133", CollectorPipeline: &CollectorPipelineExtension{ - Enabled: "true", Interval: "5m", FailureThreshold: 5}, + Enabled: "true", Interval: "5m", FailureThreshold: 4}, }, PProf: &PProfExtension{ Endpoint: "0.0.0.0:1888", // Leaving default for now, will need to change with more processes From 1f80dd77a0510f6c545351bff9c0c3484daa65f3 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 28 Dec 2022 15:38:37 -0300 Subject: [PATCH 09/56] feat(maestro): add config. Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index a7477798d..2fca6193c 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -210,6 +210,7 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl Extensions: &Extensions{ HealthCheckExtConfig: &HealthCheckExtension{ Endpoint: "0.0.0.0:13133", + Path: "/health/status", CollectorPipeline: &CollectorPipelineExtension{ Enabled: "true", Interval: "5m", FailureThreshold: 4}, }, @@ -294,6 +295,7 @@ type Extensions struct { type HealthCheckExtension struct { Endpoint string `json:"endpoint" yaml:"endpoint"` + Path string `json:"path" yaml:"path"` CollectorPipeline *CollectorPipelineExtension `json:"check_collector_pipeline,omitempty" yaml:"check_collector_pipeline,omitempty"` } From 3b602a9892e46fafafa60819602d07a05c3a7aa7 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Fri, 30 Dec 2022 10:32:36 -0300 Subject: [PATCH 10/56] fix(gitignore): ignore kind/charts changes --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0345ce02d..ca2b0ffb3 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ python-test/behave_orb test_agent_name* docker/otel-collector-config.yaml !docker/otel-collector-config.yaml.tpl + +kind/charts From b69b5e73f05e6a2129df16a351cfde49b2eb2375 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 3 Jan 2023 17:25:51 -0300 Subject: [PATCH 11/56] feat(maestro): add log collecting routine. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 89 +++++++++++++++++++++--------- maestro/kubecontrol/monitor.go | 87 +++++++++++++++++++++++------ maestro/kubecontrol/sortedset.go | 9 +++ maestro/redis/consumer/events.go | 5 ++ maestro/service.go | 11 ++++ 5 files changed, 157 insertions(+), 44 deletions(-) create mode 100644 maestro/kubecontrol/sortedset.go diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 80ac58eff..c63930f8f 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -34,21 +34,46 @@ type Service interface { // UpdateOtelCollector - update an existing collector by id UpdateOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error + + // CollectLogs - collect logs from the collector by sink-id + CollectLogs(ctx context.Context, sinkID string) ([]string, error) } -func (svc *deployService) collectorDeploy(_ context.Context, operation, sinkId, manifest string) error { +func (svc *deployService) CollectLogs(ctx context.Context, sinkId string) ([]string, error) { + cmd := exec.Command("kubectl", "get logs", fmt.Sprintf("otel-collector-%s", sinkId), "-n", namespace) + exporterLogs := make([]string, 10) + watchLogsFunction := func(out *bufio.Scanner, err *bufio.Scanner) { + if err.Scan() || out.Err() != nil { + svc.logger.Error("failed to get logs for collector on sink") + return + } + for out.Scan() && len(exporterLogs) < 10 { + logEntry := out.Text() + svc.logger.Info("debugging logEntry", zap.String("sinkId", sinkId), zap.String("logEntry", logEntry)) + exporterLogs = append(exporterLogs, logEntry) + } + } + _, _, err := execCmd(ctx, cmd, svc.logger, watchLogsFunction) + if err != nil { + svc.logger.Error("Error reading the logs") + exporterLogs = nil + } + return exporterLogs, err +} + +func (svc *deployService) collectorDeploy(ctx context.Context, operation, sinkId, manifest string) error { fileContent := []byte(manifest) tmp := strings.Split(string(fileContent), "\n") newContent := strings.Join(tmp[1:], "\n") if operation == "apply" { - if value, ok := svc.deploymentState[sinkId]; ok && value { + if value, ok := svc.getDeploymentState(sinkId); ok && value { svc.logger.Info("Already applied Sink ID=" + sinkId) return nil } } else if operation == "delete" { - if value, ok := svc.deploymentState[sinkId]; ok && !value { + if value, ok := svc.getDeploymentState(sinkId); ok && !value { svc.logger.Info("Already deleted Sink ID=" + sinkId) return nil } @@ -60,44 +85,54 @@ func (svc *deployService) collectorDeploy(_ context.Context, operation, sinkId, return err } + stdOutListenFunction := func(out *bufio.Scanner, err *bufio.Scanner) { + for out.Scan() { + fmt.Println(out.Text()) + svc.logger.Info("Deploy Info: " + out.Text()) + } + for err.Scan() { + fmt.Println(err.Text()) + svc.logger.Info("Deploy Error: " + err.Text()) + } + } + // execute action cmd := exec.Command("kubectl", operation, "-f", "/tmp/otel-collector-"+sinkId+".json", "-n", namespace) + _, _, err = execCmd(ctx, cmd, svc.logger, stdOutListenFunction) + + if err == nil { + if operation == "apply" { + svc.deploymentState[sinkId] = true + } else if operation == "delete" { + svc.deploymentState[sinkId] = false + } + } + + return nil +} + +func execCmd(ctx context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc func(stdOut *bufio.Scanner, stdErr *bufio.Scanner)) (*bufio.Scanner, *bufio.Scanner, error) { stdoutReader, _ := cmd.StdoutPipe() stdoutScanner := bufio.NewScanner(stdoutReader) - go func() { - for stdoutScanner.Scan() { - fmt.Println(stdoutScanner.Text()) - svc.logger.Info("Deploy Info: " + stdoutScanner.Text()) - } - }() stderrReader, _ := cmd.StderrPipe() stderrScanner := bufio.NewScanner(stderrReader) - go func() { - for stderrScanner.Scan() { - fmt.Println(stderrScanner.Text()) - svc.logger.Info("Deploy Error: " + stderrScanner.Text()) - } - }() - err = cmd.Start() + go stdOutFunc(stdoutScanner, stderrScanner) + err := cmd.Start() if err != nil { fmt.Printf("Error : %v \n", err) - svc.logger.Error("Collector Deploy Error", zap.Error(err)) + logger.Error("Collector Deploy Error", zap.Error(err)) } err = cmd.Wait() if err != nil { fmt.Printf("Error: %v \n", err) - svc.logger.Error("Collector Deploy Error", zap.Error(err)) - } - - if err == nil { - if operation == "apply" { - svc.deploymentState[sinkId] = true - } else if operation == "delete" { - svc.deploymentState[sinkId] = false - } + logger.Error("Collector Deploy Error", zap.Error(err)) } + return stdoutScanner, stderrScanner, err +} - return nil +func (svc *deployService) getDeploymentState(sinkId string) (bool, bool) { + value, ok := svc.deploymentState[sinkId] + return value, ok } func (svc *deployService) CreateOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error { diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index fb544313c..e14382024 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -2,31 +2,84 @@ package kubecontrol import ( "context" + "encoding/json" + "github.com/go-redis/redis/v8" + "github.com/ns1labs/orb/maestro" "go.uber.org/zap" "time" ) -func (svc *deployService) StartMonitor(ctx context.Context, cancelFunc context.CancelFunc) { - ticker := time.NewTicker(5 * time.Minute) - svc.logger.Debug("start monitor routine", zap.Any("routine", ctx.Value("#routine"))) - defer func() { - cancelFunc() - svc.logger.Debug("stopping monitor routine") +const MonitorFixedDuration = 1 * time.Minute +const TimeDiffForFetchingLogs = 5 * time.Minute + +func NewMonitorService(logger *zap.Logger, redis *redis.Client, kubecontrol *Service) maestro.Service { + return &monitorService{ + logger: logger, + redisClient: redis, + kubecontrol: *kubecontrol, + } +} + +type monitorService struct { + logger *zap.Logger + redisClient *redis.Client + kubecontrol Service +} + +func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelFunc) error { + go func() { + ticker := time.NewTicker(MonitorFixedDuration) + svc.logger.Debug("start monitor routine", zap.Any("routine", ctx.Value("#routine"))) + defer func() { + cancelFunc() + svc.logger.Debug("stopping monitor routine") + }() + for { + select { + case <-ctx.Done(): + cancelFunc() + return + case _ = <-ticker.C: + svc.monitorSinks(ctx) + } + } }() - for { - select { - case <-ctx.Done(): + return nil +} + +func (svc *monitorService) monitorSinks(ctx context.Context) { + queryCmd := svc.redisClient.ZRange(ctx, CollectorStatusKey, time.Now().Add(-TimeDiffForFetchingLogs).Unix(), 0) + if queryCmd.Err() != nil { + svc.logger.Error("error collecting collectors keys", zap.Error(queryCmd.Err())) + return + } + collectorSlice, err := queryCmd.Result() + if err != nil { + svc.logger.Error("error collecting collectors keys", zap.Error(queryCmd.Err())) + return + } + for _, collectorJson := range collectorSlice { + var collector CollectorStatusSortedSetEntry + err = json.Unmarshal([]byte(collectorJson), &collector) + logs, err := svc.kubecontrol.CollectLogs(ctx, collector.SinkId) + if err != nil { return - case _ = <-ticker.C: - svc.monitorSink() } + status, err := analyzeLogs(logs) + if status != collector.Status { + svc.logger.Info("updating status", zap.Any("before", collector), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) + collector.Status = status + if status == "error" { + errorStr := err.Error() + collector.ErrorMessage = &errorStr + } + } + return } + } -func (svc *deployService) monitorSink() { - for sink, up := range svc.deploymentState { - if up { - svc.logger.Info("check sink", zap.Any("sink", sink)) - } - } +// WIP +func analyzeLogs(logEntry []string) (string, error) { + return "active", nil } diff --git a/maestro/kubecontrol/sortedset.go b/maestro/kubecontrol/sortedset.go new file mode 100644 index 000000000..d590f0c5d --- /dev/null +++ b/maestro/kubecontrol/sortedset.go @@ -0,0 +1,9 @@ +package kubecontrol + +const CollectorStatusKey = "orb.sinks.collectors" + +type CollectorStatusSortedSetEntry struct { + SinkId string `json:"sinkId"` + Status string `json:"status"` + ErrorMessage *string `json:"error_message,omitempty"` +} diff --git a/maestro/redis/consumer/events.go b/maestro/redis/consumer/events.go index bf955e3e4..c7a0910bd 100644 --- a/maestro/redis/consumer/events.go +++ b/maestro/redis/consumer/events.go @@ -25,3 +25,8 @@ type sinkerUpdateEvent struct { state string timestamp time.Time } + +type DeploymentEvent struct { + SinkID string + DeploymentYaml string +} diff --git a/maestro/service.go b/maestro/service.go index bba29e8ef..3bb25589c 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -28,6 +28,7 @@ type maestroService struct { serviceCancelFunc context.CancelFunc kubecontrol kubecontrol.Service + monitor Service logger *zap.Logger redisClient *redis.Client sinksClient sinkspb.SinkServiceClient @@ -39,11 +40,13 @@ type maestroService struct { func NewMaestroService(logger *zap.Logger, redisClient *redis.Client, sinksGrpcClient sinkspb.SinkServiceClient, esCfg config.EsConfig, otelCfg config.OtelConfig) Service { kubectr := kubecontrol.NewService(logger) eventStore := rediscons1.NewEventStore(redisClient, otelCfg.KafkaUrl, kubectr, esCfg.Consumer, sinksGrpcClient, logger) + monitor := kubecontrol.NewMonitorService(logger, redisClient, &kubectr) return &maestroService{ logger: logger, redisClient: redisClient, sinksClient: sinksGrpcClient, kubecontrol: kubectr, + monitor: monitor, eventStore: eventStore, kafkaUrl: otelCfg.KafkaUrl, } @@ -104,6 +107,14 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can go svc.subscribeToSinksES(ctx) go svc.subscribeToSinkerES(ctx) + + monitorCtx := context.WithValue(ctx, "routine", "monitor") + err = svc.monitor.Start(monitorCtx, cancelFunction) + if err != nil { + cancelFunction() + return err + } + return nil } From 9b347e64e7f1478454b4d5452e408098fb7c4cc9 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 4 Jan 2023 17:20:57 -0300 Subject: [PATCH 12/56] feat(maestro): add new redis cache usage. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 81 +++++++++++++++++++++++------- maestro/kubecontrol/monitor.go | 14 ++++-- maestro/service.go | 2 +- 3 files changed, 74 insertions(+), 23 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index c63930f8f..70c5c2d83 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -4,9 +4,11 @@ import ( "bufio" "context" "fmt" + "github.com/go-redis/redis/v8" "os" "os/exec" "strings" + "time" "go.uber.org/zap" ) @@ -16,13 +18,12 @@ const namespace = "otelcollectors" var _ Service = (*deployService)(nil) type deployService struct { - logger *zap.Logger - deploymentState map[string]bool + logger *zap.Logger + redisClient *redis.Client } -func NewService(logger *zap.Logger) Service { - deploymentState := make(map[string]bool) - return &deployService{logger: logger, deploymentState: deploymentState} +func NewService(logger *zap.Logger, redisClient *redis.Client) Service { + return &deployService{logger: logger, redisClient: redisClient} } type Service interface { @@ -66,20 +67,23 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, sinkId fileContent := []byte(manifest) tmp := strings.Split(string(fileContent), "\n") newContent := strings.Join(tmp[1:], "\n") - + status, err := svc.getDeploymentState(ctx, sinkId) + if err != nil { + return err + } if operation == "apply" { - if value, ok := svc.getDeploymentState(sinkId); ok && value { - svc.logger.Info("Already applied Sink ID=" + sinkId) + if status != "deleted" { + svc.logger.Info("Already applied Sink ID", zap.String("sinkID", sinkId), zap.String("status", status)) return nil } } else if operation == "delete" { - if value, ok := svc.getDeploymentState(sinkId); ok && !value { - svc.logger.Info("Already deleted Sink ID=" + sinkId) + if status == "deleted" { + svc.logger.Info("Already deleted Sink ID", zap.String("sinkID", sinkId), zap.String("status", status)) return nil } } - err := os.WriteFile("/tmp/otel-collector-"+sinkId+".json", []byte(newContent), 0644) + err = os.WriteFile("/tmp/otel-collector-"+sinkId+".json", []byte(newContent), 0644) if err != nil { svc.logger.Error("failed to write file content", zap.Error(err)) return err @@ -102,16 +106,22 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, sinkId if err == nil { if operation == "apply" { - svc.deploymentState[sinkId] = true + err := svc.setNewDeploymentState(ctx, sinkId, "idle") + if err != nil { + return err + } } else if operation == "delete" { - svc.deploymentState[sinkId] = false + err := svc.setNewDeploymentState(ctx, sinkId, "idle") + if err != nil { + return err + } } } return nil } -func execCmd(ctx context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc func(stdOut *bufio.Scanner, stdErr *bufio.Scanner)) (*bufio.Scanner, *bufio.Scanner, error) { +func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc func(stdOut *bufio.Scanner, stdErr *bufio.Scanner)) (*bufio.Scanner, *bufio.Scanner, error) { stdoutReader, _ := cmd.StdoutPipe() stdoutScanner := bufio.NewScanner(stdoutReader) stderrReader, _ := cmd.StderrPipe() @@ -130,17 +140,52 @@ func execCmd(ctx context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc return stdoutScanner, stderrScanner, err } -func (svc *deployService) getDeploymentState(sinkId string) (bool, bool) { - value, ok := svc.deploymentState[sinkId] - return value, ok +func (svc *deployService) getDeploymentState(ctx context.Context, sinkId string) (string, error) { + key := CollectorStatusKey + "." + sinkId + args := redis.ZRangeArgs{ + Key: key, + Start: nil, + Stop: nil, + ByScore: false, + ByLex: false, + Rev: false, + Offset: 0, + Count: 0, + } + cmd := svc.redisClient.ZRangeArgsWithScores(ctx, args) + slice, err := cmd.Result() + if err != nil { + return "", cmd.Err() + } + svc.logger.Info("debug returned slice", zap.Any("slice", slice)) + value := slice[0] + entry := value.Member.(CollectorStatusSortedSetEntry) + return entry.Status, nil +} + +func (svc *deployService) setNewDeploymentState(ctx context.Context, sinkId, state string) error { + key := CollectorStatusKey + "." + sinkId + entry := redis.Z{ + Score: float64(time.Now().Unix()), + Member: CollectorStatusSortedSetEntry{ + SinkId: sinkId, + Status: "idle", + ErrorMessage: nil, + }, + } + intCmd := svc.redisClient.ZAdd(ctx, key, &entry) + if intCmd.Err() != nil { + return intCmd.Err() + } + return nil } func (svc *deployService) CreateOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error { err := svc.collectorDeploy(ctx, "apply", sinkID, deploymentEntry) - if err != nil { return err } + return nil } diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index e14382024..f0edc4afd 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "github.com/go-redis/redis/v8" - "github.com/ns1labs/orb/maestro" "go.uber.org/zap" "time" ) @@ -12,7 +11,7 @@ import ( const MonitorFixedDuration = 1 * time.Minute const TimeDiffForFetchingLogs = 5 * time.Minute -func NewMonitorService(logger *zap.Logger, redis *redis.Client, kubecontrol *Service) maestro.Service { +func NewMonitorService(logger *zap.Logger, redis *redis.Client, kubecontrol *Service) MonitorService { return &monitorService{ logger: logger, redisClient: redis, @@ -20,6 +19,10 @@ func NewMonitorService(logger *zap.Logger, redis *redis.Client, kubecontrol *Ser } } +type MonitorService interface { + Start(ctx context.Context, cancelFunc context.CancelFunc) error +} + type monitorService struct { logger *zap.Logger redisClient *redis.Client @@ -29,10 +32,10 @@ type monitorService struct { func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelFunc) error { go func() { ticker := time.NewTicker(MonitorFixedDuration) - svc.logger.Debug("start monitor routine", zap.Any("routine", ctx.Value("#routine"))) + svc.logger.Info("start monitor routine", zap.Any("routine", ctx)) defer func() { cancelFunc() - svc.logger.Debug("stopping monitor routine") + svc.logger.Info("stopping monitor routine") }() for { select { @@ -40,6 +43,7 @@ func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelF cancelFunc() return case _ = <-ticker.C: + svc.logger.Info("monitoring sinks") svc.monitorSinks(ctx) } } @@ -58,6 +62,7 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { svc.logger.Error("error collecting collectors keys", zap.Error(queryCmd.Err())) return } + svc.logger.Info("reading logs from collectors", zap.Int("collectors_length", len(collectorSlice))) for _, collectorJson := range collectorSlice { var collector CollectorStatusSortedSetEntry err = json.Unmarshal([]byte(collectorJson), &collector) @@ -81,5 +86,6 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { // WIP func analyzeLogs(logEntry []string) (string, error) { + return "active", nil } diff --git a/maestro/service.go b/maestro/service.go index 3bb25589c..ffffec35d 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -28,7 +28,7 @@ type maestroService struct { serviceCancelFunc context.CancelFunc kubecontrol kubecontrol.Service - monitor Service + monitor kubecontrol.MonitorService logger *zap.Logger redisClient *redis.Client sinksClient sinkspb.SinkServiceClient From 86cc5f1c11638cffa96ccc9028a7734c0cbbe18d Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 4 Jan 2023 17:21:35 -0300 Subject: [PATCH 13/56] feat(maestro): fix kubectr initialization. Signed-off-by: Luiz Pegoraro --- maestro/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/service.go b/maestro/service.go index ffffec35d..7d291d524 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -38,7 +38,7 @@ type maestroService struct { } func NewMaestroService(logger *zap.Logger, redisClient *redis.Client, sinksGrpcClient sinkspb.SinkServiceClient, esCfg config.EsConfig, otelCfg config.OtelConfig) Service { - kubectr := kubecontrol.NewService(logger) + kubectr := kubecontrol.NewService(logger, redisClient) eventStore := rediscons1.NewEventStore(redisClient, otelCfg.KafkaUrl, kubectr, esCfg.Consumer, sinksGrpcClient, logger) monitor := kubecontrol.NewMonitorService(logger, redisClient, &kubectr) return &maestroService{ From abb0b56a1295b8eb845c2e883546364be59c96f3 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 5 Jan 2023 17:35:09 -0300 Subject: [PATCH 14/56] feat(sinks): add ownerID to sinks API. Signed-off-by: Luiz Pegoraro --- sinks/api/grpc/endpoint.go | 1 + sinks/api/grpc/response.go | 1 + sinks/pb/sinks.pb.go | 32 +++++++++++++++++++++----------- sinks/pb/sinks.proto | 1 + sinks/postgres/sinks.go | 8 +++++++- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/sinks/api/grpc/endpoint.go b/sinks/api/grpc/endpoint.go index facf65f8d..336878434 100644 --- a/sinks/api/grpc/endpoint.go +++ b/sinks/api/grpc/endpoint.go @@ -81,6 +81,7 @@ func buildSinkResponse(sink sinks.Sink) (sinkRes, error) { return sinkRes{ id: sink.ID, + mfOwnerId: sink.MFOwnerID, name: sink.Name.String(), description: sink.Description, tags: tagData, diff --git a/sinks/api/grpc/response.go b/sinks/api/grpc/response.go index a3a8d01ea..1aee5e064 100644 --- a/sinks/api/grpc/response.go +++ b/sinks/api/grpc/response.go @@ -10,6 +10,7 @@ package grpc type sinkRes struct { id string + mfOwnerId string name string description string tags []byte diff --git a/sinks/pb/sinks.pb.go b/sinks/pb/sinks.pb.go index 1f3372309..260d124a0 100644 --- a/sinks/pb/sinks.pb.go +++ b/sinks/pb/sinks.pb.go @@ -182,6 +182,7 @@ type SinkRes struct { Error string `protobuf:"bytes,6,opt,name=error,proto3" json:"error,omitempty"` Backend string `protobuf:"bytes,7,opt,name=backend,proto3" json:"backend,omitempty"` Config []byte `protobuf:"bytes,8,opt,name=config,proto3" json:"config,omitempty"` + OwnerID string `protobuf:"bytes,9,opt,name=ownerID,proto3" json:"ownerID,omitempty"` } func (x *SinkRes) Reset() { @@ -272,6 +273,13 @@ func (x *SinkRes) GetConfig() []byte { return nil } +func (x *SinkRes) GetOwnerID() string { + if x != nil { + return x.OwnerID + } + return "" +} + var File_sinks_pb_sinks_proto protoreflect.FileDescriptor var file_sinks_pb_sinks_proto_rawDesc = []byte{ @@ -287,7 +295,7 @@ var file_sinks_pb_sinks_proto_rawDesc = []byte{ 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, - 0x65, 0x72, 0x49, 0x44, 0x22, 0xc1, 0x01, 0x0a, 0x07, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, + 0x65, 0x72, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, @@ -299,16 +307,18 @@ var file_sinks_pb_sinks_proto_rawDesc = []byte{ 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x32, 0x7e, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, - 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, - 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, - 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, - 0x0d, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, - 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, - 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, - 0x73, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x49, 0x44, 0x32, 0x7e, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, + 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, + 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, + 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x69, + 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, + 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, + 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, + 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2f, 0x70, 0x62, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sinks/pb/sinks.proto b/sinks/pb/sinks.proto index 1ecc1ff66..cdf2ba40e 100644 --- a/sinks/pb/sinks.proto +++ b/sinks/pb/sinks.proto @@ -30,4 +30,5 @@ message SinkRes { string error = 6; string backend = 7; bytes config = 8; + string ownerID = 9; } diff --git a/sinks/postgres/sinks.go b/sinks/postgres/sinks.go index 858707f24..acd90d43d 100644 --- a/sinks/postgres/sinks.go +++ b/sinks/postgres/sinks.go @@ -14,6 +14,7 @@ import ( "encoding/json" "fmt" "github.com/gofrs/uuid" + "github.com/jmoiron/sqlx" "github.com/lib/pq" "github.com/ns1labs/orb/pkg/db" "github.com/ns1labs/orb/pkg/errors" @@ -43,7 +44,12 @@ func (s sinksRepository) SearchAllSinks(ctx context.Context, filter sinks.Filter if err != nil { return nil, errors.Wrap(errors.ErrSelectEntity, err) } - defer rows.Close() + defer func(rows *sqlx.Rows) { + err := rows.Close() + if err != nil { + s.logger.Error("error closing rows", zap.Error(err)) + } + }(rows) items := make([]sinks.Sink, 0) for rows.Next() { From a81fd686ea68168834293b7d3efda303bbef77d8 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 5 Jan 2023 18:22:03 -0300 Subject: [PATCH 15/56] feat(maestro): add ownerID and change update to listen to api and update event with error message. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 42 ++++++++++++------------ maestro/kubecontrol/monitor.go | 51 ++++++++++++++++++------------ maestro/kubecontrol/sortedset.go | 3 +- maestro/redis/consumer/events.go | 21 +++++++++--- maestro/redis/consumer/hashset.go | 4 +-- maestro/redis/consumer/streams.go | 32 +++++++++---------- maestro/service.go | 4 +-- 7 files changed, 90 insertions(+), 67 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 70c5c2d83..634ea8f43 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -28,19 +28,19 @@ func NewService(logger *zap.Logger, redisClient *redis.Client) Service { type Service interface { // CreateOtelCollector - create an existing collector by id - CreateOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error + CreateOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error // DeleteOtelCollector - delete an existing collector by id - DeleteOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error + DeleteOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error // UpdateOtelCollector - update an existing collector by id - UpdateOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error + UpdateOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error // CollectLogs - collect logs from the collector by sink-id - CollectLogs(ctx context.Context, sinkID string) ([]string, error) + CollectLogs(ctx context.Context, ownerID, sinkID string) ([]string, error) } -func (svc *deployService) CollectLogs(ctx context.Context, sinkId string) ([]string, error) { +func (svc *deployService) CollectLogs(ctx context.Context, ownerID, sinkId string) ([]string, error) { cmd := exec.Command("kubectl", "get logs", fmt.Sprintf("otel-collector-%s", sinkId), "-n", namespace) exporterLogs := make([]string, 10) watchLogsFunction := func(out *bufio.Scanner, err *bufio.Scanner) { @@ -62,23 +62,23 @@ func (svc *deployService) CollectLogs(ctx context.Context, sinkId string) ([]str return exporterLogs, err } -func (svc *deployService) collectorDeploy(ctx context.Context, operation, sinkId, manifest string) error { +func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerID, sinkId, manifest string) error { fileContent := []byte(manifest) tmp := strings.Split(string(fileContent), "\n") newContent := strings.Join(tmp[1:], "\n") - status, err := svc.getDeploymentState(ctx, sinkId) + status, err := svc.getDeploymentState(ctx, ownerID, sinkId) if err != nil { return err } if operation == "apply" { if status != "deleted" { - svc.logger.Info("Already applied Sink ID", zap.String("sinkID", sinkId), zap.String("status", status)) + svc.logger.Info("Already applied Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", status)) return nil } } else if operation == "delete" { if status == "deleted" { - svc.logger.Info("Already deleted Sink ID", zap.String("sinkID", sinkId), zap.String("status", status)) + svc.logger.Info("Already deleted Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", status)) return nil } } @@ -106,12 +106,12 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, sinkId if err == nil { if operation == "apply" { - err := svc.setNewDeploymentState(ctx, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") if err != nil { return err } } else if operation == "delete" { - err := svc.setNewDeploymentState(ctx, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") if err != nil { return err } @@ -140,7 +140,7 @@ func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc fu return stdoutScanner, stderrScanner, err } -func (svc *deployService) getDeploymentState(ctx context.Context, sinkId string) (string, error) { +func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (string, error) { key := CollectorStatusKey + "." + sinkId args := redis.ZRangeArgs{ Key: key, @@ -163,12 +163,12 @@ func (svc *deployService) getDeploymentState(ctx context.Context, sinkId string) return entry.Status, nil } -func (svc *deployService) setNewDeploymentState(ctx context.Context, sinkId, state string) error { +func (svc *deployService) setNewDeploymentState(ctx context.Context, ownerID, sinkId, state string) error { key := CollectorStatusKey + "." + sinkId entry := redis.Z{ Score: float64(time.Now().Unix()), Member: CollectorStatusSortedSetEntry{ - SinkId: sinkId, + SinkID: sinkId, Status: "idle", ErrorMessage: nil, }, @@ -180,8 +180,8 @@ func (svc *deployService) setNewDeploymentState(ctx context.Context, sinkId, sta return nil } -func (svc *deployService) CreateOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error { - err := svc.collectorDeploy(ctx, "apply", sinkID, deploymentEntry) +func (svc *deployService) CreateOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error { + err := svc.collectorDeploy(ctx, "apply", ownerID, sinkID, deploymentEntry) if err != nil { return err } @@ -189,20 +189,20 @@ func (svc *deployService) CreateOtelCollector(ctx context.Context, sinkID, deplo return nil } -func (svc *deployService) UpdateOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error { - err := svc.DeleteOtelCollector(ctx, sinkID, deploymentEntry) +func (svc *deployService) UpdateOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error { + err := svc.DeleteOtelCollector(ctx, ownerID, sinkID, deploymentEntry) if err != nil { return err } - err = svc.CreateOtelCollector(ctx, sinkID, deploymentEntry) + err = svc.CreateOtelCollector(ctx, ownerID, sinkID, deploymentEntry) if err != nil { return err } return nil } -func (svc *deployService) DeleteOtelCollector(ctx context.Context, sinkID, deploymentEntry string) error { - err := svc.collectorDeploy(ctx, "delete", sinkID, deploymentEntry) +func (svc *deployService) DeleteOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error { + err := svc.collectorDeploy(ctx, "delete", ownerID, sinkID, deploymentEntry) if err != nil { return err } diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index f0edc4afd..c2bb39a35 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -2,19 +2,23 @@ package kubecontrol import ( "context" - "encoding/json" "github.com/go-redis/redis/v8" + rediscons1 "github.com/ns1labs/orb/maestro/redis/consumer" + sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" "time" ) +const streamID = "orb.sinker" + const MonitorFixedDuration = 1 * time.Minute const TimeDiffForFetchingLogs = 5 * time.Minute -func NewMonitorService(logger *zap.Logger, redis *redis.Client, kubecontrol *Service) MonitorService { +func NewMonitorService(logger *zap.Logger, sinksClient *sinkspb.SinkServiceClient, redisClient *redis.Client, kubecontrol *Service) MonitorService { return &monitorService{ logger: logger, - redisClient: redis, + sinksClient: *sinksClient, + redisClient: redisClient, kubecontrol: *kubecontrol, } } @@ -25,6 +29,7 @@ type MonitorService interface { type monitorService struct { logger *zap.Logger + sinksClient sinkspb.SinkServiceClient redisClient *redis.Client kubecontrol Service } @@ -52,31 +57,37 @@ func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelF } func (svc *monitorService) monitorSinks(ctx context.Context) { - queryCmd := svc.redisClient.ZRange(ctx, CollectorStatusKey, time.Now().Add(-TimeDiffForFetchingLogs).Unix(), 0) - if queryCmd.Err() != nil { - svc.logger.Error("error collecting collectors keys", zap.Error(queryCmd.Err())) - return - } - collectorSlice, err := queryCmd.Result() + + sinksRes, err := svc.sinksClient.RetrieveSinks(ctx, &sinkspb.SinksFilterReq{OtelEnabled: "enabled"}) if err != nil { - svc.logger.Error("error collecting collectors keys", zap.Error(queryCmd.Err())) + svc.logger.Error("error collecting collectors keys", zap.Error(err)) return } - svc.logger.Info("reading logs from collectors", zap.Int("collectors_length", len(collectorSlice))) - for _, collectorJson := range collectorSlice { - var collector CollectorStatusSortedSetEntry - err = json.Unmarshal([]byte(collectorJson), &collector) - logs, err := svc.kubecontrol.CollectLogs(ctx, collector.SinkId) + svc.logger.Info("reading logs from collectors", zap.Int("collectors_length", len(sinksRes.Sinks))) + for _, sink := range sinksRes.Sinks { + logs, err := svc.kubecontrol.CollectLogs(ctx, sink.OwnerID, sink.Id) if err != nil { return } status, err := analyzeLogs(logs) - if status != collector.Status { - svc.logger.Info("updating status", zap.Any("before", collector), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) - collector.Status = status + if status != sink.GetState() { + svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) + event := rediscons1.SinkerUpdateEvent{ + SinkID: sink.Id, + Owner: sink.OwnerID, + State: sink.State, + Timestamp: time.Now(), + } if status == "error" { - errorStr := err.Error() - collector.ErrorMessage = &errorStr + event.Msg = err.Error() + } + record := &redis.XAddArgs{ + Stream: streamID, + Values: event.Encode(), + } + err = svc.redisClient.XAdd(context.Background(), record).Err() + if err != nil { + svc.logger.Error("error sending event to event store", zap.Error(err)) } } return diff --git a/maestro/kubecontrol/sortedset.go b/maestro/kubecontrol/sortedset.go index d590f0c5d..bf935f44e 100644 --- a/maestro/kubecontrol/sortedset.go +++ b/maestro/kubecontrol/sortedset.go @@ -3,7 +3,8 @@ package kubecontrol const CollectorStatusKey = "orb.sinks.collectors" type CollectorStatusSortedSetEntry struct { - SinkId string `json:"sinkId"` + OwnerID string `json:"ownerID"` + SinkID string `json:"sinkID"` Status string `json:"status"` ErrorMessage *string `json:"error_message,omitempty"` } diff --git a/maestro/redis/consumer/events.go b/maestro/redis/consumer/events.go index c7a0910bd..bfbf841c2 100644 --- a/maestro/redis/consumer/events.go +++ b/maestro/redis/consumer/events.go @@ -19,11 +19,22 @@ type sinksUpdateEvent struct { timestamp time.Time } -type sinkerUpdateEvent struct { - ownerID string - sinkID string - state string - timestamp time.Time +type SinkerUpdateEvent struct { + SinkID string + Owner string + State string + Msg string + Timestamp time.Time +} + +func (cse SinkerUpdateEvent) Encode() map[string]interface{} { + return map[string]interface{}{ + "sink_id": cse.SinkID, + "owner": cse.Owner, + "state": cse.State, + "msg": cse.Msg, + "timestamp": cse.Timestamp.Unix(), + } } type DeploymentEvent struct { diff --git a/maestro/redis/consumer/hashset.go b/maestro/redis/consumer/hashset.go index fc022afc2..eacd07090 100644 --- a/maestro/redis/consumer/hashset.go +++ b/maestro/redis/consumer/hashset.go @@ -29,7 +29,7 @@ func (es eventStore) handleSinksDeleteCollector(ctx context.Context, event sinks es.logger.Error("did not find collector entry for sink", zap.String("sink-id", event.sinkID)) return err } - err = es.kubecontrol.DeleteOtelCollector(ctx, event.sinkID, deployment) + err = es.kubecontrol.DeleteOtelCollector(ctx, event.owner, event.sinkID, deployment) if err != nil { return err } @@ -96,7 +96,7 @@ func (es eventStore) handleSinksUpdateCollector(ctx context.Context, event sinks return err } es.client.HSet(ctx, deploymentKey, event.sinkID, deploy) - err = es.kubecontrol.UpdateOtelCollector(ctx, event.sinkID, deploy) + err = es.kubecontrol.UpdateOtelCollector(ctx, event.owner, event.sinkID, deploy) if err != nil { return err } diff --git a/maestro/redis/consumer/streams.go b/maestro/redis/consumer/streams.go index 9016b4356..7d0ed2842 100644 --- a/maestro/redis/consumer/streams.go +++ b/maestro/redis/consumer/streams.go @@ -78,9 +78,9 @@ func (es eventStore) SubscribeSinker(context context.Context) error { switch event["operation"] { case sinkerUpdate: rte := decodeSinkerStateUpdate(event) - if rte.state == "idle" { + if rte.State == "idle" { err = es.handleSinkerDeleteCollector(context, rte) //sinker request delete collector - } else if rte.state == "active" { + } else if rte.State == "active" { err = es.handleSinkerCreateCollector(context, rte) //sinker request create collector } } @@ -142,13 +142,13 @@ func (es eventStore) SubscribeSinks(context context.Context) error { } // handleSinkerDeleteCollector Delete collector -func (es eventStore) handleSinkerDeleteCollector(ctx context.Context, event sinkerUpdateEvent) error { - es.logger.Info("Received maestro DELETE event from sinker, sink state=" + event.state + ", , Sink ID=" + event.sinkID + ", Owner ID=" + event.ownerID) - deployment, err := es.GetDeploymentEntryFromSinkId(ctx, event.sinkID) +func (es eventStore) handleSinkerDeleteCollector(ctx context.Context, event SinkerUpdateEvent) error { + es.logger.Info("Received maestro DELETE event from sinker, sink state", zap.String("state", event.State), zap.String("sinkdID", event.SinkID), zap.String("ownerID", event.Owner)) + deployment, err := es.GetDeploymentEntryFromSinkId(ctx, event.SinkID) if err != nil { return err } - err = es.kubecontrol.DeleteOtelCollector(ctx, event.sinkID, deployment) + err = es.kubecontrol.DeleteOtelCollector(ctx, event.Owner, event.SinkID, deployment) if err != nil { return err } @@ -156,25 +156,25 @@ func (es eventStore) handleSinkerDeleteCollector(ctx context.Context, event sink } // handleSinkerCreateCollector Create collector -func (es eventStore) handleSinkerCreateCollector(ctx context.Context, event sinkerUpdateEvent) error { - es.logger.Info("Received maestro CREATE event from sinker, sink state=" + event.state + ", Sink ID=" + event.sinkID + ", Owner ID=" + event.ownerID) - deploymentEntry, err := es.GetDeploymentEntryFromSinkId(ctx, event.sinkID) +func (es eventStore) handleSinkerCreateCollector(ctx context.Context, event SinkerUpdateEvent) error { + es.logger.Info("Received maestro CREATE event from sinker, sink state", zap.String("state", event.State), zap.String("sinkdID", event.SinkID), zap.String("ownerID", event.Owner)) + deploymentEntry, err := es.GetDeploymentEntryFromSinkId(ctx, event.SinkID) if err != nil { return err } - err = es.kubecontrol.CreateOtelCollector(ctx, event.sinkID, deploymentEntry) + err = es.kubecontrol.CreateOtelCollector(ctx, event.Owner, event.SinkID, deploymentEntry) if err != nil { return err } return nil } -func decodeSinkerStateUpdate(event map[string]interface{}) sinkerUpdateEvent { - val := sinkerUpdateEvent{ - ownerID: read(event, "owner", ""), - sinkID: read(event, "sink_id", ""), - state: read(event, "state", ""), - timestamp: time.Time{}, +func decodeSinkerStateUpdate(event map[string]interface{}) SinkerUpdateEvent { + val := SinkerUpdateEvent{ + Owner: read(event, "owner", ""), + SinkID: read(event, "sink_id", ""), + State: read(event, "state", ""), + Timestamp: time.Time{}, } return val } diff --git a/maestro/service.go b/maestro/service.go index 7d291d524..dd0e57734 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -40,7 +40,7 @@ type maestroService struct { func NewMaestroService(logger *zap.Logger, redisClient *redis.Client, sinksGrpcClient sinkspb.SinkServiceClient, esCfg config.EsConfig, otelCfg config.OtelConfig) Service { kubectr := kubecontrol.NewService(logger, redisClient) eventStore := rediscons1.NewEventStore(redisClient, otelCfg.KafkaUrl, kubectr, esCfg.Consumer, sinksGrpcClient, logger) - monitor := kubecontrol.NewMonitorService(logger, redisClient, &kubectr) + monitor := kubecontrol.NewMonitorService(logger, &sinksGrpcClient, redisClient, &kubectr) return &maestroService{ logger: logger, redisClient: redisClient, @@ -96,7 +96,7 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can svc.logger.Warn("failed to fetch deploymentEntry for sink, skipping", zap.String("sink-id", sinkRes.Id)) continue } - err = svc.kubecontrol.CreateOtelCollector(sinkContext, sinkRes.Id, deploymentEntry) + err = svc.kubecontrol.CreateOtelCollector(sinkContext, sinkRes.OwnerID, sinkRes.Id, deploymentEntry) if err != nil { svc.logger.Warn("failed to deploy OtelCollector for sink, skipping", zap.String("sink-id", sinkRes.Id)) continue From c51cd6cc27848c12c3e91c961afef4a13922e610 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 5 Jan 2023 18:29:32 -0300 Subject: [PATCH 16/56] feat(maestro): add stateFilter to api. Signed-off-by: Luiz Pegoraro --- sinks/api/grpc/endpoint.go | 1 + sinks/api/grpc/request.go | 1 + sinks/pb/sinks.pb.go | 67 +++++++++++++++++++++----------------- sinks/pb/sinks.proto | 1 + 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/sinks/api/grpc/endpoint.go b/sinks/api/grpc/endpoint.go index 336878434..466dcc1e8 100644 --- a/sinks/api/grpc/endpoint.go +++ b/sinks/api/grpc/endpoint.go @@ -50,6 +50,7 @@ func retrieveSinksEndpoint(svc sinks.SinkService) endpoint.Endpoint { req := request.(sinksFilter) filter := sinks.Filter{ OpenTelemetry: req.isOtel, + StateFilter: req.state, } sinksInternal, err := svc.ListSinksInternal(ctx, filter) if err != nil { diff --git a/sinks/api/grpc/request.go b/sinks/api/grpc/request.go index 38b031fe3..d2dc11626 100644 --- a/sinks/api/grpc/request.go +++ b/sinks/api/grpc/request.go @@ -19,6 +19,7 @@ type accessByIDReq struct { type sinksFilter struct { isOtel string + state string } func (req accessByIDReq) validate() error { diff --git a/sinks/pb/sinks.pb.go b/sinks/pb/sinks.pb.go index 260d124a0..433430cad 100644 --- a/sinks/pb/sinks.pb.go +++ b/sinks/pb/sinks.pb.go @@ -73,6 +73,7 @@ type SinksFilterReq struct { unknownFields protoimpl.UnknownFields OtelEnabled string `protobuf:"bytes,1,opt,name=otelEnabled,proto3" json:"otelEnabled,omitempty"` + State string `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"` } func (x *SinksFilterReq) Reset() { @@ -114,6 +115,13 @@ func (x *SinksFilterReq) GetOtelEnabled() string { return "" } +func (x *SinksFilterReq) GetState() string { + if x != nil { + return x.State + } + return "" +} + type SinkByIDReq struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -288,37 +296,38 @@ var file_sinks_pb_sinks_proto_rawDesc = []byte{ 0x08, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x22, - 0x32, 0x0a, 0x0e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, + 0x48, 0x0a, 0x0e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x22, 0x3f, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, - 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, - 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, - 0x65, 0x72, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, - 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, - 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, - 0x72, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, - 0x49, 0x44, 0x32, 0x7e, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, - 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, - 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, - 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0d, 0x52, 0x65, 0x74, 0x72, 0x69, - 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, - 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, - 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, - 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2f, 0x70, 0x62, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x3f, 0x0a, 0x0b, 0x53, 0x69, 0x6e, + 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x6b, + 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, + 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x53, + 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x74, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, + 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, + 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x32, 0x7e, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, + 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, + 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, + 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, + 0x0d, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, + 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, + 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, + 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, + 0x73, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sinks/pb/sinks.proto b/sinks/pb/sinks.proto index cdf2ba40e..5286a163f 100644 --- a/sinks/pb/sinks.proto +++ b/sinks/pb/sinks.proto @@ -14,6 +14,7 @@ message SinksRes { message SinksFilterReq { string otelEnabled = 1; + string state = 2; } message SinkByIDReq { From 705fa7493d32743d6ab72bd252f92071ea6648f7 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 5 Jan 2023 19:13:29 -0300 Subject: [PATCH 17/56] feat(maestro): fix cycle dependency. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 2 +- maestro/redis/consumer/hashset.go | 57 +++++++++++++------------- maestro/redis/consumer/streams.go | 9 ++-- maestro/redis/{consumer => }/events.go | 12 +++--- 4 files changed, 41 insertions(+), 39 deletions(-) rename maestro/redis/{consumer => }/events.go (86%) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index c2bb39a35..f1d186705 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -3,7 +3,7 @@ package kubecontrol import ( "context" "github.com/go-redis/redis/v8" - rediscons1 "github.com/ns1labs/orb/maestro/redis/consumer" + rediscons1 "github.com/ns1labs/orb/maestro/redis" sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" "time" diff --git a/maestro/redis/consumer/hashset.go b/maestro/redis/consumer/hashset.go index eacd07090..f3c6bebc7 100644 --- a/maestro/redis/consumer/hashset.go +++ b/maestro/redis/consumer/hashset.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "github.com/ns1labs/orb/maestro/config" + "github.com/ns1labs/orb/maestro/redis" "github.com/ns1labs/orb/pkg/types" sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" @@ -22,30 +23,30 @@ func (es eventStore) GetDeploymentEntryFromSinkId(ctx context.Context, sinkId st } // handleSinksDeleteCollector will delete Deployment Entry and force delete otel collector -func (es eventStore) handleSinksDeleteCollector(ctx context.Context, event sinksUpdateEvent) error { - es.logger.Info("Received maestro DELETE event from sinks ID=" + event.sinkID + ", Owner ID=" + event.owner) - deployment, err := es.GetDeploymentEntryFromSinkId(ctx, event.sinkID) +func (es eventStore) handleSinksDeleteCollector(ctx context.Context, event redis.SinksUpdateEvent) error { + es.logger.Info("Received maestro DELETE event from sinks ID", zap.String("sinkID", event.SinkID), zap.String("owner", event.Owner)) + deployment, err := es.GetDeploymentEntryFromSinkId(ctx, event.SinkID) if err != nil { - es.logger.Error("did not find collector entry for sink", zap.String("sink-id", event.sinkID)) + es.logger.Error("did not find collector entry for sink", zap.String("sink-id", event.SinkID)) return err } - err = es.kubecontrol.DeleteOtelCollector(ctx, event.owner, event.sinkID, deployment) + err = es.kubecontrol.DeleteOtelCollector(ctx, event.Owner, event.SinkID, deployment) if err != nil { return err } - es.client.HDel(ctx, deploymentKey, event.sinkID) + es.client.HDel(ctx, deploymentKey, event.SinkID) return nil } // handleSinksCreateCollector will create Deployment Entry in Redis -func (es eventStore) handleSinksCreateCollector(ctx context.Context, event sinksUpdateEvent) error { - es.logger.Info("Received event to Create DeploymentEntry from sinks ID=" + event.sinkID + ", Owner ID=" + event.owner) +func (es eventStore) handleSinksCreateCollector(ctx context.Context, event redis.SinksUpdateEvent) error { + es.logger.Info("Received event to Create DeploymentEntry from sinks ID", zap.String("sinkID", event.SinkID), zap.String("owner", event.Owner)) sinkData, err := es.sinksClient.RetrieveSink(ctx, &sinkspb.SinkByIDReq{ - SinkID: event.sinkID, - OwnerID: event.owner, + SinkID: event.SinkID, + OwnerID: event.Owner, }) if err != nil { - es.logger.Error("could not fetch info for sink", zap.String("sink-id", event.sinkID), zap.Error(err)) + es.logger.Error("could not fetch info for sink", zap.String("sink-id", event.SinkID), zap.Error(err)) } var data config.SinkData if err := json.Unmarshal(sinkData.Config, &data); err != nil { @@ -54,7 +55,7 @@ func (es eventStore) handleSinksCreateCollector(ctx context.Context, event sinks sinkUrl := data.Url sinkUsername := data.User sinkPassword := data.Password - err2 := es.CreateDeploymentEntry(ctx, event.sinkID, sinkUrl, sinkUsername, sinkPassword) + err2 := es.CreateDeploymentEntry(ctx, event.SinkID, sinkUrl, sinkUsername, sinkPassword) if err2 != nil { return err2 } @@ -74,14 +75,14 @@ func (es eventStore) CreateDeploymentEntry(ctx context.Context, sinkId, sinkUrl, } // handleSinksUpdateCollector will update Deployment Entry in Redis and force update otel collector -func (es eventStore) handleSinksUpdateCollector(ctx context.Context, event sinksUpdateEvent) error { - es.logger.Info("Received event to Update DeploymentEntry from sinks ID=" + event.sinkID + ", Owner ID=" + event.owner) +func (es eventStore) handleSinksUpdateCollector(ctx context.Context, event redis.SinksUpdateEvent) error { + es.logger.Info("Received event to Update DeploymentEntry from sinks ID", zap.String("sinkID", event.SinkID), zap.String("owner", event.Owner)) sinkData, err := es.sinksClient.RetrieveSink(ctx, &sinkspb.SinkByIDReq{ - SinkID: event.sinkID, - OwnerID: event.owner, + SinkID: event.SinkID, + OwnerID: event.Owner, }) if err != nil { - es.logger.Error("could not fetch info for sink", zap.String("sink-id", event.sinkID), zap.Error(err)) + es.logger.Error("could not fetch info for sink", zap.String("sink-id", event.SinkID), zap.Error(err)) } var data config.SinkData if err := json.Unmarshal(sinkData.Config, &data); err != nil { @@ -90,13 +91,13 @@ func (es eventStore) handleSinksUpdateCollector(ctx context.Context, event sinks sinkUrl := data.Url sinkUsername := data.User sinkPassword := data.Password - deploy, err := config.GetDeploymentJson(es.kafkaUrl, event.sinkID, sinkUrl, sinkUsername, sinkPassword) + deploy, err := config.GetDeploymentJson(es.kafkaUrl, event.SinkID, sinkUrl, sinkUsername, sinkPassword) if err != nil { - es.logger.Error("error trying to get deployment json for sink ID", zap.String("sinkId", event.sinkID)) + es.logger.Error("error trying to get deployment json for sink ID", zap.String("sinkId", event.SinkID)) return err } - es.client.HSet(ctx, deploymentKey, event.sinkID, deploy) - err = es.kubecontrol.UpdateOtelCollector(ctx, event.owner, event.sinkID, deploy) + es.client.HSet(ctx, deploymentKey, event.SinkID, deploy) + err = es.kubecontrol.UpdateOtelCollector(ctx, event.Owner, event.SinkID, deploy) if err != nil { return err } @@ -104,18 +105,18 @@ func (es eventStore) handleSinksUpdateCollector(ctx context.Context, event sinks return nil } -func decodeSinksEvent(event map[string]interface{}, operation string) (sinksUpdateEvent, error) { - val := sinksUpdateEvent{ - sinkID: read(event, "sink_id", ""), - owner: read(event, "owner", ""), - timestamp: time.Now(), +func decodeSinksEvent(event map[string]interface{}, operation string) (redis.SinksUpdateEvent, error) { + val := redis.SinksUpdateEvent{ + SinkID: read(event, "sink_id", ""), + Owner: read(event, "owner", ""), + Timestamp: time.Now(), } if operation != sinksDelete { var metadata types.Metadata if err := json.Unmarshal([]byte(read(event, "config", "")), &metadata); err != nil { - return sinksUpdateEvent{}, err + return redis.SinksUpdateEvent{}, err } - val.config = metadata + val.Config = metadata return val, nil } return val, nil diff --git a/maestro/redis/consumer/streams.go b/maestro/redis/consumer/streams.go index 7d0ed2842..34da0d251 100644 --- a/maestro/redis/consumer/streams.go +++ b/maestro/redis/consumer/streams.go @@ -3,6 +3,7 @@ package consumer import ( "context" "github.com/ns1labs/orb/maestro/kubecontrol" + redis2 "github.com/ns1labs/orb/maestro/redis" "github.com/ns1labs/orb/pkg/types" sinkspb "github.com/ns1labs/orb/sinks/pb" "time" @@ -142,7 +143,7 @@ func (es eventStore) SubscribeSinks(context context.Context) error { } // handleSinkerDeleteCollector Delete collector -func (es eventStore) handleSinkerDeleteCollector(ctx context.Context, event SinkerUpdateEvent) error { +func (es eventStore) handleSinkerDeleteCollector(ctx context.Context, event redis2.SinkerUpdateEvent) error { es.logger.Info("Received maestro DELETE event from sinker, sink state", zap.String("state", event.State), zap.String("sinkdID", event.SinkID), zap.String("ownerID", event.Owner)) deployment, err := es.GetDeploymentEntryFromSinkId(ctx, event.SinkID) if err != nil { @@ -156,7 +157,7 @@ func (es eventStore) handleSinkerDeleteCollector(ctx context.Context, event Sink } // handleSinkerCreateCollector Create collector -func (es eventStore) handleSinkerCreateCollector(ctx context.Context, event SinkerUpdateEvent) error { +func (es eventStore) handleSinkerCreateCollector(ctx context.Context, event redis2.SinkerUpdateEvent) error { es.logger.Info("Received maestro CREATE event from sinker, sink state", zap.String("state", event.State), zap.String("sinkdID", event.SinkID), zap.String("ownerID", event.Owner)) deploymentEntry, err := es.GetDeploymentEntryFromSinkId(ctx, event.SinkID) if err != nil { @@ -169,8 +170,8 @@ func (es eventStore) handleSinkerCreateCollector(ctx context.Context, event Sink return nil } -func decodeSinkerStateUpdate(event map[string]interface{}) SinkerUpdateEvent { - val := SinkerUpdateEvent{ +func decodeSinkerStateUpdate(event map[string]interface{}) redis2.SinkerUpdateEvent { + val := redis2.SinkerUpdateEvent{ Owner: read(event, "owner", ""), SinkID: read(event, "sink_id", ""), State: read(event, "state", ""), diff --git a/maestro/redis/consumer/events.go b/maestro/redis/events.go similarity index 86% rename from maestro/redis/consumer/events.go rename to maestro/redis/events.go index bfbf841c2..a66b8a75b 100644 --- a/maestro/redis/consumer/events.go +++ b/maestro/redis/events.go @@ -5,18 +5,18 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -package consumer +package redis import ( "github.com/ns1labs/orb/pkg/types" "time" ) -type sinksUpdateEvent struct { - sinkID string - owner string - config types.Metadata - timestamp time.Time +type SinksUpdateEvent struct { + SinkID string + Owner string + Config types.Metadata + Timestamp time.Time } type SinkerUpdateEvent struct { From 43d9640cd2375a3330f9649223cf3c9416eec473 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 5 Jan 2023 19:21:01 -0300 Subject: [PATCH 18/56] feat(maestro): fix cycle dependency. Signed-off-by: Luiz Pegoraro --- maestro/redis/consumer/streams.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro/redis/consumer/streams.go b/maestro/redis/consumer/streams.go index 34da0d251..b930e77db 100644 --- a/maestro/redis/consumer/streams.go +++ b/maestro/redis/consumer/streams.go @@ -120,12 +120,12 @@ func (es eventStore) SubscribeSinks(context context.Context) error { } switch event["operation"] { case sinksCreate: - if v, ok := rte.config["opentelemetry"]; ok && v.(string) == "enabled" { + if v, ok := rte.Config["opentelemetry"]; ok && v.(string) == "enabled" { err = es.handleSinksCreateCollector(context, rte) //should create collector } case sinksUpdate: - if v, ok := rte.config["opentelemetry"]; ok && v.(string) == "enabled" { + if v, ok := rte.Config["opentelemetry"]; ok && v.(string) == "enabled" { err = es.handleSinksUpdateCollector(context, rte) //should create collector } From 9bac01b318be3ac1942091397b8efd4d15045f7e Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 5 Jan 2023 21:31:35 -0300 Subject: [PATCH 19/56] feat(maestro): wip get logs. Signed-off-by: Luiz Pegoraro --- go.mod | 22 +++++++------- go.sum | 49 +++++++++++++++--------------- maestro/kubecontrol/kubecontrol.go | 4 +-- maestro/kubecontrol/monitor.go | 47 ++++++++++++++++++++++++++-- maestro/service.go | 1 + 5 files changed, 82 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index 73a76a10f..6936dae01 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( go.opentelemetry.io/otel/metric v0.32.1 go.opentelemetry.io/otel/trace v1.10.0 google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de - k8s.io/client-go v0.25.2 + k8s.io/client-go v0.26.0 ) require ( @@ -186,22 +186,22 @@ require ( go.uber.org/goleak v1.2.0 // indirect golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20220927171203-f486391704dc // indirect + golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect - golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/api v0.98.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - k8s.io/api v0.25.2 // indirect - k8s.io/apimachinery v0.25.2 // indirect - k8s.io/klog/v2 v2.70.1 // indirect - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect + k8s.io/api v0.26.0 // indirect + k8s.io/apimachinery v0.26.0 // indirect + k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) @@ -214,7 +214,7 @@ require ( github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect @@ -245,6 +245,6 @@ require ( github.com/xdg-go/stringprep v1.0.3 // indirect go.opentelemetry.io/collector/semconv v0.62.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.1 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect ) diff --git a/go.sum b/go.sum index 01b27555d..fddf09e93 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.4.1 h1:tUSpviiL5G3P9SZZJPC4ZULZJsxQKXxfENpMvdbAXAI= github.com/eclipse/paho.mqtt.golang v1.4.1/go.mod h1:JGt0RsEwEX+Xa/agj90YJ9d9DH2b7upDZMK9HRbFvCA= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -294,7 +294,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= @@ -752,11 +751,11 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.62.0 h1:EoidrEk6Dmap+Cw+lXipNL7IVGicS0N6V+oCvesIj/c= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter v0.62.0/go.mod h1:4BwkK9Fb1xZDxmXt7gSm5nxCxtVWJf61/UaCt54gVjU= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.62.0 h1:PMUgwDspM+2DX2Ol8Tj/jUBQqzvykVwnFily/HjRDPA= @@ -901,7 +900,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= @@ -1160,8 +1158,8 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= -golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1303,11 +1301,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho= -golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1317,8 +1316,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1631,19 +1631,18 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.25.2 h1:v6G8RyFcwf0HR5jQGIAYlvtRNrxMJQG1xJzaSeVnIS8= -k8s.io/api v0.25.2/go.mod h1:qP1Rn4sCVFwx/xIhe+we2cwBLTXNcheRyYXwajonhy0= -k8s.io/apimachinery v0.25.2 h1:WbxfAjCx+AeN8Ilp9joWnyJ6xu9OMeS/fsfjK/5zaQs= -k8s.io/apimachinery v0.25.2/go.mod h1:hqqA1X0bsgsxI6dXsJ4HnNTBOmJNxyPp8dw3u2fSHwA= -k8s.io/client-go v0.25.2 h1:SUPp9p5CwM0yXGQrwYurw9LWz+YtMwhWd0GqOsSiefo= -k8s.io/client-go v0.25.2/go.mod h1:i7cNU7N+yGQmJkewcRD2+Vuj4iz7b30kI8OcL3horQ4= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= +k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= +k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= +k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 634ea8f43..36cc197bf 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -41,11 +41,11 @@ type Service interface { } func (svc *deployService) CollectLogs(ctx context.Context, ownerID, sinkId string) ([]string, error) { - cmd := exec.Command("kubectl", "get logs", fmt.Sprintf("otel-collector-%s", sinkId), "-n", namespace) + cmd := exec.Command("kubectl", "logs", fmt.Sprintf("otel-%s", sinkId), "-n", namespace) exporterLogs := make([]string, 10) watchLogsFunction := func(out *bufio.Scanner, err *bufio.Scanner) { if err.Scan() || out.Err() != nil { - svc.logger.Error("failed to get logs for collector on sink") + svc.logger.Error("failed to get logs for collector on sink", zap.Error(err.Err()), zap.Error(out.Err())) return } for out.Scan() && len(exporterLogs) < 10 { diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index f1d186705..3986887bf 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -1,11 +1,19 @@ package kubecontrol import ( + "bytes" "context" + "fmt" "github.com/go-redis/redis/v8" rediscons1 "github.com/ns1labs/orb/maestro/redis" sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" + "io" + k8scorev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "strings" "time" ) @@ -35,7 +43,7 @@ type monitorService struct { } func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelFunc) error { - go func() { + go func(ctx context.Context, cancelFunc context.CancelFunc) { ticker := time.NewTicker(MonitorFixedDuration) svc.logger.Info("start monitor routine", zap.Any("routine", ctx)) defer func() { @@ -52,10 +60,43 @@ func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelF svc.monitorSinks(ctx) } } - }() + }(ctx, cancelFunc) return nil } +func (svc *monitorService) getPodLogs(ctx context.Context, collectorName string) ([]string, error) { + pod := k8scorev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: collectorName}} + podLogOpts := k8scorev1.PodLogOptions{} + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + podLogs, err := req.Stream(ctx) + if err != nil { + return nil, err + } + defer func(podLogs io.ReadCloser) { + err := podLogs.Close() + if err != nil { + + } + }(podLogs) + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return nil, err + } + str := buf.String() + + return strings.Split(str, "\n"), nil +} + func (svc *monitorService) monitorSinks(ctx context.Context) { sinksRes, err := svc.sinksClient.RetrieveSinks(ctx, &sinkspb.SinksFilterReq{OtelEnabled: "enabled"}) @@ -65,7 +106,7 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { } svc.logger.Info("reading logs from collectors", zap.Int("collectors_length", len(sinksRes.Sinks))) for _, sink := range sinksRes.Sinks { - logs, err := svc.kubecontrol.CollectLogs(ctx, sink.OwnerID, sink.Id) + logs, err := svc.getPodLogs(ctx, fmt.Sprintf("otel-%s", sink.Id)) if err != nil { return } diff --git a/maestro/service.go b/maestro/service.go index dd0e57734..4e291a6e1 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -111,6 +111,7 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can monitorCtx := context.WithValue(ctx, "routine", "monitor") err = svc.monitor.Start(monitorCtx, cancelFunction) if err != nil { + svc.logger.Error("error duriong monitor routine start", zap.Error(err)) cancelFunction() return err } From ebcdff85a371e8c94777893c9a15dd90fb741777 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 9 Jan 2023 10:56:48 -0300 Subject: [PATCH 20/56] feat(maestro): wip get logs. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 3986887bf..200c262bd 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" "io" k8scorev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "strings" @@ -65,7 +65,7 @@ func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelF } func (svc *monitorService) getPodLogs(ctx context.Context, collectorName string) ([]string, error) { - pod := k8scorev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: collectorName}} + pod := k8scorev1.Pod{ObjectMeta: k8smetav1.ObjectMeta{Name: collectorName}} podLogOpts := k8scorev1.PodLogOptions{} config, err := rest.InClusterConfig() if err != nil { From f791bf0ce601d4d03e79baed04a4d770f197a4ac Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 9 Jan 2023 13:52:55 -0300 Subject: [PATCH 21/56] feat(maestro): get logs properly. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 66 +++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 200c262bd..bc02c4b63 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -3,7 +3,6 @@ package kubecontrol import ( "bytes" "context" - "fmt" "github.com/go-redis/redis/v8" rediscons1 "github.com/ns1labs/orb/maestro/redis" sinkspb "github.com/ns1labs/orb/sinks/pb" @@ -64,54 +63,95 @@ func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelF return nil } -func (svc *monitorService) getPodLogs(ctx context.Context, collectorName string) ([]string, error) { - pod := k8scorev1.Pod{ObjectMeta: k8smetav1.ObjectMeta{Name: collectorName}} - podLogOpts := k8scorev1.PodLogOptions{} +func (svc *monitorService) getPodLogs(ctx context.Context, pod k8scorev1.Pod) ([]string, error) { + maxTailLines := int64(10) + podLogOpts := k8scorev1.PodLogOptions{TailLines: &maxTailLines} config, err := rest.InClusterConfig() if err != nil { + svc.logger.Error("error on get cluster config", zap.Error(err)) return nil, err } - clientset, err := kubernetes.NewForConfig(config) + clientSet, err := kubernetes.NewForConfig(config) if err != nil { + svc.logger.Error("error on get client", zap.Error(err)) return nil, err } - req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + svc.logger.Info("DEBUG log pod", zap.Any("pod", pod), zap.Any("clientSet", clientSet)) + req := clientSet.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) podLogs, err := req.Stream(ctx) if err != nil { + svc.logger.Error("error on get logs", zap.Error(err)) return nil, err } defer func(podLogs io.ReadCloser) { err := podLogs.Close() if err != nil { - + svc.logger.Error("error closing log stream", zap.Error(err)) } }(podLogs) buf := new(bytes.Buffer) _, err = io.Copy(buf, podLogs) if err != nil { + svc.logger.Error("error on copying buffer", zap.Error(err)) return nil, err } str := buf.String() + splitLogs := strings.Split(str, "\n") + svc.logger.Info("DEBUG podLogs", zap.Strings("logs", splitLogs)) + return splitLogs, nil +} - return strings.Split(str, "\n"), nil +func (svc *monitorService) getRunningPods(ctx context.Context) ([]k8scorev1.Pod, error) { + config, err := rest.InClusterConfig() + if err != nil { + svc.logger.Error("error on get cluster config", zap.Error(err)) + return nil, err + } + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + svc.logger.Error("error on get client", zap.Error(err)) + return nil, err + } + pods, err := clientSet.CoreV1().Pods(namespace).List(ctx, k8smetav1.ListOptions{}) + return pods.Items, err } func (svc *monitorService) monitorSinks(ctx context.Context) { - + runningCollectors, err := svc.getRunningPods(ctx) + if err != nil { + svc.logger.Error("error getting running pods on namespace", zap.Error(err)) + return + } + if len(runningCollectors) == 0 { + svc.logger.Info("skipping, no running collectors") + return + } sinksRes, err := svc.sinksClient.RetrieveSinks(ctx, &sinkspb.SinksFilterReq{OtelEnabled: "enabled"}) if err != nil { - svc.logger.Error("error collecting collectors keys", zap.Error(err)) + svc.logger.Error("error collecting sinks", zap.Error(err)) return } svc.logger.Info("reading logs from collectors", zap.Int("collectors_length", len(sinksRes.Sinks))) for _, sink := range sinksRes.Sinks { - logs, err := svc.getPodLogs(ctx, fmt.Sprintf("otel-%s", sink.Id)) + var sinkCollector *k8scorev1.Pod + for _, collector := range runningCollectors { + if strings.Contains(collector.Name, sink.Id) { + sinkCollector = &collector + break + } + } + if sinkCollector == nil { + svc.logger.Warn("collector not found for sink, skipping", zap.String("sinkID", sink.Id)) + continue + } + logs, err := svc.getPodLogs(ctx, *sinkCollector) if err != nil { - return + svc.logger.Error("error on getting logs, skipping", zap.Error(err)) + continue } status, err := analyzeLogs(logs) - if status != sink.GetState() { + if status != sink.State { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) event := rediscons1.SinkerUpdateEvent{ SinkID: sink.Id, From fee58543fd7aab53cd2f76f33df22b0e08efd157 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 9 Jan 2023 14:04:24 -0300 Subject: [PATCH 22/56] feat(maestro): fix crash. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index bc02c4b63..c992d5975 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -151,8 +151,12 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { continue } status, err := analyzeLogs(logs) - if status != sink.State { - svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) + if sink.State != status { + if err != nil { + svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) + } else { + svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status)) + } event := rediscons1.SinkerUpdateEvent{ SinkID: sink.Id, Owner: sink.OwnerID, From 689371ad3b0c3f88cf71dce0f9b7c7c9b8c71904 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 9 Jan 2023 16:08:54 -0300 Subject: [PATCH 23/56] feat(maestro): add simple log analysis. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 37 ++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index c992d5975..b7979b010 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -3,8 +3,10 @@ package kubecontrol import ( "bytes" "context" + "encoding/json" "github.com/go-redis/redis/v8" rediscons1 "github.com/ns1labs/orb/maestro/redis" + "github.com/ns1labs/orb/pkg/errors" sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" "io" @@ -76,7 +78,6 @@ func (svc *monitorService) getPodLogs(ctx context.Context, pod k8scorev1.Pod) ([ svc.logger.Error("error on get client", zap.Error(err)) return nil, err } - svc.logger.Info("DEBUG log pod", zap.Any("pod", pod), zap.Any("clientSet", clientSet)) req := clientSet.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) podLogs, err := req.Stream(ctx) if err != nil { @@ -98,7 +99,7 @@ func (svc *monitorService) getPodLogs(ctx context.Context, pod k8scorev1.Pod) ([ } str := buf.String() splitLogs := strings.Split(str, "\n") - svc.logger.Info("DEBUG podLogs", zap.Strings("logs", splitLogs)) + svc.logger.Info("logs length", zap.Int("amount line logs", len(splitLogs))) return splitLogs, nil } @@ -150,7 +151,10 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { svc.logger.Error("error on getting logs, skipping", zap.Error(err)) continue } - status, err := analyzeLogs(logs) + status, err := svc.analyzeLogs(logs) + if status == "fail" { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } if sink.State != status { if err != nil { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) @@ -180,8 +184,29 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { } -// WIP -func analyzeLogs(logEntry []string) (string, error) { - +// analyzeLogs, will check for errors in exporter, and will return as follow +// +// for errors 479 will send a "warning" state, plus message of too many requests +// for any other errors, will add error and message +// if no error message on exporter, will log as active +func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err error) { + for _, logLine := range logEntry { + if strings.Contains(logLine, "error") { + errStringLog := strings.TrimRight(logLine, "error") + jsonError := strings.Split(errStringLog, "\t")[4] + errorJson := make(map[string]interface{}) + err := json.Unmarshal([]byte(jsonError), &errorJson) + if err != nil { + return "fail", err + } + svc.logger.Info("DEBUG -", zap.Any("errorJson", errorJson)) + if errorJson != nil && errorJson["error"] != nil { + errorMessage := errorJson["error"].(string) + if strings.Contains(errorMessage, "429") { + return "warn", errors.New(errorMessage) + } + } + } + } return "active", nil } From 18fd15d0caee90b7290cafaffa2bd6d18f646177 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 9 Jan 2023 16:54:20 -0300 Subject: [PATCH 24/56] feat(maestro): add warning and notify change of state. Signed-off-by: Luiz Pegoraro --- maestro/config/types.go | 7 ++++-- maestro/kubecontrol/monitor.go | 46 ++++++++++++++++++++++------------ sinker/config/types.go | 3 +++ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/maestro/config/types.go b/maestro/config/types.go index 1f7482f49..2f32fe37f 100644 --- a/maestro/config/types.go +++ b/maestro/config/types.go @@ -22,6 +22,7 @@ const ( Active Error Idle + Warning ) type PrometheusState int @@ -31,6 +32,7 @@ var promStateMap = [...]string{ "active", "error", "idle", + "warning", } var promStateRevMap = map[string]PrometheusState{ @@ -38,14 +40,15 @@ var promStateRevMap = map[string]PrometheusState{ "active": Active, "error": Error, "idle": Idle, + "warning": Warning, } func (p PrometheusState) String() string { return promStateMap[p] } -func (p *PrometheusState) Scan(value interface{}) error { - *p = promStateRevMap[string(value.([]byte))] +func (p *PrometheusState) SetFromString(value string) error { + *p = promStateRevMap[value] return nil } diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index b7979b010..2c9a09535 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -4,8 +4,9 @@ import ( "bytes" "context" "encoding/json" + "fmt" "github.com/go-redis/redis/v8" - rediscons1 "github.com/ns1labs/orb/maestro/redis" + maestroconfig "github.com/ns1labs/orb/maestro/config" "github.com/ns1labs/orb/pkg/errors" sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" @@ -129,6 +130,7 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { return } sinksRes, err := svc.sinksClient.RetrieveSinks(ctx, &sinkspb.SinksFilterReq{OtelEnabled: "enabled"}) + if err != nil { svc.logger.Error("error collecting sinks", zap.Error(err)) return @@ -146,35 +148,45 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { svc.logger.Warn("collector not found for sink, skipping", zap.String("sinkID", sink.Id)) continue } + var data maestroconfig.SinkData + if err := json.Unmarshal(sink.Config, &data); err != nil { + svc.logger.Warn("failed to unmarshal sink, skipping", zap.String("sink-id", sink.Id)) + continue + } logs, err := svc.getPodLogs(ctx, *sinkCollector) if err != nil { svc.logger.Error("error on getting logs, skipping", zap.Error(err)) continue } - status, err := svc.analyzeLogs(logs) + status, logsErr := svc.analyzeLogs(logs) if status == "fail" { - svc.logger.Error("error during analyze logs", zap.Error(err)) + svc.logger.Error("error during analyze logs", zap.Error(logsErr)) } - if sink.State != status { + if data.State.String() != status { if err != nil { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status), zap.String("error_message (opt)", err.Error())) } else { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status)) } - event := rediscons1.SinkerUpdateEvent{ - SinkID: sink.Id, - Owner: sink.OwnerID, - State: sink.State, - Timestamp: time.Now(), + keyPrefix := "sinker_key" + skey := fmt.Sprintf("%s-%s:%s", keyPrefix, data.OwnerID, data.SinkID) + err := data.State.SetFromString(status) + if err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) } - if status == "error" { - event.Msg = err.Error() + if logsErr != nil { + data.Msg = logsErr.Error() + } + if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + byteSink, err := json.Marshal(data) + if err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) } - record := &redis.XAddArgs{ - Stream: streamID, - Values: event.Encode(), + if err = svc.redisClient.Set(context.Background(), skey, byteSink, 0).Err(); err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) } - err = svc.redisClient.XAdd(context.Background(), record).Err() if err != nil { svc.logger.Error("error sending event to event store", zap.Error(err)) } @@ -203,7 +215,9 @@ func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err er if errorJson != nil && errorJson["error"] != nil { errorMessage := errorJson["error"].(string) if strings.Contains(errorMessage, "429") { - return "warn", errors.New(errorMessage) + return "warning", errors.New(errorMessage) + } else { + return "error", errors.New(errorMessage) } } } diff --git a/sinker/config/types.go b/sinker/config/types.go index 7522a46a2..122a3ced3 100644 --- a/sinker/config/types.go +++ b/sinker/config/types.go @@ -26,6 +26,7 @@ const ( Active Error Idle + Warning ) type PrometheusState int @@ -35,6 +36,7 @@ var promStateMap = [...]string{ "active", "error", "idle", + "warning", } var promStateRevMap = map[string]PrometheusState{ @@ -42,6 +44,7 @@ var promStateRevMap = map[string]PrometheusState{ "active": Active, "error": Error, "idle": Idle, + "warning": Warning, } func (p PrometheusState) String() string { From 8e7fedffc2a47a05cf4d8c9c6596d32bad97233b Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 9 Jan 2023 17:26:33 -0300 Subject: [PATCH 25/56] feat(maestro): add missing data. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 2c9a09535..47abc8c5c 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -153,6 +153,9 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { svc.logger.Warn("failed to unmarshal sink, skipping", zap.String("sink-id", sink.Id)) continue } + data.SinkID = sink.Id + data.OwnerID = sink.OwnerID + data.LastRemoteWrite = time.Now() logs, err := svc.getPodLogs(ctx, *sinkCollector) if err != nil { svc.logger.Error("error on getting logs, skipping", zap.Error(err)) @@ -187,9 +190,6 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { if err = svc.redisClient.Set(context.Background(), skey, byteSink, 0).Err(); err != nil { svc.logger.Error("error during analyze logs", zap.Error(err)) } - if err != nil { - svc.logger.Error("error sending event to event store", zap.Error(err)) - } } return } From f299e2f8a36d79cc05e3e1f8c12d6cf5a528f5d6 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 9 Jan 2023 18:52:46 -0300 Subject: [PATCH 26/56] feat(maestro): integrate maestro and sinks, and prepare maestro to fetch sinker redis pod. Signed-off-by: Luiz Pegoraro --- cmd/maestro/main.go | 50 ++++++++++++++++++++++++++++++---- maestro/kubecontrol/monitor.go | 1 + maestro/service.go | 40 ++++++++++++++------------- sinks/api/grpc/client.go | 1 + 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/cmd/maestro/main.go b/cmd/maestro/main.go index 9c195a616..745877c23 100644 --- a/cmd/maestro/main.go +++ b/cmd/maestro/main.go @@ -13,6 +13,7 @@ import ( "fmt" sinksgrpc "github.com/ns1labs/orb/sinks/api/grpc" "github.com/opentracing/opentracing-go" + "github.com/spf13/viper" jconfig "github.com/uber/jaeger-client-go/config" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -41,7 +42,8 @@ const ( func main() { - esCfg := config.LoadEsConfig(envPrefix) + streamEsCfg := loadStreamEsConfig(envPrefix) + sinkerEsCfg := loadSinkerEsConfig(envPrefix) svcCfg := config.LoadBaseServiceConfig(envPrefix, httpPort) jCfg := config.LoadJaegerConfig(envPrefix) sinksGRPCCfg := config.LoadGRPCConfig("orb", "sinks") @@ -71,14 +73,20 @@ func main() { _ = logger.Sync() }(logger) log := logger.Sugar() - esClient := connectToRedis(esCfg.URL, esCfg.Pass, esCfg.DB, logger) + streamEsClient := connectToRedis(streamEsCfg.URL, streamEsCfg.Pass, streamEsCfg.DB, logger) defer func(esClient *r.Client) { err := esClient.Close() if err != nil { return } - }(esClient) - + }(streamEsClient) + sinkerEsClient := connectToRedis(sinkerEsCfg.URL, sinkerEsCfg.Pass, sinkerEsCfg.DB, logger) + defer func(esClient *r.Client) { + err := esClient.Close() + if err != nil { + return + } + }(sinkerEsClient) tracer, tracerCloser := initJaeger(svcName, jCfg.URL, logger) defer func(tracerCloser io.Closer) { err := tracerCloser.Close() @@ -102,7 +110,7 @@ func main() { sinksGRPCClient := sinksgrpc.NewClient(tracer, sinksGRPCConn, sinksGRPCTimeout, logger) otelCfg := config.LoadOtelConfig(envPrefix) - svc := maestro.NewMaestroService(logger, esClient, sinksGRPCClient, esCfg, otelCfg) + svc := maestro.NewMaestroService(logger, streamEsClient, sinkerEsClient, sinksGRPCClient, streamEsCfg, otelCfg) errs := make(chan error, 2) mainContext, mainCancelFunction := context.WithCancel(context.Background()) @@ -189,3 +197,35 @@ func connectToRedis(redisURL, redisPass, redisDB string, logger *zap.Logger) *r. DB: db, }) } + +func loadStreamEsConfig(prefix string) config.EsConfig { + cfg := viper.New() + cfg.SetEnvPrefix(fmt.Sprintf("%s_stream_es", prefix)) + + cfg.SetDefault("url", "localhost:6379") + cfg.SetDefault("pass", "") + cfg.SetDefault("db", "0") + cfg.SetDefault("consumer", fmt.Sprintf("%s-es-consumer", prefix)) + + cfg.AllowEmptyEnv(true) + cfg.AutomaticEnv() + var esC config.EsConfig + cfg.Unmarshal(&esC) + return esC +} + +func loadSinkerEsConfig(prefix string) config.EsConfig { + cfg := viper.New() + cfg.SetEnvPrefix(fmt.Sprintf("%s_sinker_es", prefix)) + + cfg.SetDefault("url", "localhost:6378") + cfg.SetDefault("pass", "") + cfg.SetDefault("db", "0") + cfg.SetDefault("consumer", fmt.Sprintf("%s-sinker-es-consumer", prefix)) + + cfg.AllowEmptyEnv(true) + cfg.AutomaticEnv() + var esC config.EsConfig + cfg.Unmarshal(&esC) + return esC +} diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 47abc8c5c..c30ae55dd 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -156,6 +156,7 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { data.SinkID = sink.Id data.OwnerID = sink.OwnerID data.LastRemoteWrite = time.Now() + svc.logger.Info("DEBUG DATA", zap.Any("data", data)) logs, err := svc.getPodLogs(ctx, *sinkCollector) if err != nil { svc.logger.Error("error on getting logs, skipping", zap.Error(err)) diff --git a/maestro/service.go b/maestro/service.go index 4e291a6e1..6c2aead3a 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -27,28 +27,30 @@ type maestroService struct { serviceContext context.Context serviceCancelFunc context.CancelFunc - kubecontrol kubecontrol.Service - monitor kubecontrol.MonitorService - logger *zap.Logger - redisClient *redis.Client - sinksClient sinkspb.SinkServiceClient - esCfg config.EsConfig - eventStore rediscons1.Subscriber - kafkaUrl string + kubecontrol kubecontrol.Service + monitor kubecontrol.MonitorService + logger *zap.Logger + streamRedisClient *redis.Client + sinkerRedisClient *redis.Client + sinksClient sinkspb.SinkServiceClient + esCfg config.EsConfig + eventStore rediscons1.Subscriber + kafkaUrl string } -func NewMaestroService(logger *zap.Logger, redisClient *redis.Client, sinksGrpcClient sinkspb.SinkServiceClient, esCfg config.EsConfig, otelCfg config.OtelConfig) Service { - kubectr := kubecontrol.NewService(logger, redisClient) - eventStore := rediscons1.NewEventStore(redisClient, otelCfg.KafkaUrl, kubectr, esCfg.Consumer, sinksGrpcClient, logger) - monitor := kubecontrol.NewMonitorService(logger, &sinksGrpcClient, redisClient, &kubectr) +func NewMaestroService(logger *zap.Logger, streamRedisClient *redis.Client, sinkerRedisClient *redis.Client, sinksGrpcClient sinkspb.SinkServiceClient, esCfg config.EsConfig, otelCfg config.OtelConfig) Service { + kubectr := kubecontrol.NewService(logger, streamRedisClient) + eventStore := rediscons1.NewEventStore(streamRedisClient, otelCfg.KafkaUrl, kubectr, esCfg.Consumer, sinksGrpcClient, logger) + monitor := kubecontrol.NewMonitorService(logger, &sinksGrpcClient, sinkerRedisClient, &kubectr) return &maestroService{ - logger: logger, - redisClient: redisClient, - sinksClient: sinksGrpcClient, - kubecontrol: kubectr, - monitor: monitor, - eventStore: eventStore, - kafkaUrl: otelCfg.KafkaUrl, + logger: logger, + streamRedisClient: streamRedisClient, + sinkerRedisClient: sinkerRedisClient, + sinksClient: sinksGrpcClient, + kubecontrol: kubectr, + monitor: monitor, + eventStore: eventStore, + kafkaUrl: otelCfg.KafkaUrl, } } diff --git a/sinks/api/grpc/client.go b/sinks/api/grpc/client.go index 9db58dfec..059605996 100644 --- a/sinks/api/grpc/client.go +++ b/sinks/api/grpc/client.go @@ -47,6 +47,7 @@ func (client grpcClient) RetrieveSinks(ctx context.Context, in *pb.SinksFilterRe for i, sinkResponse := range ir.sinks { sinkList[i] = &pb.SinkRes{ Id: sinkResponse.id, + OwnerID: sinkResponse.mfOwnerId, Name: sinkResponse.name, Description: sinkResponse.description, Tags: sinkResponse.tags, From e455804fd02bc19927e21f547dbe91f0bb22a99e Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 11 Jan 2023 17:11:46 -0300 Subject: [PATCH 27/56] feat(maestro): add logging exporter, and analyse metrics as active or error. Signed-off-by: Luiz Pegoraro --- kind/Chart.yaml | 2 +- maestro/config/config_builder.go | 12 ++++++++++++ maestro/kubecontrol/monitor.go | 19 ++++++++++++++----- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/kind/Chart.yaml b/kind/Chart.yaml index 92074223f..bc15d3f64 100644 --- a/kind/Chart.yaml +++ b/kind/Chart.yaml @@ -17,5 +17,5 @@ appVersion: "1.0.0" dependencies: - name: orb - version: "1.0.39" + version: "1.0.40" repository: "@ns1labs-orb" diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index 2fca6193c..05ca0d05c 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -231,6 +231,11 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl Authenticator string `json:"authenticator" yaml:"authenticator"` }{Authenticator: "basicauth/exporter"}, }, + LoggingExporter: &LoggingExporterConfig{ + Verbosity: "detailed", + SamplingInitial: 5, + SamplingThereAfter: 50, + }, }, Service: ServiceConfig{ Extensions: []string{"pprof", "health_check", "basicauth/exporter"}, @@ -322,6 +327,13 @@ type BasicAuthenticationExtension struct { type Exporters struct { PrometheusRemoteWrite *PrometheusRemoteWriteExporterConfig `json:"prometheusremotewrite,omitempty" yaml:"prometheusremotewrite,omitempty"` + LoggingExporter *LoggingExporterConfig `json:"logging,omitempty" yaml:"logging,omitempty"` +} + +type LoggingExporterConfig struct { + Verbosity string `json:"verbosity,omitempty" yaml:"verbosity,omitempty"` + SamplingInitial int `json:"sampling_initial,omitempty" yaml:"verbosity,omitempty"` + SamplingThereAfter int `json:"sampling_thereafter,omitempty" yaml:"verbosity,omitempty"` } type PrometheusRemoteWriteExporterConfig struct { diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index c30ae55dd..443f9e0f5 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -19,10 +19,8 @@ import ( "time" ) -const streamID = "orb.sinker" - const MonitorFixedDuration = 1 * time.Minute -const TimeDiffForFetchingLogs = 5 * time.Minute +const TimeDiffActiveIdle = 5 * time.Minute func NewMonitorService(logger *zap.Logger, sinksClient *sinkspb.SinkServiceClient, redisClient *redis.Client, kubecontrol *Service) MonitorService { return &monitorService{ @@ -197,13 +195,16 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { } -// analyzeLogs, will check for errors in exporter, and will return as follow +// analyzeLogs, will check for errors in exporter, and will return as follows // +// for active, the timestamp should not be longer than 5 minutes of the last metric export // for errors 479 will send a "warning" state, plus message of too many requests // for any other errors, will add error and message // if no error message on exporter, will log as active func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err error) { + var lastTimeStamp string for _, logLine := range logEntry { + lastTimeStamp = logLine[0:24] if strings.Contains(logLine, "error") { errStringLog := strings.TrimRight(logLine, "error") jsonError := strings.Split(errStringLog, "\t")[4] @@ -223,5 +224,13 @@ func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err er } } } - return "active", nil + lastLogTime, err := time.Parse(time.RFC3339, lastTimeStamp) + if err != nil { + return "fail", err + } + if lastLogTime.After(time.Now().Add(-TimeDiffActiveIdle)) { + return "active", nil + } else { + return "idle", nil + } } From 45e68a65d62073f09605973f78105c69df2d61b2 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 11 Jan 2023 17:32:49 -0300 Subject: [PATCH 28/56] feat(maestro): fix config. Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index 05ca0d05c..5a2959d33 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -332,8 +332,8 @@ type Exporters struct { type LoggingExporterConfig struct { Verbosity string `json:"verbosity,omitempty" yaml:"verbosity,omitempty"` - SamplingInitial int `json:"sampling_initial,omitempty" yaml:"verbosity,omitempty"` - SamplingThereAfter int `json:"sampling_thereafter,omitempty" yaml:"verbosity,omitempty"` + SamplingInitial int `json:"sampling_initial,omitempty" yaml:"sampling_initial,omitempty"` + SamplingThereAfter int `json:"sampling_thereafter,omitempty" yaml:"sampling_thereafter,omitempty"` } type PrometheusRemoteWriteExporterConfig struct { From 03734c7b412b349e637075584f368e3a755d5fec Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 11 Jan 2023 17:58:19 -0300 Subject: [PATCH 29/56] feat(maestro): remove debug logs. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 443f9e0f5..ff103274b 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -154,7 +154,6 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { data.SinkID = sink.Id data.OwnerID = sink.OwnerID data.LastRemoteWrite = time.Now() - svc.logger.Info("DEBUG DATA", zap.Any("data", data)) logs, err := svc.getPodLogs(ctx, *sinkCollector) if err != nil { svc.logger.Error("error on getting logs, skipping", zap.Error(err)) @@ -213,7 +212,6 @@ func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err er if err != nil { return "fail", err } - svc.logger.Info("DEBUG -", zap.Any("errorJson", errorJson)) if errorJson != nil && errorJson["error"] != nil { errorMessage := errorJson["error"].(string) if strings.Contains(errorMessage, "429") { From ec0f2e7e82f76be382dceca51287688d48b9da85 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 11 Jan 2023 17:59:35 -0300 Subject: [PATCH 30/56] feat(maestro): fix time. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index ff103274b..c1ce6fb81 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -227,8 +227,8 @@ func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err er return "fail", err } if lastLogTime.After(time.Now().Add(-TimeDiffActiveIdle)) { - return "active", nil - } else { return "idle", nil + } else { + return "active", nil } } From 5ff2871d420b5c18e354af9a3629871d045d7bd9 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 11 Jan 2023 18:13:39 -0300 Subject: [PATCH 31/56] feat(maestro): fix test. Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/config/config_builder_test.go b/maestro/config/config_builder_test.go index 81e381daa..2ba091ae2 100644 --- a/maestro/config/config_builder_test.go +++ b/maestro/config/config_builder_test.go @@ -27,7 +27,7 @@ func TestReturnConfigYamlFromSink(t *testing.T) { sinkUrl: "https://mysinkurl:9922", sinkUsername: "1234123", sinkPassword: "CarnivorousVulgaris", - }, want: `---\nreceivers:\n kafka:\n brokers:\n - kafka:9092\n topic: otlp_metrics-sink-id-222\n protocol_version: 2.0.0\nextensions:\n health_check:\n endpoint: 0.0.0.0:13133\n pprof:\n endpoint: 0.0.0.0:1888\n basicauth/exporter:\n client_auth:\n username: 1234123\n password: CarnivorousVulgaris\nexporters:\n prometheusremotewrite:\n endpoint: https://mysinkurl:9922\n auth:\n authenticator: basicauth/exporter\nservice:\n extensions:\n - pprof\n - health_check\n - basicauth/exporter\n pipelines:\n metrics:\n receivers:\n - kafka\n exporters:\n - prometheusremotewrite\n`, + }, want: `---\nreceivers:\n kafka:\n brokers:\n - kafka:9092\n topic: otlp_metrics-sink-id-222\n protocol_version: 2.0.0\nextensions:\n health_check:\n endpoint: 0.0.0.0:13133\n path: /health/status\n check_collector_pipeline:\n enabled: true\n interval: 5m\n exporter_failure_threshold: 4\n pprof:\n endpoint: 0.0.0.0:1888\n basicauth/exporter:\n client_auth:\n username: 1234123\n password: CarnivorousVulgaris\nexporters:\n prometheusremotewrite:\n endpoint: https://mysinkurl:9922\n auth:\n authenticator: basicauth/exporter\n logging:\n verbosity: detailed\n sampling_initial: 5\n sampling_thereafter: 50\nservice:\n extensions:\n - pprof\n - health_check\n - basicauth/exporter\n pipelines:\n metrics:\n receivers:\n - kafka\n exporters:\n - prometheusremotewrite\n`, wantErr: false}, } for _, tt := range tests { From 1d6c4832ce68a8e4d207bb1492adc8743e0f4745 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 12 Jan 2023 08:21:01 -0300 Subject: [PATCH 32/56] feat(maestro): code review adjustments. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 6 ------ maestro/kubecontrol/monitor.go | 11 +++++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 36cc197bf..b877dda76 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -50,7 +50,6 @@ func (svc *deployService) CollectLogs(ctx context.Context, ownerID, sinkId strin } for out.Scan() && len(exporterLogs) < 10 { logEntry := out.Text() - svc.logger.Info("debugging logEntry", zap.String("sinkId", sinkId), zap.String("logEntry", logEntry)) exporterLogs = append(exporterLogs, logEntry) } } @@ -91,11 +90,9 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI stdOutListenFunction := func(out *bufio.Scanner, err *bufio.Scanner) { for out.Scan() { - fmt.Println(out.Text()) svc.logger.Info("Deploy Info: " + out.Text()) } for err.Scan() { - fmt.Println(err.Text()) svc.logger.Info("Deploy Error: " + err.Text()) } } @@ -129,12 +126,10 @@ func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc fu go stdOutFunc(stdoutScanner, stderrScanner) err := cmd.Start() if err != nil { - fmt.Printf("Error : %v \n", err) logger.Error("Collector Deploy Error", zap.Error(err)) } err = cmd.Wait() if err != nil { - fmt.Printf("Error: %v \n", err) logger.Error("Collector Deploy Error", zap.Error(err)) } return stdoutScanner, stderrScanner, err @@ -157,7 +152,6 @@ func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkI if err != nil { return "", cmd.Err() } - svc.logger.Info("debug returned slice", zap.Any("slice", slice)) value := slice[0] entry := value.Member.(CollectorStatusSortedSetEntry) return entry.Status, nil diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index c1ce6fb81..29fa3530d 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -196,10 +196,13 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { // analyzeLogs, will check for errors in exporter, and will return as follows // -// for active, the timestamp should not be longer than 5 minutes of the last metric export -// for errors 479 will send a "warning" state, plus message of too many requests -// for any other errors, will add error and message -// if no error message on exporter, will log as active +// for active, the timestamp should not be longer than 5 minutes of the last metric export +// for errors 479 will send a "warning" state, plus message of too many requests +// for any other errors, will add error and message +// if no error message on exporter, will log as active +// logs from otel-collector are coming in the standard from https://pkg.go.dev/log, +// +// TODO changing the logs from otel-collector to a json format that we can read and check for errors, will affect this func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err error) { var lastTimeStamp string for _, logLine := range logEntry { From b1876925d500038c504add2ecd8801302ea0ff8b Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 12 Jan 2023 10:56:53 -0300 Subject: [PATCH 33/56] feat(maestro): remove previous way of getting logs, it was changed using k8s client within monitor.go. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index b877dda76..449349238 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -3,7 +3,6 @@ package kubecontrol import ( "bufio" "context" - "fmt" "github.com/go-redis/redis/v8" "os" "os/exec" @@ -35,30 +34,6 @@ type Service interface { // UpdateOtelCollector - update an existing collector by id UpdateOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error - - // CollectLogs - collect logs from the collector by sink-id - CollectLogs(ctx context.Context, ownerID, sinkID string) ([]string, error) -} - -func (svc *deployService) CollectLogs(ctx context.Context, ownerID, sinkId string) ([]string, error) { - cmd := exec.Command("kubectl", "logs", fmt.Sprintf("otel-%s", sinkId), "-n", namespace) - exporterLogs := make([]string, 10) - watchLogsFunction := func(out *bufio.Scanner, err *bufio.Scanner) { - if err.Scan() || out.Err() != nil { - svc.logger.Error("failed to get logs for collector on sink", zap.Error(err.Err()), zap.Error(out.Err())) - return - } - for out.Scan() && len(exporterLogs) < 10 { - logEntry := out.Text() - exporterLogs = append(exporterLogs, logEntry) - } - } - _, _, err := execCmd(ctx, cmd, svc.logger, watchLogsFunction) - if err != nil { - svc.logger.Error("Error reading the logs") - exporterLogs = nil - } - return exporterLogs, err } func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerID, sinkId, manifest string) error { From a66b47b88182d727e774c80a50b356257d7ad6b1 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 12 Jan 2023 11:27:18 -0300 Subject: [PATCH 34/56] feat(maestro): fix typo. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 29fa3530d..50b97278d 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -170,7 +170,7 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status)) } keyPrefix := "sinker_key" - skey := fmt.Sprintf("%s-%s:%s", keyPrefix, data.OwnerID, data.SinkID) + skey := fmt.Sprintf("%s-%s-%s", keyPrefix, data.OwnerID, data.SinkID) err := data.State.SetFromString(status) if err != nil { svc.logger.Error("error during analyze logs", zap.Error(err)) From 4340767c292a9f3b6f7525a7b459c09ddde0e202 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 12 Jan 2023 11:53:35 -0300 Subject: [PATCH 35/56] feat(maestro): fix maestro-sinks interaction. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 3 ++- sinks/api/grpc/server.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 50b97278d..b46501d9a 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -170,7 +170,8 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status)) } keyPrefix := "sinker_key" - skey := fmt.Sprintf("%s-%s-%s", keyPrefix, data.OwnerID, data.SinkID) + skey := fmt.Sprintf("%s-%s:%s", keyPrefix, data.OwnerID, data.SinkID) + svc.logger.Info("debugging ownerID", zap.String("dest", data.OwnerID), zap.String("orig", sink.OwnerID)) err := data.State.SetFromString(status) if err != nil { svc.logger.Error("error during analyze logs", zap.Error(err)) diff --git a/sinks/api/grpc/server.go b/sinks/api/grpc/server.go index 82d913cd5..d7972ba96 100644 --- a/sinks/api/grpc/server.go +++ b/sinks/api/grpc/server.go @@ -75,6 +75,7 @@ func encodeSinksResponse(_ context.Context, grpcRes interface{}) (interface{}, e for i, sink := range res.sinks { sList[i] = &pb.SinkRes{ Id: sink.id, + OwnerID: sink.mfOwnerId, Name: sink.name, Description: sink.description, Tags: sink.tags, From e19902996a93008b6f933d67f4953dcb59c76dc0 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 12 Jan 2023 22:16:32 -0300 Subject: [PATCH 36/56] feat(maestro): fix maestro-sinks interaction. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 82 +++++++++++++++--------------- maestro/kubecontrol/monitor.go | 50 +++++++++--------- sinks/api/grpc/server.go | 1 + 3 files changed, 69 insertions(+), 64 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 449349238..911eb54c9 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -3,13 +3,15 @@ package kubecontrol import ( "bufio" "context" + "encoding/json" + "fmt" "github.com/go-redis/redis/v8" + maestroconfig "github.com/ns1labs/orb/maestro/config" + "go.uber.org/zap" "os" "os/exec" "strings" "time" - - "go.uber.org/zap" ) const namespace = "otelcollectors" @@ -41,18 +43,18 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI fileContent := []byte(manifest) tmp := strings.Split(string(fileContent), "\n") newContent := strings.Join(tmp[1:], "\n") - status, err := svc.getDeploymentState(ctx, ownerID, sinkId) + sinkData, err := svc.getDeploymentState(ctx, ownerID, sinkId) if err != nil { return err } if operation == "apply" { - if status != "deleted" { - svc.logger.Info("Already applied Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", status)) + if sinkData.State != maestroconfig.Active { + svc.logger.Info("Already applied Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", sinkData.State.String())) return nil } } else if operation == "delete" { - if status == "deleted" { - svc.logger.Info("Already deleted Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", status)) + if sinkData.State == maestroconfig.Idle { + svc.logger.Info("Already deleted Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", sinkData.State.String())) return nil } } @@ -78,12 +80,12 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI if err == nil { if operation == "apply" { - err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, *sinkData, ownerID, sinkId, "idle") if err != nil { return err } } else if operation == "delete" { - err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, *sinkData, ownerID, sinkId, "idle") if err != nil { return err } @@ -110,41 +112,41 @@ func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc fu return stdoutScanner, stderrScanner, err } -func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (string, error) { - key := CollectorStatusKey + "." + sinkId - args := redis.ZRangeArgs{ - Key: key, - Start: nil, - Stop: nil, - ByScore: false, - ByLex: false, - Rev: false, - Offset: 0, - Count: 0, - } - cmd := svc.redisClient.ZRangeArgsWithScores(ctx, args) - slice, err := cmd.Result() +func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (*maestroconfig.SinkData, error) { + skey := fmt.Sprintf("%s-%s:%s", sinkerKey, ownerID, sinkId) + + get := svc.redisClient.Get(ctx, skey) + if err := get.Err(); err != nil { + return nil, err + } + var unmarshalled maestroconfig.SinkData + bytes, err := get.Bytes() + if err != nil { + return nil, err + } + err = json.Unmarshal(bytes, &unmarshalled) if err != nil { - return "", cmd.Err() + return nil, err } - value := slice[0] - entry := value.Member.(CollectorStatusSortedSetEntry) - return entry.Status, nil + return &unmarshalled, nil } -func (svc *deployService) setNewDeploymentState(ctx context.Context, ownerID, sinkId, state string) error { - key := CollectorStatusKey + "." + sinkId - entry := redis.Z{ - Score: float64(time.Now().Unix()), - Member: CollectorStatusSortedSetEntry{ - SinkID: sinkId, - Status: "idle", - ErrorMessage: nil, - }, - } - intCmd := svc.redisClient.ZAdd(ctx, key, &entry) - if intCmd.Err() != nil { - return intCmd.Err() +func (svc *deployService) setNewDeploymentState(ctx context.Context, previousData maestroconfig.SinkData, ownerID, sinkId, state string) error { + skey := fmt.Sprintf("%s-%s:%s", sinkerKey, ownerID, sinkId) + err := previousData.State.SetFromString(state) + previousData.LastRemoteWrite = time.Now() + if err != nil { + return err + } + byteSink, err := json.Marshal(previousData) + if err != nil { + return err + } + if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + if err = svc.redisClient.Set(ctx, skey, byteSink, 0).Err(); err != nil { + return err } return nil } diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index b46501d9a..768e7f841 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -19,8 +19,9 @@ import ( "time" ) -const MonitorFixedDuration = 1 * time.Minute -const TimeDiffActiveIdle = 5 * time.Minute +const monitorFixedDuration = 1 * time.Minute +const timeDiffActiveIdle = 5 * time.Minute +const sinkerKey = "sinker_key" func NewMonitorService(logger *zap.Logger, sinksClient *sinkspb.SinkServiceClient, redisClient *redis.Client, kubecontrol *Service) MonitorService { return &monitorService{ @@ -44,7 +45,7 @@ type monitorService struct { func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelFunc) error { go func(ctx context.Context, cancelFunc context.CancelFunc) { - ticker := time.NewTicker(MonitorFixedDuration) + ticker := time.NewTicker(monitorFixedDuration) svc.logger.Info("start monitor routine", zap.Any("routine", ctx)) defer func() { cancelFunc() @@ -169,30 +170,31 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { } else { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status)) } - keyPrefix := "sinker_key" - skey := fmt.Sprintf("%s-%s:%s", keyPrefix, data.OwnerID, data.SinkID) - svc.logger.Info("debugging ownerID", zap.String("dest", data.OwnerID), zap.String("orig", sink.OwnerID)) - err := data.State.SetFromString(status) - if err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - if logsErr != nil { - data.Msg = logsErr.Error() - } - if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - byteSink, err := json.Marshal(data) - if err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - if err = svc.redisClient.Set(context.Background(), skey, byteSink, 0).Err(); err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } + svc.updateStatus(ctx, data, status, logsErr) } return } +} +func (svc *monitorService) updateStatus(ctx context.Context, data maestroconfig.SinkData, status string, logsErr error) { + skey := fmt.Sprintf("%s-%s:%s", sinkerKey, data.OwnerID, data.SinkID) + err := data.State.SetFromString(status) + if err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + if logsErr != nil { + data.Msg = logsErr.Error() + } + if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + byteSink, err := json.Marshal(data) + if err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + if err = svc.redisClient.Set(ctx, skey, byteSink, 0).Err(); err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } } // analyzeLogs, will check for errors in exporter, and will return as follows @@ -230,7 +232,7 @@ func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err er if err != nil { return "fail", err } - if lastLogTime.After(time.Now().Add(-TimeDiffActiveIdle)) { + if lastLogTime.After(time.Now().Add(-timeDiffActiveIdle)) { return "idle", nil } else { return "active", nil diff --git a/sinks/api/grpc/server.go b/sinks/api/grpc/server.go index d7972ba96..dad190065 100644 --- a/sinks/api/grpc/server.go +++ b/sinks/api/grpc/server.go @@ -100,6 +100,7 @@ func encodeSinkResponse(_ context.Context, grpcRes interface{}) (interface{}, er return &pb.SinkRes{ Id: res.id, Name: res.name, + OwnerID: res.mfOwnerId, Description: res.description, Tags: res.tags, State: res.state, From 27ab2bafd471627a6b8948bb6bb4eed7060573d1 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Thu, 12 Jan 2023 22:25:59 -0300 Subject: [PATCH 37/56] feat(mod): add go mod k8s client. Signed-off-by: Luiz Pegoraro --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 6936dae01..80f62e6e4 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,8 @@ require ( google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.26.0 + k8s.io/apimachinery v0.26.0 ) //These libs are used to allow orb extend opentelemetry features @@ -198,8 +200,6 @@ require ( google.golang.org/appengine v1.6.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - k8s.io/api v0.26.0 // indirect - k8s.io/apimachinery v0.26.0 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect From c23a907ec0e0d6a28299f8117b59b4099da1fb4d Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Fri, 13 Jan 2023 00:12:14 -0300 Subject: [PATCH 38/56] Revert "feat(maestro): fix maestro-sinks interaction." This reverts commit e19902996a93008b6f933d67f4953dcb59c76dc0. --- maestro/kubecontrol/kubecontrol.go | 82 +++++++++++++++--------------- maestro/kubecontrol/monitor.go | 50 +++++++++--------- sinks/api/grpc/server.go | 1 - 3 files changed, 64 insertions(+), 69 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 911eb54c9..449349238 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -3,15 +3,13 @@ package kubecontrol import ( "bufio" "context" - "encoding/json" - "fmt" "github.com/go-redis/redis/v8" - maestroconfig "github.com/ns1labs/orb/maestro/config" - "go.uber.org/zap" "os" "os/exec" "strings" "time" + + "go.uber.org/zap" ) const namespace = "otelcollectors" @@ -43,18 +41,18 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI fileContent := []byte(manifest) tmp := strings.Split(string(fileContent), "\n") newContent := strings.Join(tmp[1:], "\n") - sinkData, err := svc.getDeploymentState(ctx, ownerID, sinkId) + status, err := svc.getDeploymentState(ctx, ownerID, sinkId) if err != nil { return err } if operation == "apply" { - if sinkData.State != maestroconfig.Active { - svc.logger.Info("Already applied Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", sinkData.State.String())) + if status != "deleted" { + svc.logger.Info("Already applied Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", status)) return nil } } else if operation == "delete" { - if sinkData.State == maestroconfig.Idle { - svc.logger.Info("Already deleted Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", sinkData.State.String())) + if status == "deleted" { + svc.logger.Info("Already deleted Sink ID", zap.String("ownerID", ownerID), zap.String("sinkID", sinkId), zap.String("status", status)) return nil } } @@ -80,12 +78,12 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI if err == nil { if operation == "apply" { - err := svc.setNewDeploymentState(ctx, *sinkData, ownerID, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") if err != nil { return err } } else if operation == "delete" { - err := svc.setNewDeploymentState(ctx, *sinkData, ownerID, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") if err != nil { return err } @@ -112,41 +110,41 @@ func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc fu return stdoutScanner, stderrScanner, err } -func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (*maestroconfig.SinkData, error) { - skey := fmt.Sprintf("%s-%s:%s", sinkerKey, ownerID, sinkId) - - get := svc.redisClient.Get(ctx, skey) - if err := get.Err(); err != nil { - return nil, err - } - var unmarshalled maestroconfig.SinkData - bytes, err := get.Bytes() - if err != nil { - return nil, err - } - err = json.Unmarshal(bytes, &unmarshalled) +func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (string, error) { + key := CollectorStatusKey + "." + sinkId + args := redis.ZRangeArgs{ + Key: key, + Start: nil, + Stop: nil, + ByScore: false, + ByLex: false, + Rev: false, + Offset: 0, + Count: 0, + } + cmd := svc.redisClient.ZRangeArgsWithScores(ctx, args) + slice, err := cmd.Result() if err != nil { - return nil, err + return "", cmd.Err() } - return &unmarshalled, nil + value := slice[0] + entry := value.Member.(CollectorStatusSortedSetEntry) + return entry.Status, nil } -func (svc *deployService) setNewDeploymentState(ctx context.Context, previousData maestroconfig.SinkData, ownerID, sinkId, state string) error { - skey := fmt.Sprintf("%s-%s:%s", sinkerKey, ownerID, sinkId) - err := previousData.State.SetFromString(state) - previousData.LastRemoteWrite = time.Now() - if err != nil { - return err - } - byteSink, err := json.Marshal(previousData) - if err != nil { - return err - } - if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - if err = svc.redisClient.Set(ctx, skey, byteSink, 0).Err(); err != nil { - return err +func (svc *deployService) setNewDeploymentState(ctx context.Context, ownerID, sinkId, state string) error { + key := CollectorStatusKey + "." + sinkId + entry := redis.Z{ + Score: float64(time.Now().Unix()), + Member: CollectorStatusSortedSetEntry{ + SinkID: sinkId, + Status: "idle", + ErrorMessage: nil, + }, + } + intCmd := svc.redisClient.ZAdd(ctx, key, &entry) + if intCmd.Err() != nil { + return intCmd.Err() } return nil } diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 768e7f841..b46501d9a 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -19,9 +19,8 @@ import ( "time" ) -const monitorFixedDuration = 1 * time.Minute -const timeDiffActiveIdle = 5 * time.Minute -const sinkerKey = "sinker_key" +const MonitorFixedDuration = 1 * time.Minute +const TimeDiffActiveIdle = 5 * time.Minute func NewMonitorService(logger *zap.Logger, sinksClient *sinkspb.SinkServiceClient, redisClient *redis.Client, kubecontrol *Service) MonitorService { return &monitorService{ @@ -45,7 +44,7 @@ type monitorService struct { func (svc *monitorService) Start(ctx context.Context, cancelFunc context.CancelFunc) error { go func(ctx context.Context, cancelFunc context.CancelFunc) { - ticker := time.NewTicker(monitorFixedDuration) + ticker := time.NewTicker(MonitorFixedDuration) svc.logger.Info("start monitor routine", zap.Any("routine", ctx)) defer func() { cancelFunc() @@ -170,31 +169,30 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { } else { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status)) } - svc.updateStatus(ctx, data, status, logsErr) + keyPrefix := "sinker_key" + skey := fmt.Sprintf("%s-%s:%s", keyPrefix, data.OwnerID, data.SinkID) + svc.logger.Info("debugging ownerID", zap.String("dest", data.OwnerID), zap.String("orig", sink.OwnerID)) + err := data.State.SetFromString(status) + if err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + if logsErr != nil { + data.Msg = logsErr.Error() + } + if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + byteSink, err := json.Marshal(data) + if err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } + if err = svc.redisClient.Set(context.Background(), skey, byteSink, 0).Err(); err != nil { + svc.logger.Error("error during analyze logs", zap.Error(err)) + } } return } -} -func (svc *monitorService) updateStatus(ctx context.Context, data maestroconfig.SinkData, status string, logsErr error) { - skey := fmt.Sprintf("%s-%s:%s", sinkerKey, data.OwnerID, data.SinkID) - err := data.State.SetFromString(status) - if err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - if logsErr != nil { - data.Msg = logsErr.Error() - } - if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - byteSink, err := json.Marshal(data) - if err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - if err = svc.redisClient.Set(ctx, skey, byteSink, 0).Err(); err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } } // analyzeLogs, will check for errors in exporter, and will return as follows @@ -232,7 +230,7 @@ func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err er if err != nil { return "fail", err } - if lastLogTime.After(time.Now().Add(-timeDiffActiveIdle)) { + if lastLogTime.After(time.Now().Add(-TimeDiffActiveIdle)) { return "idle", nil } else { return "active", nil diff --git a/sinks/api/grpc/server.go b/sinks/api/grpc/server.go index dad190065..d7972ba96 100644 --- a/sinks/api/grpc/server.go +++ b/sinks/api/grpc/server.go @@ -100,7 +100,6 @@ func encodeSinkResponse(_ context.Context, grpcRes interface{}) (interface{}, er return &pb.SinkRes{ Id: res.id, Name: res.name, - OwnerID: res.mfOwnerId, Description: res.description, Tags: res.tags, State: res.state, From 181d404975fbbe01641950b0f4a4068ee9527da5 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Fri, 13 Jan 2023 17:32:16 -0300 Subject: [PATCH 39/56] feat(maestro): add time control. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 36 +++++++++++++----------------- maestro/kubecontrol/monitor.go | 4 ++++ maestro/service.go | 6 +++-- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 449349238..e8169ef3d 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -3,6 +3,7 @@ package kubecontrol import ( "bufio" "context" + "encoding/json" "github.com/go-redis/redis/v8" "os" "os/exec" @@ -78,12 +79,12 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI if err == nil { if operation == "apply" { - err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "new") if err != nil { return err } } else if operation == "delete" { - err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "idle") + err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "stopped") if err != nil { return err } @@ -111,38 +112,31 @@ func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc fu } func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (string, error) { - key := CollectorStatusKey + "." + sinkId - args := redis.ZRangeArgs{ - Key: key, - Start: nil, - Stop: nil, - ByScore: false, - ByLex: false, - Rev: false, - Offset: 0, - Count: 0, - } - cmd := svc.redisClient.ZRangeArgsWithScores(ctx, args) - slice, err := cmd.Result() + key := CollectorStatusKey + "." + ownerID + "." + sinkId + cmd := svc.redisClient.Get(ctx, key) + collectorStatusAsString, err := cmd.Result() if err != nil { return "", cmd.Err() } - value := slice[0] - entry := value.Member.(CollectorStatusSortedSetEntry) - return entry.Status, nil + var collectorStatus CollectorStatusSortedSetEntry + err = json.Unmarshal([]byte(collectorStatusAsString), &collectorStatus) + if err != nil { + return "", err + } + return collectorStatus.Status, nil } func (svc *deployService) setNewDeploymentState(ctx context.Context, ownerID, sinkId, state string) error { - key := CollectorStatusKey + "." + sinkId + key := CollectorStatusKey + "." + ownerID + "." + sinkId entry := redis.Z{ Score: float64(time.Now().Unix()), Member: CollectorStatusSortedSetEntry{ SinkID: sinkId, - Status: "idle", + Status: state, ErrorMessage: nil, }, } - intCmd := svc.redisClient.ZAdd(ctx, key, &entry) + intCmd := svc.redisClient.Set(ctx, key, &entry, 0) if intCmd.Err() != nil { return intCmd.Err() } diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index b46501d9a..bcec83ac3 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -151,6 +151,10 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { svc.logger.Warn("failed to unmarshal sink, skipping", zap.String("sink-id", sink.Id)) continue } + if data.LastRemoteWrite.After(time.Now().Add(-TimeDiffActiveIdle)) { + svc.logger.Warn("collector recently updated, skipping", zap.String("sink-id", sink.Id)) + continue + } data.SinkID = sink.Id data.OwnerID = sink.OwnerID data.LastRemoteWrite = time.Now() diff --git a/maestro/service.go b/maestro/service.go index 6c2aead3a..3322ffb29 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -113,7 +113,7 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can monitorCtx := context.WithValue(ctx, "routine", "monitor") err = svc.monitor.Start(monitorCtx, cancelFunction) if err != nil { - svc.logger.Error("error duriong monitor routine start", zap.Error(err)) + svc.logger.Error("error during monitor routine start", zap.Error(err)) cancelFunction() return err } @@ -124,13 +124,15 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can func (svc *maestroService) subscribeToSinkerES(ctx context.Context) { if err := svc.eventStore.SubscribeSinker(ctx); err != nil { svc.logger.Error("Bootstrap service failed to subscribe to event sourcing sinker", zap.Error(err)) + return } svc.logger.Info("Subscribed to Redis Event Store for sinker") } func (svc *maestroService) subscribeToSinksES(ctx context.Context) { - svc.logger.Info("Subscribed to Redis Event Store for sinks") if err := svc.eventStore.SubscribeSinks(ctx); err != nil { svc.logger.Error("Bootstrap service failed to subscribe to event sourcing sinks", zap.Error(err)) + return } + svc.logger.Info("Subscribed to Redis Event Store for sinks") } From 62f9ac2c1c9488f4fea36eaebe38a250ab255499 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 16 Jan 2023 11:51:59 -0300 Subject: [PATCH 40/56] feat(sinks): update proto. Signed-off-by: Luiz Pegoraro --- sinks/pb/sinks.pb.go | 209 ++++++++++++++++++++++++++++---------- sinks/pb/sinks.proto | 8 ++ sinks/pb/sinks_grpc.pb.go | 36 +++++++ 3 files changed, 197 insertions(+), 56 deletions(-) diff --git a/sinks/pb/sinks.pb.go b/sinks/pb/sinks.pb.go index 433430cad..cb7356e80 100644 --- a/sinks/pb/sinks.pb.go +++ b/sinks/pb/sinks.pb.go @@ -67,6 +67,77 @@ func (x *SinksRes) GetSinks() []*SinkRes { return nil } +type SinkStateUpdateReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + OwnerID string `protobuf:"bytes,2,opt,name=ownerID,proto3" json:"ownerID,omitempty"` + State string `protobuf:"bytes,3,opt,name=state,proto3" json:"state,omitempty"` + Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` +} + +func (x *SinkStateUpdateReq) Reset() { + *x = SinkStateUpdateReq{} + if protoimpl.UnsafeEnabled { + mi := &file_sinks_pb_sinks_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SinkStateUpdateReq) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SinkStateUpdateReq) ProtoMessage() {} + +func (x *SinkStateUpdateReq) ProtoReflect() protoreflect.Message { + mi := &file_sinks_pb_sinks_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SinkStateUpdateReq.ProtoReflect.Descriptor instead. +func (*SinkStateUpdateReq) Descriptor() ([]byte, []int) { + return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{1} +} + +func (x *SinkStateUpdateReq) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SinkStateUpdateReq) GetOwnerID() string { + if x != nil { + return x.OwnerID + } + return "" +} + +func (x *SinkStateUpdateReq) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *SinkStateUpdateReq) GetError() string { + if x != nil { + return x.Error + } + return "" +} + type SinksFilterReq struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -79,7 +150,7 @@ type SinksFilterReq struct { func (x *SinksFilterReq) Reset() { *x = SinksFilterReq{} if protoimpl.UnsafeEnabled { - mi := &file_sinks_pb_sinks_proto_msgTypes[1] + mi := &file_sinks_pb_sinks_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -92,7 +163,7 @@ func (x *SinksFilterReq) String() string { func (*SinksFilterReq) ProtoMessage() {} func (x *SinksFilterReq) ProtoReflect() protoreflect.Message { - mi := &file_sinks_pb_sinks_proto_msgTypes[1] + mi := &file_sinks_pb_sinks_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -105,7 +176,7 @@ func (x *SinksFilterReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SinksFilterReq.ProtoReflect.Descriptor instead. func (*SinksFilterReq) Descriptor() ([]byte, []int) { - return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{1} + return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{2} } func (x *SinksFilterReq) GetOtelEnabled() string { @@ -134,7 +205,7 @@ type SinkByIDReq struct { func (x *SinkByIDReq) Reset() { *x = SinkByIDReq{} if protoimpl.UnsafeEnabled { - mi := &file_sinks_pb_sinks_proto_msgTypes[2] + mi := &file_sinks_pb_sinks_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -147,7 +218,7 @@ func (x *SinkByIDReq) String() string { func (*SinkByIDReq) ProtoMessage() {} func (x *SinkByIDReq) ProtoReflect() protoreflect.Message { - mi := &file_sinks_pb_sinks_proto_msgTypes[2] + mi := &file_sinks_pb_sinks_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -160,7 +231,7 @@ func (x *SinkByIDReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SinkByIDReq.ProtoReflect.Descriptor instead. func (*SinkByIDReq) Descriptor() ([]byte, []int) { - return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{2} + return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{3} } func (x *SinkByIDReq) GetSinkID() string { @@ -196,7 +267,7 @@ type SinkRes struct { func (x *SinkRes) Reset() { *x = SinkRes{} if protoimpl.UnsafeEnabled { - mi := &file_sinks_pb_sinks_proto_msgTypes[3] + mi := &file_sinks_pb_sinks_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -209,7 +280,7 @@ func (x *SinkRes) String() string { func (*SinkRes) ProtoMessage() {} func (x *SinkRes) ProtoReflect() protoreflect.Message { - mi := &file_sinks_pb_sinks_proto_msgTypes[3] + mi := &file_sinks_pb_sinks_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -222,7 +293,7 @@ func (x *SinkRes) ProtoReflect() protoreflect.Message { // Deprecated: Use SinkRes.ProtoReflect.Descriptor instead. func (*SinkRes) Descriptor() ([]byte, []int) { - return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{3} + return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{4} } func (x *SinkRes) GetId() string { @@ -296,38 +367,49 @@ var file_sinks_pb_sinks_proto_rawDesc = []byte{ 0x08, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x22, - 0x48, 0x0a, 0x0e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, - 0x71, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x3f, 0x0a, 0x0b, 0x53, 0x69, 0x6e, - 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x6b, - 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, - 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x53, - 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, - 0x74, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, - 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, - 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, - 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x32, 0x7e, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, - 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, - 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, - 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, - 0x0d, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, - 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, - 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, - 0x73, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6a, 0x0a, 0x12, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, + 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x48, 0x0a, 0x0e, 0x53, + 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x20, 0x0a, + 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, + 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x3f, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, 0x49, + 0x44, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x53, 0x69, 0x6e, 0x6b, 0x52, + 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x14, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x61, 0x63, 0x6b, + 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, + 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x49, 0x44, 0x32, 0xbe, 0x01, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, + 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, + 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, + 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0d, 0x52, 0x65, + 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, 0x2e, 0x73, 0x69, + 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, + 0x65, 0x71, 0x1a, 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, + 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, + 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, + 0x52, 0x65, 0x73, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2f, 0x70, + 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -342,21 +424,24 @@ func file_sinks_pb_sinks_proto_rawDescGZIP() []byte { return file_sinks_pb_sinks_proto_rawDescData } -var file_sinks_pb_sinks_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_sinks_pb_sinks_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_sinks_pb_sinks_proto_goTypes = []interface{}{ - (*SinksRes)(nil), // 0: sinks.SinksRes - (*SinksFilterReq)(nil), // 1: sinks.SinksFilterReq - (*SinkByIDReq)(nil), // 2: sinks.SinkByIDReq - (*SinkRes)(nil), // 3: sinks.SinkRes + (*SinksRes)(nil), // 0: sinks.SinksRes + (*SinkStateUpdateReq)(nil), // 1: sinks.SinkStateUpdateReq + (*SinksFilterReq)(nil), // 2: sinks.SinksFilterReq + (*SinkByIDReq)(nil), // 3: sinks.SinkByIDReq + (*SinkRes)(nil), // 4: sinks.SinkRes } var file_sinks_pb_sinks_proto_depIdxs = []int32{ - 3, // 0: sinks.SinksRes.sinks:type_name -> sinks.SinkRes - 2, // 1: sinks.SinkService.RetrieveSink:input_type -> sinks.SinkByIDReq - 1, // 2: sinks.SinkService.RetrieveSinks:input_type -> sinks.SinksFilterReq - 3, // 3: sinks.SinkService.RetrieveSink:output_type -> sinks.SinkRes - 0, // 4: sinks.SinkService.RetrieveSinks:output_type -> sinks.SinksRes - 3, // [3:5] is the sub-list for method output_type - 1, // [1:3] is the sub-list for method input_type + 4, // 0: sinks.SinksRes.sinks:type_name -> sinks.SinkRes + 3, // 1: sinks.SinkService.RetrieveSink:input_type -> sinks.SinkByIDReq + 2, // 2: sinks.SinkService.RetrieveSinks:input_type -> sinks.SinksFilterReq + 1, // 3: sinks.SinkService.UpdateSinkState:input_type -> sinks.SinkStateUpdateReq + 4, // 4: sinks.SinkService.RetrieveSink:output_type -> sinks.SinkRes + 0, // 5: sinks.SinkService.RetrieveSinks:output_type -> sinks.SinksRes + 4, // 6: sinks.SinkService.UpdateSinkState:output_type -> sinks.SinkRes + 4, // [4:7] is the sub-list for method output_type + 1, // [1:4] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name 1, // [1:1] is the sub-list for extension extendee 0, // [0:1] is the sub-list for field type_name @@ -381,7 +466,7 @@ func file_sinks_pb_sinks_proto_init() { } } file_sinks_pb_sinks_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SinksFilterReq); i { + switch v := v.(*SinkStateUpdateReq); i { case 0: return &v.state case 1: @@ -393,7 +478,7 @@ func file_sinks_pb_sinks_proto_init() { } } file_sinks_pb_sinks_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SinkByIDReq); i { + switch v := v.(*SinksFilterReq); i { case 0: return &v.state case 1: @@ -405,6 +490,18 @@ func file_sinks_pb_sinks_proto_init() { } } file_sinks_pb_sinks_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SinkByIDReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_sinks_pb_sinks_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SinkRes); i { case 0: return &v.state @@ -423,7 +520,7 @@ func file_sinks_pb_sinks_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_sinks_pb_sinks_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 1, }, diff --git a/sinks/pb/sinks.proto b/sinks/pb/sinks.proto index 5286a163f..03e4951aa 100644 --- a/sinks/pb/sinks.proto +++ b/sinks/pb/sinks.proto @@ -6,12 +6,20 @@ option go_package = "sinks/pb"; service SinkService { rpc RetrieveSink(SinkByIDReq) returns (SinkRes) {} rpc RetrieveSinks(SinksFilterReq) returns (SinksRes) {} + rpc UpdateSinkState(SinkStateUpdateReq) returns (SinkRes) {} } message SinksRes { repeated SinkRes sinks = 1; } +message SinkStateUpdateReq { + string id = 1; + string ownerID = 2; + string state = 3; + string error = 4; +} + message SinksFilterReq { string otelEnabled = 1; string state = 2; diff --git a/sinks/pb/sinks_grpc.pb.go b/sinks/pb/sinks_grpc.pb.go index 864a656f7..779666d03 100644 --- a/sinks/pb/sinks_grpc.pb.go +++ b/sinks/pb/sinks_grpc.pb.go @@ -24,6 +24,7 @@ const _ = grpc.SupportPackageIsVersion7 type SinkServiceClient interface { RetrieveSink(ctx context.Context, in *SinkByIDReq, opts ...grpc.CallOption) (*SinkRes, error) RetrieveSinks(ctx context.Context, in *SinksFilterReq, opts ...grpc.CallOption) (*SinksRes, error) + UpdateSinkState(ctx context.Context, in *SinkStateUpdateReq, opts ...grpc.CallOption) (*SinkRes, error) } type sinkServiceClient struct { @@ -52,12 +53,22 @@ func (c *sinkServiceClient) RetrieveSinks(ctx context.Context, in *SinksFilterRe return out, nil } +func (c *sinkServiceClient) UpdateSinkState(ctx context.Context, in *SinkStateUpdateReq, opts ...grpc.CallOption) (*SinkRes, error) { + out := new(SinkRes) + err := c.cc.Invoke(ctx, "/sinks.SinkService/UpdateSinkState", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // SinkServiceServer is the server API for SinkService service. // All implementations must embed UnimplementedSinkServiceServer // for forward compatibility type SinkServiceServer interface { RetrieveSink(context.Context, *SinkByIDReq) (*SinkRes, error) RetrieveSinks(context.Context, *SinksFilterReq) (*SinksRes, error) + UpdateSinkState(context.Context, *SinkStateUpdateReq) (*SinkRes, error) mustEmbedUnimplementedSinkServiceServer() } @@ -71,6 +82,9 @@ func (UnimplementedSinkServiceServer) RetrieveSink(context.Context, *SinkByIDReq func (UnimplementedSinkServiceServer) RetrieveSinks(context.Context, *SinksFilterReq) (*SinksRes, error) { return nil, status.Errorf(codes.Unimplemented, "method RetrieveSinks not implemented") } +func (UnimplementedSinkServiceServer) UpdateSinkState(context.Context, *SinkStateUpdateReq) (*SinkRes, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateSinkState not implemented") +} func (UnimplementedSinkServiceServer) mustEmbedUnimplementedSinkServiceServer() {} // UnsafeSinkServiceServer may be embedded to opt out of forward compatibility for this service. @@ -120,6 +134,24 @@ func _SinkService_RetrieveSinks_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _SinkService_UpdateSinkState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SinkStateUpdateReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SinkServiceServer).UpdateSinkState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/sinks.SinkService/UpdateSinkState", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SinkServiceServer).UpdateSinkState(ctx, req.(*SinkStateUpdateReq)) + } + return interceptor(ctx, in, info, handler) +} + // SinkService_ServiceDesc is the grpc.ServiceDesc for SinkService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -135,6 +167,10 @@ var SinkService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RetrieveSinks", Handler: _SinkService_RetrieveSinks_Handler, }, + { + MethodName: "UpdateSinkState", + Handler: _SinkService_UpdateSinkState_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "sinks/pb/sinks.proto", From 7ba80ff05fffe39f30b806d5f49bf5c6e748f660 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 16 Jan 2023 15:20:41 -0300 Subject: [PATCH 41/56] Revert "feat(sinks): update proto." This reverts commit 62f9ac2c1c9488f4fea36eaebe38a250ab255499. --- sinks/pb/sinks.pb.go | 209 ++++++++++---------------------------- sinks/pb/sinks.proto | 8 -- sinks/pb/sinks_grpc.pb.go | 36 ------- 3 files changed, 56 insertions(+), 197 deletions(-) diff --git a/sinks/pb/sinks.pb.go b/sinks/pb/sinks.pb.go index cb7356e80..433430cad 100644 --- a/sinks/pb/sinks.pb.go +++ b/sinks/pb/sinks.pb.go @@ -67,77 +67,6 @@ func (x *SinksRes) GetSinks() []*SinkRes { return nil } -type SinkStateUpdateReq struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - OwnerID string `protobuf:"bytes,2,opt,name=ownerID,proto3" json:"ownerID,omitempty"` - State string `protobuf:"bytes,3,opt,name=state,proto3" json:"state,omitempty"` - Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` -} - -func (x *SinkStateUpdateReq) Reset() { - *x = SinkStateUpdateReq{} - if protoimpl.UnsafeEnabled { - mi := &file_sinks_pb_sinks_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SinkStateUpdateReq) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SinkStateUpdateReq) ProtoMessage() {} - -func (x *SinkStateUpdateReq) ProtoReflect() protoreflect.Message { - mi := &file_sinks_pb_sinks_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SinkStateUpdateReq.ProtoReflect.Descriptor instead. -func (*SinkStateUpdateReq) Descriptor() ([]byte, []int) { - return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{1} -} - -func (x *SinkStateUpdateReq) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -func (x *SinkStateUpdateReq) GetOwnerID() string { - if x != nil { - return x.OwnerID - } - return "" -} - -func (x *SinkStateUpdateReq) GetState() string { - if x != nil { - return x.State - } - return "" -} - -func (x *SinkStateUpdateReq) GetError() string { - if x != nil { - return x.Error - } - return "" -} - type SinksFilterReq struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -150,7 +79,7 @@ type SinksFilterReq struct { func (x *SinksFilterReq) Reset() { *x = SinksFilterReq{} if protoimpl.UnsafeEnabled { - mi := &file_sinks_pb_sinks_proto_msgTypes[2] + mi := &file_sinks_pb_sinks_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -163,7 +92,7 @@ func (x *SinksFilterReq) String() string { func (*SinksFilterReq) ProtoMessage() {} func (x *SinksFilterReq) ProtoReflect() protoreflect.Message { - mi := &file_sinks_pb_sinks_proto_msgTypes[2] + mi := &file_sinks_pb_sinks_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -176,7 +105,7 @@ func (x *SinksFilterReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SinksFilterReq.ProtoReflect.Descriptor instead. func (*SinksFilterReq) Descriptor() ([]byte, []int) { - return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{2} + return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{1} } func (x *SinksFilterReq) GetOtelEnabled() string { @@ -205,7 +134,7 @@ type SinkByIDReq struct { func (x *SinkByIDReq) Reset() { *x = SinkByIDReq{} if protoimpl.UnsafeEnabled { - mi := &file_sinks_pb_sinks_proto_msgTypes[3] + mi := &file_sinks_pb_sinks_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -218,7 +147,7 @@ func (x *SinkByIDReq) String() string { func (*SinkByIDReq) ProtoMessage() {} func (x *SinkByIDReq) ProtoReflect() protoreflect.Message { - mi := &file_sinks_pb_sinks_proto_msgTypes[3] + mi := &file_sinks_pb_sinks_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -231,7 +160,7 @@ func (x *SinkByIDReq) ProtoReflect() protoreflect.Message { // Deprecated: Use SinkByIDReq.ProtoReflect.Descriptor instead. func (*SinkByIDReq) Descriptor() ([]byte, []int) { - return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{3} + return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{2} } func (x *SinkByIDReq) GetSinkID() string { @@ -267,7 +196,7 @@ type SinkRes struct { func (x *SinkRes) Reset() { *x = SinkRes{} if protoimpl.UnsafeEnabled { - mi := &file_sinks_pb_sinks_proto_msgTypes[4] + mi := &file_sinks_pb_sinks_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -280,7 +209,7 @@ func (x *SinkRes) String() string { func (*SinkRes) ProtoMessage() {} func (x *SinkRes) ProtoReflect() protoreflect.Message { - mi := &file_sinks_pb_sinks_proto_msgTypes[4] + mi := &file_sinks_pb_sinks_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -293,7 +222,7 @@ func (x *SinkRes) ProtoReflect() protoreflect.Message { // Deprecated: Use SinkRes.ProtoReflect.Descriptor instead. func (*SinkRes) Descriptor() ([]byte, []int) { - return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{4} + return file_sinks_pb_sinks_proto_rawDescGZIP(), []int{3} } func (x *SinkRes) GetId() string { @@ -367,49 +296,38 @@ var file_sinks_pb_sinks_proto_rawDesc = []byte{ 0x08, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x22, - 0x6a, 0x0a, 0x12, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x71, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, - 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x48, 0x0a, 0x0e, 0x53, - 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x20, 0x0a, - 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, - 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x3f, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, 0x49, - 0x44, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, - 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, - 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x53, 0x69, 0x6e, 0x6b, 0x52, - 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x14, 0x0a, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x61, 0x63, 0x6b, - 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, 0x61, 0x63, 0x6b, 0x65, - 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, - 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, - 0x65, 0x72, 0x49, 0x44, 0x32, 0xbe, 0x01, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, - 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, - 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, - 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x0d, 0x52, 0x65, - 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, 0x2e, 0x73, 0x69, - 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, - 0x65, 0x71, 0x1a, 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, - 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, - 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x19, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, - 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x53, 0x74, 0x61, 0x74, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, - 0x52, 0x65, 0x73, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2f, 0x70, - 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x48, 0x0a, 0x0e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, + 0x71, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x74, 0x65, 0x6c, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x3f, 0x0a, 0x0b, 0x53, 0x69, 0x6e, + 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, 0x6b, + 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x6b, 0x49, 0x44, + 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x22, 0xdb, 0x01, 0x0a, 0x07, 0x53, + 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x74, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, + 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x62, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, + 0x0a, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x32, 0x7e, 0x0a, 0x0b, 0x53, 0x69, 0x6e, 0x6b, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x0c, 0x52, 0x65, 0x74, 0x72, 0x69, + 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, + 0x53, 0x69, 0x6e, 0x6b, 0x42, 0x79, 0x49, 0x44, 0x52, 0x65, 0x71, 0x1a, 0x0e, 0x2e, 0x73, 0x69, + 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x22, 0x00, 0x12, 0x39, 0x0a, + 0x0d, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x15, + 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x73, 0x46, 0x69, 0x6c, 0x74, + 0x65, 0x72, 0x52, 0x65, 0x71, 0x1a, 0x0f, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x73, 0x2e, 0x53, 0x69, + 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x73, 0x69, 0x6e, 0x6b, + 0x73, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -424,24 +342,21 @@ func file_sinks_pb_sinks_proto_rawDescGZIP() []byte { return file_sinks_pb_sinks_proto_rawDescData } -var file_sinks_pb_sinks_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_sinks_pb_sinks_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_sinks_pb_sinks_proto_goTypes = []interface{}{ - (*SinksRes)(nil), // 0: sinks.SinksRes - (*SinkStateUpdateReq)(nil), // 1: sinks.SinkStateUpdateReq - (*SinksFilterReq)(nil), // 2: sinks.SinksFilterReq - (*SinkByIDReq)(nil), // 3: sinks.SinkByIDReq - (*SinkRes)(nil), // 4: sinks.SinkRes + (*SinksRes)(nil), // 0: sinks.SinksRes + (*SinksFilterReq)(nil), // 1: sinks.SinksFilterReq + (*SinkByIDReq)(nil), // 2: sinks.SinkByIDReq + (*SinkRes)(nil), // 3: sinks.SinkRes } var file_sinks_pb_sinks_proto_depIdxs = []int32{ - 4, // 0: sinks.SinksRes.sinks:type_name -> sinks.SinkRes - 3, // 1: sinks.SinkService.RetrieveSink:input_type -> sinks.SinkByIDReq - 2, // 2: sinks.SinkService.RetrieveSinks:input_type -> sinks.SinksFilterReq - 1, // 3: sinks.SinkService.UpdateSinkState:input_type -> sinks.SinkStateUpdateReq - 4, // 4: sinks.SinkService.RetrieveSink:output_type -> sinks.SinkRes - 0, // 5: sinks.SinkService.RetrieveSinks:output_type -> sinks.SinksRes - 4, // 6: sinks.SinkService.UpdateSinkState:output_type -> sinks.SinkRes - 4, // [4:7] is the sub-list for method output_type - 1, // [1:4] is the sub-list for method input_type + 3, // 0: sinks.SinksRes.sinks:type_name -> sinks.SinkRes + 2, // 1: sinks.SinkService.RetrieveSink:input_type -> sinks.SinkByIDReq + 1, // 2: sinks.SinkService.RetrieveSinks:input_type -> sinks.SinksFilterReq + 3, // 3: sinks.SinkService.RetrieveSink:output_type -> sinks.SinkRes + 0, // 4: sinks.SinkService.RetrieveSinks:output_type -> sinks.SinksRes + 3, // [3:5] is the sub-list for method output_type + 1, // [1:3] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name 1, // [1:1] is the sub-list for extension extendee 0, // [0:1] is the sub-list for field type_name @@ -466,18 +381,6 @@ func file_sinks_pb_sinks_proto_init() { } } file_sinks_pb_sinks_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SinkStateUpdateReq); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_sinks_pb_sinks_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SinksFilterReq); i { case 0: return &v.state @@ -489,7 +392,7 @@ func file_sinks_pb_sinks_proto_init() { return nil } } - file_sinks_pb_sinks_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_sinks_pb_sinks_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SinkByIDReq); i { case 0: return &v.state @@ -501,7 +404,7 @@ func file_sinks_pb_sinks_proto_init() { return nil } } - file_sinks_pb_sinks_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_sinks_pb_sinks_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SinkRes); i { case 0: return &v.state @@ -520,7 +423,7 @@ func file_sinks_pb_sinks_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_sinks_pb_sinks_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 4, NumExtensions: 0, NumServices: 1, }, diff --git a/sinks/pb/sinks.proto b/sinks/pb/sinks.proto index 03e4951aa..5286a163f 100644 --- a/sinks/pb/sinks.proto +++ b/sinks/pb/sinks.proto @@ -6,20 +6,12 @@ option go_package = "sinks/pb"; service SinkService { rpc RetrieveSink(SinkByIDReq) returns (SinkRes) {} rpc RetrieveSinks(SinksFilterReq) returns (SinksRes) {} - rpc UpdateSinkState(SinkStateUpdateReq) returns (SinkRes) {} } message SinksRes { repeated SinkRes sinks = 1; } -message SinkStateUpdateReq { - string id = 1; - string ownerID = 2; - string state = 3; - string error = 4; -} - message SinksFilterReq { string otelEnabled = 1; string state = 2; diff --git a/sinks/pb/sinks_grpc.pb.go b/sinks/pb/sinks_grpc.pb.go index 779666d03..864a656f7 100644 --- a/sinks/pb/sinks_grpc.pb.go +++ b/sinks/pb/sinks_grpc.pb.go @@ -24,7 +24,6 @@ const _ = grpc.SupportPackageIsVersion7 type SinkServiceClient interface { RetrieveSink(ctx context.Context, in *SinkByIDReq, opts ...grpc.CallOption) (*SinkRes, error) RetrieveSinks(ctx context.Context, in *SinksFilterReq, opts ...grpc.CallOption) (*SinksRes, error) - UpdateSinkState(ctx context.Context, in *SinkStateUpdateReq, opts ...grpc.CallOption) (*SinkRes, error) } type sinkServiceClient struct { @@ -53,22 +52,12 @@ func (c *sinkServiceClient) RetrieveSinks(ctx context.Context, in *SinksFilterRe return out, nil } -func (c *sinkServiceClient) UpdateSinkState(ctx context.Context, in *SinkStateUpdateReq, opts ...grpc.CallOption) (*SinkRes, error) { - out := new(SinkRes) - err := c.cc.Invoke(ctx, "/sinks.SinkService/UpdateSinkState", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - // SinkServiceServer is the server API for SinkService service. // All implementations must embed UnimplementedSinkServiceServer // for forward compatibility type SinkServiceServer interface { RetrieveSink(context.Context, *SinkByIDReq) (*SinkRes, error) RetrieveSinks(context.Context, *SinksFilterReq) (*SinksRes, error) - UpdateSinkState(context.Context, *SinkStateUpdateReq) (*SinkRes, error) mustEmbedUnimplementedSinkServiceServer() } @@ -82,9 +71,6 @@ func (UnimplementedSinkServiceServer) RetrieveSink(context.Context, *SinkByIDReq func (UnimplementedSinkServiceServer) RetrieveSinks(context.Context, *SinksFilterReq) (*SinksRes, error) { return nil, status.Errorf(codes.Unimplemented, "method RetrieveSinks not implemented") } -func (UnimplementedSinkServiceServer) UpdateSinkState(context.Context, *SinkStateUpdateReq) (*SinkRes, error) { - return nil, status.Errorf(codes.Unimplemented, "method UpdateSinkState not implemented") -} func (UnimplementedSinkServiceServer) mustEmbedUnimplementedSinkServiceServer() {} // UnsafeSinkServiceServer may be embedded to opt out of forward compatibility for this service. @@ -134,24 +120,6 @@ func _SinkService_RetrieveSinks_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } -func _SinkService_UpdateSinkState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SinkStateUpdateReq) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SinkServiceServer).UpdateSinkState(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/sinks.SinkService/UpdateSinkState", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SinkServiceServer).UpdateSinkState(ctx, req.(*SinkStateUpdateReq)) - } - return interceptor(ctx, in, info, handler) -} - // SinkService_ServiceDesc is the grpc.ServiceDesc for SinkService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -167,10 +135,6 @@ var SinkService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RetrieveSinks", Handler: _SinkService_RetrieveSinks_Handler, }, - { - MethodName: "UpdateSinkState", - Handler: _SinkService_UpdateSinkState_Handler, - }, }, Streams: []grpc.StreamDesc{}, Metadata: "sinks/pb/sinks.proto", From 398be53166dcf11b72a6f58dec70274d24c0a7df Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 10:04:42 -0300 Subject: [PATCH 42/56] feat(sinks/maestro): update status using stream. --- maestro/kubecontrol/monitor.go | 44 +++++++++++++++++---------------- maestro/redis/events.go | 6 +++++ sinks/redis/consumer/streams.go | 12 +++++---- sinks/sinks.go | 13 +++++++--- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index bcec83ac3..13b8ae9f9 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -4,9 +4,9 @@ import ( "bytes" "context" "encoding/json" - "fmt" "github.com/go-redis/redis/v8" maestroconfig "github.com/ns1labs/orb/maestro/config" + maestroredis "github.com/ns1labs/orb/maestro/redis" "github.com/ns1labs/orb/pkg/errors" sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" @@ -166,6 +166,7 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { status, logsErr := svc.analyzeLogs(logs) if status == "fail" { svc.logger.Error("error during analyze logs", zap.Error(logsErr)) + continue } if data.State.String() != status { if err != nil { @@ -173,32 +174,33 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { } else { svc.logger.Info("updating status", zap.Any("before", sink.GetState()), zap.String("new status", status)) } - keyPrefix := "sinker_key" - skey := fmt.Sprintf("%s-%s:%s", keyPrefix, data.OwnerID, data.SinkID) - svc.logger.Info("debugging ownerID", zap.String("dest", data.OwnerID), zap.String("orig", sink.OwnerID)) - err := data.State.SetFromString(status) - if err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - if logsErr != nil { - data.Msg = logsErr.Error() - } - if err := svc.redisClient.Del(ctx, skey).Err(); err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - byteSink, err := json.Marshal(data) - if err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } - if err = svc.redisClient.Set(context.Background(), skey, byteSink, 0).Err(); err != nil { - svc.logger.Error("error during analyze logs", zap.Error(err)) - } + svc.publishSinkStateChange(sink, status, logsErr, err) } return } } +func (svc *monitorService) publishSinkStateChange(sink *sinkspb.SinkRes, status string, logsErr error, err error) { + streamID := "orb.sinker" + event := maestroredis.SinkerUpdateEvent{ + SinkID: sink.Id, + Owner: sink.OwnerID, + State: status, + Msg: logsErr.Error(), + Timestamp: time.Now(), + } + + record := &redis.XAddArgs{ + Stream: streamID, + Values: event.Encode(), + } + err = svc.redisClient.XAdd(context.Background(), record).Err() + if err != nil { + svc.logger.Error("error sending event to event store", zap.Error(err)) + } +} + // analyzeLogs, will check for errors in exporter, and will return as follows // // for active, the timestamp should not be longer than 5 minutes of the last metric export diff --git a/maestro/redis/events.go b/maestro/redis/events.go index a66b8a75b..f5507e91c 100644 --- a/maestro/redis/events.go +++ b/maestro/redis/events.go @@ -12,6 +12,11 @@ import ( "time" ) +const ( + SinkerPrefix = "sinker." + SinkerUpdate = SinkerPrefix + "update" +) + type SinksUpdateEvent struct { SinkID string Owner string @@ -34,6 +39,7 @@ func (cse SinkerUpdateEvent) Encode() map[string]interface{} { "state": cse.State, "msg": cse.Msg, "timestamp": cse.Timestamp.Unix(), + "operation": SinkerUpdate, } } diff --git a/sinks/redis/consumer/streams.go b/sinks/redis/consumer/streams.go index 68baea35b..84ab8244b 100644 --- a/sinks/redis/consumer/streams.go +++ b/sinks/redis/consumer/streams.go @@ -15,8 +15,6 @@ const ( sinkerPrefix = "sinker." sinkerUpdate = sinkerPrefix + "update" - otelYamlPrefix = "otel.yaml.sinker." - exists = "BUSYGROUP Consumer Group name already exists" ) @@ -63,7 +61,7 @@ func (es eventStore) Subscribe(context context.Context) error { var err error switch event["operation"] { case sinkerUpdate: - rte := decodeSinkerStateUpdate(event) + rte := es.decodeSinkerStateUpdate(event) err = es.handleSinkerStateUpdate(context, rte) } if err != nil { @@ -83,14 +81,18 @@ func (es eventStore) handleSinkerStateUpdate(ctx context.Context, event stateUpd return nil } -func decodeSinkerStateUpdate(event map[string]interface{}) stateUpdateEvent { +func (es eventStore) decodeSinkerStateUpdate(event map[string]interface{}) stateUpdateEvent { val := stateUpdateEvent{ ownerID: read(event, "owner", ""), sinkID: read(event, "sink_id", ""), msg: read(event, "msg", ""), timestamp: time.Time{}, } - val.state.Scan(event["state"]) + err := val.state.Scan(event["state"]) + if err != nil { + es.logger.Error("error parsing the state", zap.Error(err)) + return stateUpdateEvent{} + } return val } diff --git a/sinks/sinks.go b/sinks/sinks.go index 67760cb1b..8f7766914 100644 --- a/sinks/sinks.go +++ b/sinks/sinks.go @@ -48,6 +48,8 @@ const ( Active Error Idle + Warning + New ) type State int @@ -57,6 +59,8 @@ var stateMap = [...]string{ "active", "error", "idle", + "warning", + "new", } const MetadataLabelOtel = "opentelemetry" @@ -71,10 +75,11 @@ var stateRevMap = map[string]State{ "active": Active, "error": Error, "idle": Idle, + "warning": Warning, } -func (s State) String() string { - return stateMap[s] +func (s *State) String() string { + return stateMap[*s] } func (s *State) Scan(value interface{}) error { @@ -86,10 +91,10 @@ func (s *State) Scan(value interface{}) error { } asString = string(asBytes) } - *s = stateRevMap[string(asString)] + *s = stateRevMap[asString] return nil } -func (s State) Value() (driver.Value, error) { return s.String(), nil } +func (s *State) Value() (driver.Value, error) { return s.String(), nil } type Sink struct { ID string From a9a741ba0db3c909495f91f5f347c46daacdb2a7 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 12:41:42 -0300 Subject: [PATCH 43/56] feat(maestro): fix redis client Signed-off-by: Luiz Pegoraro --- .gitignore | 3 ++- kind/Chart.lock | 2 +- kind/Chart.yaml | 2 +- kind/values.yaml | 2 +- maestro/service.go | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index ca2b0ffb3..4cccce3b1 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ test_agent_name* docker/otel-collector-config.yaml !docker/otel-collector-config.yaml.tpl -kind/charts +kind/* +!kind/README.md diff --git a/kind/Chart.lock b/kind/Chart.lock index 4e8b08b1c..49b49c60d 100644 --- a/kind/Chart.lock +++ b/kind/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: orb repository: https://ns1labs.github.io/orb-helm/ - version: 1.0.39 + version: 1.0.40 digest: sha256:d7ae1413691620cf6f7a03fff2b562a9ca79fc5da1d1f4892ed364d45486d56d generated: "2022-12-19T21:18:59.933210512-03:00" diff --git a/kind/Chart.yaml b/kind/Chart.yaml index bc15d3f64..258f511f2 100644 --- a/kind/Chart.yaml +++ b/kind/Chart.yaml @@ -18,4 +18,4 @@ appVersion: "1.0.0" dependencies: - name: orb version: "1.0.40" - repository: "@ns1labs-orb" + repository: "" diff --git a/kind/values.yaml b/kind/values.yaml index 7b6485b52..d91ff7099 100644 --- a/kind/values.yaml +++ b/kind/values.yaml @@ -100,7 +100,7 @@ orb: dsn: "postgres://postgres:orb@kind-orb-postgresql-keto:5432/keto" maestro: - replicaCount: 1 + replicaCount: 2 kafka: host: kind-orb-kafka.orb.svc.cluster.local port: 9092 diff --git a/maestro/service.go b/maestro/service.go index 3322ffb29..164fab015 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -41,7 +41,7 @@ type maestroService struct { func NewMaestroService(logger *zap.Logger, streamRedisClient *redis.Client, sinkerRedisClient *redis.Client, sinksGrpcClient sinkspb.SinkServiceClient, esCfg config.EsConfig, otelCfg config.OtelConfig) Service { kubectr := kubecontrol.NewService(logger, streamRedisClient) eventStore := rediscons1.NewEventStore(streamRedisClient, otelCfg.KafkaUrl, kubectr, esCfg.Consumer, sinksGrpcClient, logger) - monitor := kubecontrol.NewMonitorService(logger, &sinksGrpcClient, sinkerRedisClient, &kubectr) + monitor := kubecontrol.NewMonitorService(logger, &sinksGrpcClient, streamRedisClient, &kubectr) return &maestroService{ logger: logger, streamRedisClient: streamRedisClient, From 5c15f939fb19671d5ccb6305d22bf0095fa60bbd Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 12:43:50 -0300 Subject: [PATCH 44/56] feat(maestro): rollback new status Signed-off-by: Luiz Pegoraro --- sinks/sinks.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/sinks/sinks.go b/sinks/sinks.go index 8f7766914..67760cb1b 100644 --- a/sinks/sinks.go +++ b/sinks/sinks.go @@ -48,8 +48,6 @@ const ( Active Error Idle - Warning - New ) type State int @@ -59,8 +57,6 @@ var stateMap = [...]string{ "active", "error", "idle", - "warning", - "new", } const MetadataLabelOtel = "opentelemetry" @@ -75,11 +71,10 @@ var stateRevMap = map[string]State{ "active": Active, "error": Error, "idle": Idle, - "warning": Warning, } -func (s *State) String() string { - return stateMap[*s] +func (s State) String() string { + return stateMap[s] } func (s *State) Scan(value interface{}) error { @@ -91,10 +86,10 @@ func (s *State) Scan(value interface{}) error { } asString = string(asBytes) } - *s = stateRevMap[asString] + *s = stateRevMap[string(asString)] return nil } -func (s *State) Value() (driver.Value, error) { return s.String(), nil } +func (s State) Value() (driver.Value, error) { return s.String(), nil } type Sink struct { ID string From 940863a1d7e360a4620c7c63f217e0a225431fd9 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 17:01:46 -0300 Subject: [PATCH 45/56] feat(maestro): fix interaction and added second layer of confirmation. Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 4 ++++ maestro/kubecontrol/monitor.go | 13 +++++++++++++ maestro/redis/consumer/streams.go | 2 ++ maestro/service.go | 29 +++++++++++++++++++++-------- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index e8169ef3d..cf9ae531c 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "encoding/json" + "fmt" "github.com/go-redis/redis/v8" "os" "os/exec" @@ -78,14 +79,17 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI _, _, err = execCmd(ctx, cmd, svc.logger, stdOutListenFunction) if err == nil { + svc.logger.Info(fmt.Sprintf("successfully %s the otel-collector for sink-id: %s", operation, sinkId)) if operation == "apply" { err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "new") if err != nil { + svc.logger.Error("error setting the new state", zap.Error(err)) return err } } else if operation == "delete" { err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "stopped") if err != nil { + svc.logger.Error("error setting the new state", zap.Error(err)) return err } } diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 13b8ae9f9..c94107525 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -33,6 +33,7 @@ func NewMonitorService(logger *zap.Logger, sinksClient *sinkspb.SinkServiceClien type MonitorService interface { Start(ctx context.Context, cancelFunc context.CancelFunc) error + GetRunningPods(ctx context.Context) ([]string, error) } type monitorService struct { @@ -102,6 +103,18 @@ func (svc *monitorService) getPodLogs(ctx context.Context, pod k8scorev1.Pod) ([ return splitLogs, nil } +func (svc *monitorService) GetRunningPods(ctx context.Context) (runningSinks []string, err error) { + pods, err := svc.getRunningPods(ctx) + if err != nil { + svc.logger.Error("error getting running collectors") + return + } + for i, pod := range pods { + runningSinks[i] = strings.TrimPrefix(pod.Name, "otel-") + } + return +} + func (svc *monitorService) getRunningPods(ctx context.Context) ([]k8scorev1.Pod, error) { config, err := rest.InClusterConfig() if err != nil { diff --git a/maestro/redis/consumer/streams.go b/maestro/redis/consumer/streams.go index b930e77db..5ab1d47ad 100644 --- a/maestro/redis/consumer/streams.go +++ b/maestro/redis/consumer/streams.go @@ -161,10 +161,12 @@ func (es eventStore) handleSinkerCreateCollector(ctx context.Context, event redi es.logger.Info("Received maestro CREATE event from sinker, sink state", zap.String("state", event.State), zap.String("sinkdID", event.SinkID), zap.String("ownerID", event.Owner)) deploymentEntry, err := es.GetDeploymentEntryFromSinkId(ctx, event.SinkID) if err != nil { + es.logger.Error("could not find deployment entry from sink-id", zap.String("sinkID", event.SinkID), zap.Error(err)) return err } err = es.kubecontrol.CreateOtelCollector(ctx, event.Owner, event.SinkID, deploymentEntry) if err != nil { + es.logger.Error("could not find deployment entry from sink-id", zap.String("sinkID", event.SinkID), zap.Error(err)) return err } return nil diff --git a/maestro/service.go b/maestro/service.go index 164fab015..1ef4c5617 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -11,6 +11,7 @@ package maestro import ( "context" "encoding/json" + "strings" "github.com/go-redis/redis/v8" maestroconfig "github.com/ns1labs/orb/maestro/config" @@ -71,6 +72,12 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can return err } + pods, err := svc.monitor.GetRunningPods(ctx) + if err != nil { + loadCancelFunction() + return err + } + for _, sinkRes := range sinksRes.Sinks { sinkContext := context.WithValue(loadCtx, "sink-id", sinkRes.Id) var data maestroconfig.SinkData @@ -81,18 +88,24 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can if val, _ := svc.eventStore.GetDeploymentEntryFromSinkId(ctx, sinkRes.Id); val != "" { svc.logger.Info("Skipping deploymentEntry because it is already created") - continue + } else { + err := svc.eventStore.CreateDeploymentEntry(sinkContext, sinkRes.Id, data.Url, data.User, data.Password) + if err != nil { + svc.logger.Warn("failed to create deploymentEntry for sink, skipping", zap.String("sink-id", sinkRes.Id)) + continue + } + svc.logger.Info("successfully created deploymentEntry for sink", zap.String("sink-id", sinkRes.Id), zap.String("state", sinkRes.State)) } - err := svc.eventStore.CreateDeploymentEntry(sinkContext, sinkRes.Id, data.Url, data.User, data.Password) - if err != nil { - svc.logger.Warn("failed to create deploymentEntry for sink, skipping", zap.String("sink-id", sinkRes.Id)) - continue + isDeployed := false + for _, pod := range pods { + if strings.Contains(pod, sinkRes.Id) { + isDeployed = true + break + } } - svc.logger.Info("successfully created deploymentEntry for sink", zap.String("sink-id", sinkRes.Id), zap.String("state", sinkRes.State)) - // if State is Active, deploy OtelCollector - if sinkRes.State == "1" || sinkRes.State == "active" { + if (sinkRes.State == "1" || sinkRes.State == "active") && !isDeployed { deploymentEntry, err := svc.eventStore.GetDeploymentEntryFromSinkId(sinkContext, sinkRes.Id) if err != nil { svc.logger.Warn("failed to fetch deploymentEntry for sink, skipping", zap.String("sink-id", sinkRes.Id)) From 230b6bc15ded37c5adf7d64c417525c3c6066ac3 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 18:22:44 -0300 Subject: [PATCH 46/56] feat(maestro): fix Hset Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index cf9ae531c..4d0943121 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -116,8 +116,8 @@ func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc fu } func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (string, error) { - key := CollectorStatusKey + "." + ownerID + "." + sinkId - cmd := svc.redisClient.Get(ctx, key) + key := CollectorStatusKey + cmd := svc.redisClient.HGet(ctx, key, sinkId) collectorStatusAsString, err := cmd.Result() if err != nil { return "", cmd.Err() @@ -131,7 +131,7 @@ func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkI } func (svc *deployService) setNewDeploymentState(ctx context.Context, ownerID, sinkId, state string) error { - key := CollectorStatusKey + "." + ownerID + "." + sinkId + key := CollectorStatusKey entry := redis.Z{ Score: float64(time.Now().Unix()), Member: CollectorStatusSortedSetEntry{ @@ -140,7 +140,7 @@ func (svc *deployService) setNewDeploymentState(ctx context.Context, ownerID, si ErrorMessage: nil, }, } - intCmd := svc.redisClient.Set(ctx, key, &entry, 0) + intCmd := svc.redisClient.HSet(ctx, key, &entry) if intCmd.Err() != nil { return intCmd.Err() } From d01ddfd7b8fe2014c2c552748c1ebba69c4773dd Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 18:31:49 -0300 Subject: [PATCH 47/56] feat(maestro): simplify Signed-off-by: Luiz Pegoraro --- maestro/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maestro/service.go b/maestro/service.go index 1ef4c5617..8778a1292 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -105,7 +105,7 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can } } // if State is Active, deploy OtelCollector - if (sinkRes.State == "1" || sinkRes.State == "active") && !isDeployed { + if sinkRes.State == "active" && !isDeployed { deploymentEntry, err := svc.eventStore.GetDeploymentEntryFromSinkId(sinkContext, sinkRes.Id) if err != nil { svc.logger.Warn("failed to fetch deploymentEntry for sink, skipping", zap.String("sink-id", sinkRes.Id)) From 0fa93ee776e937f024d9c316c58292d7fd244d8f Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 18:32:51 -0300 Subject: [PATCH 48/56] feat(maestro): add err to logs Signed-off-by: Luiz Pegoraro --- maestro/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maestro/service.go b/maestro/service.go index 8778a1292..a410828ea 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -108,12 +108,12 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can if sinkRes.State == "active" && !isDeployed { deploymentEntry, err := svc.eventStore.GetDeploymentEntryFromSinkId(sinkContext, sinkRes.Id) if err != nil { - svc.logger.Warn("failed to fetch deploymentEntry for sink, skipping", zap.String("sink-id", sinkRes.Id)) + svc.logger.Warn("failed to fetch deploymentEntry for sink, skipping", zap.String("sink-id", sinkRes.Id), zap.Error(err)) continue } err = svc.kubecontrol.CreateOtelCollector(sinkContext, sinkRes.OwnerID, sinkRes.Id, deploymentEntry) if err != nil { - svc.logger.Warn("failed to deploy OtelCollector for sink, skipping", zap.String("sink-id", sinkRes.Id)) + svc.logger.Warn("failed to deploy OtelCollector for sink, skipping", zap.String("sink-id", sinkRes.Id), zap.Error(err)) continue } svc.logger.Info("successfully created otel collector for sink", zap.String("sink-id", sinkRes.Id)) From 3654e499097125f9394c3d19843e02b030d92d94 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 19:01:42 -0300 Subject: [PATCH 49/56] feat(maestro): change state methods Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/kubecontrol.go | 58 +++++++++--------------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/maestro/kubecontrol/kubecontrol.go b/maestro/kubecontrol/kubecontrol.go index 4d0943121..86b45bc36 100644 --- a/maestro/kubecontrol/kubecontrol.go +++ b/maestro/kubecontrol/kubecontrol.go @@ -3,15 +3,15 @@ package kubecontrol import ( "bufio" "context" - "encoding/json" "fmt" "github.com/go-redis/redis/v8" + "go.uber.org/zap" + k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "os" "os/exec" "strings" - "time" - - "go.uber.org/zap" ) const namespace = "otelcollectors" @@ -45,6 +45,7 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI newContent := strings.Join(tmp[1:], "\n") status, err := svc.getDeploymentState(ctx, ownerID, sinkId) if err != nil { + svc.logger.Error("error getting deployment state", zap.Error(err)) return err } if operation == "apply" { @@ -80,19 +81,6 @@ func (svc *deployService) collectorDeploy(ctx context.Context, operation, ownerI if err == nil { svc.logger.Info(fmt.Sprintf("successfully %s the otel-collector for sink-id: %s", operation, sinkId)) - if operation == "apply" { - err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "new") - if err != nil { - svc.logger.Error("error setting the new state", zap.Error(err)) - return err - } - } else if operation == "delete" { - err := svc.setNewDeploymentState(ctx, ownerID, sinkId, "stopped") - if err != nil { - svc.logger.Error("error setting the new state", zap.Error(err)) - return err - } - } } return nil @@ -115,36 +103,24 @@ func execCmd(_ context.Context, cmd *exec.Cmd, logger *zap.Logger, stdOutFunc fu return stdoutScanner, stderrScanner, err } -func (svc *deployService) getDeploymentState(ctx context.Context, ownerID, sinkId string) (string, error) { - key := CollectorStatusKey - cmd := svc.redisClient.HGet(ctx, key, sinkId) - collectorStatusAsString, err := cmd.Result() +func (svc *deployService) getDeploymentState(ctx context.Context, _, sinkId string) (string, error) { + config, err := rest.InClusterConfig() if err != nil { - return "", cmd.Err() + svc.logger.Error("error on get cluster config", zap.Error(err)) + return "", err } - var collectorStatus CollectorStatusSortedSetEntry - err = json.Unmarshal([]byte(collectorStatusAsString), &collectorStatus) + clientSet, err := kubernetes.NewForConfig(config) if err != nil { + svc.logger.Error("error on get client", zap.Error(err)) return "", err } - return collectorStatus.Status, nil -} - -func (svc *deployService) setNewDeploymentState(ctx context.Context, ownerID, sinkId, state string) error { - key := CollectorStatusKey - entry := redis.Z{ - Score: float64(time.Now().Unix()), - Member: CollectorStatusSortedSetEntry{ - SinkID: sinkId, - Status: state, - ErrorMessage: nil, - }, - } - intCmd := svc.redisClient.HSet(ctx, key, &entry) - if intCmd.Err() != nil { - return intCmd.Err() + pods, err := clientSet.CoreV1().Pods(namespace).List(ctx, k8smetav1.ListOptions{}) + for _, pod := range pods.Items { + if strings.Contains(pod.Name, sinkId) { + return "active", nil + } } - return nil + return "deleted", nil } func (svc *deployService) CreateOtelCollector(ctx context.Context, ownerID, sinkID, deploymentEntry string) error { From 8887da9536b3554704bc0f95c64d8ddcc58385c0 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 19:12:33 -0300 Subject: [PATCH 50/56] feat(maestro): fix empty line logs Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 35 ++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index c94107525..734d93c6e 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -4,10 +4,10 @@ import ( "bytes" "context" "encoding/json" + "errors" "github.com/go-redis/redis/v8" maestroconfig "github.com/ns1labs/orb/maestro/config" maestroredis "github.com/ns1labs/orb/maestro/redis" - "github.com/ns1labs/orb/pkg/errors" sinkspb "github.com/ns1labs/orb/sinks/pb" "go.uber.org/zap" "io" @@ -226,21 +226,24 @@ func (svc *monitorService) publishSinkStateChange(sink *sinkspb.SinkRes, status func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err error) { var lastTimeStamp string for _, logLine := range logEntry { - lastTimeStamp = logLine[0:24] - if strings.Contains(logLine, "error") { - errStringLog := strings.TrimRight(logLine, "error") - jsonError := strings.Split(errStringLog, "\t")[4] - errorJson := make(map[string]interface{}) - err := json.Unmarshal([]byte(jsonError), &errorJson) - if err != nil { - return "fail", err - } - if errorJson != nil && errorJson["error"] != nil { - errorMessage := errorJson["error"].(string) - if strings.Contains(errorMessage, "429") { - return "warning", errors.New(errorMessage) - } else { - return "error", errors.New(errorMessage) + if len(logLine) > 24 { + lastTimeStamp = logLine[0:24] + svc.logger.Info("DEBUG LOG LINE", zap.String("log", logLine)) + if strings.Contains(logLine, "error") { + errStringLog := strings.TrimRight(logLine, "error") + jsonError := strings.Split(errStringLog, "\t")[4] + errorJson := make(map[string]interface{}) + err := json.Unmarshal([]byte(jsonError), &errorJson) + if err != nil { + return "fail", err + } + if errorJson != nil && errorJson["error"] != nil { + errorMessage := errorJson["error"].(string) + if strings.Contains(errorMessage, "429") { + return "warning", errors.New(errorMessage) + } else { + return "error", errors.New(errorMessage) + } } } } From acee05cff7efd52d662371d106dd85cf1ea17c60 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 19:13:18 -0300 Subject: [PATCH 51/56] feat(maestro): remove debug Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 1 - 1 file changed, 1 deletion(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 734d93c6e..4c68fbfce 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -228,7 +228,6 @@ func (svc *monitorService) analyzeLogs(logEntry []string) (status string, err er for _, logLine := range logEntry { if len(logLine) > 24 { lastTimeStamp = logLine[0:24] - svc.logger.Info("DEBUG LOG LINE", zap.String("log", logLine)) if strings.Contains(logLine, "error") { errStringLog := strings.TrimRight(logLine, "error") jsonError := strings.Split(errStringLog, "\t")[4] From d1c5efa54b7ad8ade4dd0c841b84d48faea9c4f9 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 17 Jan 2023 19:19:03 -0300 Subject: [PATCH 52/56] feat(maestro): nil safe Signed-off-by: Luiz Pegoraro --- maestro/kubecontrol/monitor.go | 7 +++++-- maestro/service.go | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index 4c68fbfce..a6dfefd5d 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -109,8 +109,11 @@ func (svc *monitorService) GetRunningPods(ctx context.Context) (runningSinks []s svc.logger.Error("error getting running collectors") return } - for i, pod := range pods { - runningSinks[i] = strings.TrimPrefix(pod.Name, "otel-") + if len(pods) > 0 { + for i, pod := range pods { + runningSinks[i] = strings.TrimPrefix(pod.Name, "otel-") + } + return } return } diff --git a/maestro/service.go b/maestro/service.go index a410828ea..9da2e25a0 100644 --- a/maestro/service.go +++ b/maestro/service.go @@ -98,10 +98,12 @@ func (svc *maestroService) Start(ctx context.Context, cancelFunction context.Can } isDeployed := false - for _, pod := range pods { - if strings.Contains(pod, sinkRes.Id) { - isDeployed = true - break + if len(pods) > 0 { + for _, pod := range pods { + if strings.Contains(pod, sinkRes.Id) { + isDeployed = true + break + } } } // if State is Active, deploy OtelCollector From 14b35c0f3eee27460260bf0b02237946ec616f21 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Wed, 18 Jan 2023 11:14:21 -0300 Subject: [PATCH 53/56] feat(maestro): fix duplicity of active and overriding error. Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder.go | 2 +- maestro/config/config_builder_test.go | 2 +- maestro/kubecontrol/monitor.go | 16 ++++++++----- sinker/otel/bridgeservice/bridge.go | 33 +++++++++++++++------------ 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index 5a2959d33..c96e0a65c 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -252,7 +252,7 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl Exporters []string `json:"exporters" yaml:"exporters"` }{ Receivers: []string{"kafka"}, - Exporters: []string{"prometheusremotewrite"}, + Exporters: []string{"prometheusremotewrite", "logging"}, }, }, }, diff --git a/maestro/config/config_builder_test.go b/maestro/config/config_builder_test.go index 2ba091ae2..8ba942045 100644 --- a/maestro/config/config_builder_test.go +++ b/maestro/config/config_builder_test.go @@ -27,7 +27,7 @@ func TestReturnConfigYamlFromSink(t *testing.T) { sinkUrl: "https://mysinkurl:9922", sinkUsername: "1234123", sinkPassword: "CarnivorousVulgaris", - }, want: `---\nreceivers:\n kafka:\n brokers:\n - kafka:9092\n topic: otlp_metrics-sink-id-222\n protocol_version: 2.0.0\nextensions:\n health_check:\n endpoint: 0.0.0.0:13133\n path: /health/status\n check_collector_pipeline:\n enabled: true\n interval: 5m\n exporter_failure_threshold: 4\n pprof:\n endpoint: 0.0.0.0:1888\n basicauth/exporter:\n client_auth:\n username: 1234123\n password: CarnivorousVulgaris\nexporters:\n prometheusremotewrite:\n endpoint: https://mysinkurl:9922\n auth:\n authenticator: basicauth/exporter\n logging:\n verbosity: detailed\n sampling_initial: 5\n sampling_thereafter: 50\nservice:\n extensions:\n - pprof\n - health_check\n - basicauth/exporter\n pipelines:\n metrics:\n receivers:\n - kafka\n exporters:\n - prometheusremotewrite\n`, + }, want: `---\nreceivers:\n kafka:\n brokers:\n - kafka:9092\n topic: otlp_metrics-sink-id-222\n protocol_version: 2.0.0\nextensions:\n health_check:\n endpoint: 0.0.0.0:13133\n path: /health/status\n check_collector_pipeline:\n enabled: true\n interval: 5m\n exporter_failure_threshold: 4\n pprof:\n endpoint: 0.0.0.0:1888\n basicauth/exporter:\n client_auth:\n username: 1234123\n password: CarnivorousVulgaris\nexporters:\n prometheusremotewrite:\n endpoint: https://mysinkurl:9922\n auth:\n authenticator: basicauth/exporter\n logging:\n verbosity: detailed\n sampling_initial: 5\n sampling_thereafter: 50\nservice:\n extensions:\n - pprof\n - health_check\n - basicauth/exporter\n pipelines:\n metrics:\n receivers:\n - kafka\n exporters:\n - prometheusremotewrite\n - logging\n`, wantErr: false}, } for _, tt := range tests { diff --git a/maestro/kubecontrol/monitor.go b/maestro/kubecontrol/monitor.go index a6dfefd5d..da1e588e2 100644 --- a/maestro/kubecontrol/monitor.go +++ b/maestro/kubecontrol/monitor.go @@ -103,19 +103,20 @@ func (svc *monitorService) getPodLogs(ctx context.Context, pod k8scorev1.Pod) ([ return splitLogs, nil } -func (svc *monitorService) GetRunningPods(ctx context.Context) (runningSinks []string, err error) { +func (svc *monitorService) GetRunningPods(ctx context.Context) ([]string, error) { pods, err := svc.getRunningPods(ctx) if err != nil { svc.logger.Error("error getting running collectors") - return + return nil, err } + runningSinks := make([]string, len(pods)) if len(pods) > 0 { for i, pod := range pods { runningSinks[i] = strings.TrimPrefix(pod.Name, "otel-") } - return + return runningSinks, nil } - return + return nil, nil } func (svc *monitorService) getRunningPods(ctx context.Context) ([]k8scorev1.Pod, error) { @@ -192,18 +193,21 @@ func (svc *monitorService) monitorSinks(ctx context.Context) { } svc.publishSinkStateChange(sink, status, logsErr, err) } - return } } func (svc *monitorService) publishSinkStateChange(sink *sinkspb.SinkRes, status string, logsErr error, err error) { streamID := "orb.sinker" + logMessage := "" + if logsErr != nil { + logMessage = logsErr.Error() + } event := maestroredis.SinkerUpdateEvent{ SinkID: sink.Id, Owner: sink.OwnerID, State: status, - Msg: logsErr.Error(), + Msg: logMessage, Timestamp: time.Now(), } diff --git a/sinker/otel/bridgeservice/bridge.go b/sinker/otel/bridgeservice/bridge.go index 0582acfc7..e67adcaaa 100644 --- a/sinker/otel/bridgeservice/bridge.go +++ b/sinker/otel/bridgeservice/bridge.go @@ -43,20 +43,25 @@ func (bs *SinkerOtelBridgeService) NotifyActiveSink(_ context.Context, mfOwnerId bs.logger.Error("unable to retrieve the sink config", zap.Error(err)) return err } - err = cfgRepo.State.SetFromString(newState) - if err != nil { - bs.logger.Error("unable to set state", zap.String("new_state", newState), zap.Error(err)) - return err - } - if cfgRepo.State == config.Error { - cfgRepo.Msg = message - } else if cfgRepo.State == config.Active { - cfgRepo.LastRemoteWrite = time.Now() - } - err = bs.sinkerCache.Edit(cfgRepo) - if err != nil { - bs.logger.Error("error during update sink cache", zap.String("sinkId", sinkId), zap.Error(err)) - return err + // only updates if not Idle or Unknown + if cfgRepo.State == config.Idle || cfgRepo.State == config.Unknown { + err = cfgRepo.State.SetFromString(newState) + if err != nil { + bs.logger.Error("unable to set state", zap.String("new_state", newState), zap.Error(err)) + return err + } + if cfgRepo.State == config.Error { + cfgRepo.Msg = message + } else if cfgRepo.State == config.Active { + cfgRepo.LastRemoteWrite = time.Now() + } + err = bs.sinkerCache.Edit(cfgRepo) + if err != nil { + bs.logger.Error("error during update sink cache", zap.String("sinkId", sinkId), zap.Error(err)) + return err + } + } else { + bs.logger.Info("sink is already active, no need to update state") } return nil From 9f4ce215f2ca3175514ea20e8824b558fb72fb5b Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Fri, 20 Jan 2023 15:16:02 -0300 Subject: [PATCH 54/56] feat(sinker): remove healthcheck. Signed-off-by: Luiz Pegoraro --- maestro/config/config_builder.go | 6 ------ maestro/config/config_builder_test.go | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/maestro/config/config_builder.go b/maestro/config/config_builder.go index c96e0a65c..52e0a2848 100644 --- a/maestro/config/config_builder.go +++ b/maestro/config/config_builder.go @@ -208,12 +208,6 @@ func ReturnConfigYamlFromSink(_ context.Context, kafkaUrlConfig, sinkId, sinkUrl }, }, Extensions: &Extensions{ - HealthCheckExtConfig: &HealthCheckExtension{ - Endpoint: "0.0.0.0:13133", - Path: "/health/status", - CollectorPipeline: &CollectorPipelineExtension{ - Enabled: "true", Interval: "5m", FailureThreshold: 4}, - }, PProf: &PProfExtension{ Endpoint: "0.0.0.0:1888", // Leaving default for now, will need to change with more processes }, diff --git a/maestro/config/config_builder_test.go b/maestro/config/config_builder_test.go index 8ba942045..e910a0937 100644 --- a/maestro/config/config_builder_test.go +++ b/maestro/config/config_builder_test.go @@ -27,7 +27,7 @@ func TestReturnConfigYamlFromSink(t *testing.T) { sinkUrl: "https://mysinkurl:9922", sinkUsername: "1234123", sinkPassword: "CarnivorousVulgaris", - }, want: `---\nreceivers:\n kafka:\n brokers:\n - kafka:9092\n topic: otlp_metrics-sink-id-222\n protocol_version: 2.0.0\nextensions:\n health_check:\n endpoint: 0.0.0.0:13133\n path: /health/status\n check_collector_pipeline:\n enabled: true\n interval: 5m\n exporter_failure_threshold: 4\n pprof:\n endpoint: 0.0.0.0:1888\n basicauth/exporter:\n client_auth:\n username: 1234123\n password: CarnivorousVulgaris\nexporters:\n prometheusremotewrite:\n endpoint: https://mysinkurl:9922\n auth:\n authenticator: basicauth/exporter\n logging:\n verbosity: detailed\n sampling_initial: 5\n sampling_thereafter: 50\nservice:\n extensions:\n - pprof\n - health_check\n - basicauth/exporter\n pipelines:\n metrics:\n receivers:\n - kafka\n exporters:\n - prometheusremotewrite\n - logging\n`, + }, want: `---\nreceivers:\n kafka:\n brokers:\n - kafka:9092\n topic: otlp_metrics-sink-id-222\n protocol_version: 2.0.0\nextensions:\n pprof:\n endpoint: 0.0.0.0:1888\n basicauth/exporter:\n client_auth:\n username: 1234123\n password: CarnivorousVulgaris\nexporters:\n prometheusremotewrite:\n endpoint: https://mysinkurl:9922\n auth:\n authenticator: basicauth/exporter\n logging:\n verbosity: detailed\n sampling_initial: 5\n sampling_thereafter: 50\nservice:\n extensions:\n - pprof\n - health_check\n - basicauth/exporter\n pipelines:\n metrics:\n receivers:\n - kafka\n exporters:\n - prometheusremotewrite\n - logging\n`, wantErr: false}, } for _, tt := range tests { From 3202d29d99428b4c67951ab9c49657c1c43d58e9 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Mon, 23 Jan 2023 16:09:28 -0300 Subject: [PATCH 55/56] feat(sinker/maestro): update notifying method, fixing override of status from sinker. Signed-off-by: Luiz Pegoraro --- maestro/redis/consumer/streams.go | 12 ++++---- sinker/config/repo.go | 3 ++ sinker/otel/bridgeservice/bridge.go | 4 +-- sinker/redis/sinker.go | 47 +++++++++++++++++++++-------- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/maestro/redis/consumer/streams.go b/maestro/redis/consumer/streams.go index 5ab1d47ad..376d8d97f 100644 --- a/maestro/redis/consumer/streams.go +++ b/maestro/redis/consumer/streams.go @@ -13,9 +13,9 @@ import ( ) const ( - streamSinker = "orb.sinker" - streamSinks = "orb.sinks" - group = "orb.collectors" + streamMaestro = "orb.maestro" + streamSinks = "orb.sinks" + group = "orb.collectors" sinkerPrefix = "sinker." sinkerUpdate = sinkerPrefix + "update" @@ -57,7 +57,7 @@ func NewEventStore(client *redis.Client, kafkaUrl string, kubecontrol kubecontro func (es eventStore) SubscribeSinker(context context.Context) error { //listening sinker events - err := es.client.XGroupCreateMkStream(context, streamSinker, group, "$").Err() + err := es.client.XGroupCreateMkStream(context, streamMaestro, group, "$").Err() if err != nil && err.Error() != exists { return err } @@ -66,7 +66,7 @@ func (es eventStore) SubscribeSinker(context context.Context) error { streams, err := es.client.XReadGroup(context, &redis.XReadGroupArgs{ Group: group, Consumer: es.esconsumer, - Streams: []string{streamSinker, ">"}, + Streams: []string{streamMaestro, ">"}, Count: 100, }).Result() if err != nil || len(streams) == 0 { @@ -89,7 +89,7 @@ func (es eventStore) SubscribeSinker(context context.Context) error { es.logger.Error("Failed to handle sinker event", zap.String("operation", event["operation"].(string)), zap.Error(err)) break } - es.client.XAck(context, streamSinker, group, msg.ID) + es.client.XAck(context, streamMaestro, group, msg.ID) } } } diff --git a/sinker/config/repo.go b/sinker/config/repo.go index d11d9a40e..cbbd38561 100644 --- a/sinker/config/repo.go +++ b/sinker/config/repo.go @@ -4,6 +4,8 @@ package config +import "context" + type ConfigRepo interface { Exists(ownerID string, sinkID string) bool Add(config SinkConfig) error @@ -12,4 +14,5 @@ type ConfigRepo interface { Edit(config SinkConfig) error GetAll(ownerID string) ([]SinkConfig, error) GetAllOwners() ([]string, error) + DeployCollector(ctx context.Context, config SinkConfig) error } diff --git a/sinker/otel/bridgeservice/bridge.go b/sinker/otel/bridgeservice/bridge.go index e67adcaaa..3e744dcbb 100644 --- a/sinker/otel/bridgeservice/bridge.go +++ b/sinker/otel/bridgeservice/bridge.go @@ -37,7 +37,7 @@ type SinkerOtelBridgeService struct { fleetClient fleetpb.FleetServiceClient } -func (bs *SinkerOtelBridgeService) NotifyActiveSink(_ context.Context, mfOwnerId, sinkId, newState, message string) error { +func (bs *SinkerOtelBridgeService) NotifyActiveSink(ctx context.Context, mfOwnerId, sinkId, newState, message string) error { cfgRepo, err := bs.sinkerCache.Get(mfOwnerId, sinkId) if err != nil { bs.logger.Error("unable to retrieve the sink config", zap.Error(err)) @@ -55,7 +55,7 @@ func (bs *SinkerOtelBridgeService) NotifyActiveSink(_ context.Context, mfOwnerId } else if cfgRepo.State == config.Active { cfgRepo.LastRemoteWrite = time.Now() } - err = bs.sinkerCache.Edit(cfgRepo) + err = bs.sinkerCache.DeployCollector(ctx, cfgRepo) if err != nil { bs.logger.Error("error during update sink cache", zap.String("sinkId", sinkId), zap.Error(err)) return err diff --git a/sinker/redis/sinker.go b/sinker/redis/sinker.go index 86de8cd13..85bc167d7 100644 --- a/sinker/redis/sinker.go +++ b/sinker/redis/sinker.go @@ -4,27 +4,29 @@ import ( "context" "encoding/json" "fmt" + "github.com/ns1labs/orb/sinker/redis/producer" "strings" + "time" "github.com/go-redis/redis/v8" "github.com/ns1labs/orb/sinker" - "github.com/ns1labs/orb/sinker/config" + sinkerconfig "github.com/ns1labs/orb/sinker/config" "go.uber.org/zap" ) const ( keyPrefix = "sinker_key" - idPrefix = "sinker" + idPrefix = "orb.maestro" ) -var _ config.ConfigRepo = (*sinkerCache)(nil) +var _ sinkerconfig.ConfigRepo = (*sinkerCache)(nil) type sinkerCache struct { client *redis.Client logger *zap.Logger } -func NewSinkerCache(client *redis.Client, logger *zap.Logger) config.ConfigRepo { +func NewSinkerCache(client *redis.Client, logger *zap.Logger) sinkerconfig.ConfigRepo { return &sinkerCache{client: client, logger: logger} } @@ -39,7 +41,7 @@ func (s *sinkerCache) Exists(ownerID string, sinkID string) bool { return false } -func (s *sinkerCache) Add(config config.SinkConfig) error { +func (s *sinkerCache) Add(config sinkerconfig.SinkConfig) error { skey := fmt.Sprintf("%s-%s:%s", keyPrefix, config.OwnerID, config.SinkID) bytes, err := json.Marshal(config) if err != nil { @@ -59,23 +61,23 @@ func (s *sinkerCache) Remove(ownerID string, sinkID string) error { return nil } -func (s *sinkerCache) Get(ownerID string, sinkID string) (config.SinkConfig, error) { +func (s *sinkerCache) Get(ownerID string, sinkID string) (sinkerconfig.SinkConfig, error) { if ownerID == "" || sinkID == "" { - return config.SinkConfig{}, sinker.ErrNotFound + return sinkerconfig.SinkConfig{}, sinker.ErrNotFound } skey := fmt.Sprintf("%s-%s:%s", keyPrefix, ownerID, sinkID) cachedConfig, err := s.client.Get(context.Background(), skey).Result() if err != nil { - return config.SinkConfig{}, err + return sinkerconfig.SinkConfig{}, err } - var cfgSinker config.SinkConfig + var cfgSinker sinkerconfig.SinkConfig if err := json.Unmarshal([]byte(cachedConfig), &cfgSinker); err != nil { - return config.SinkConfig{}, err + return sinkerconfig.SinkConfig{}, err } return cfgSinker, nil } -func (s *sinkerCache) Edit(config config.SinkConfig) error { +func (s *sinkerCache) Edit(config sinkerconfig.SinkConfig) error { if err := s.Remove(config.OwnerID, config.SinkID); err != nil { return err } @@ -85,6 +87,25 @@ func (s *sinkerCache) Edit(config config.SinkConfig) error { return nil } +func (s *sinkerCache) DeployCollector(ctx context.Context, config sinkerconfig.SinkConfig) error { + event := producer.SinkerUpdateEvent{ + SinkID: config.SinkID, + Owner: config.OwnerID, + State: config.State.String(), + Msg: config.Msg, + Timestamp: time.Now(), + } + encodeEvent := redis.XAddArgs{ + ID: config.SinkID, + Stream: idPrefix, + Values: event, + } + if cmd := s.client.XAdd(ctx, &encodeEvent); cmd.Err() != nil { + return cmd.Err() + } + return nil +} + func (s *sinkerCache) GetAllOwners() ([]string, error) { iter := s.client.Scan(context.Background(), 0, fmt.Sprintf("%s-*", keyPrefix), 0).Iterator() var owners []string @@ -101,9 +122,9 @@ func (s *sinkerCache) GetAllOwners() ([]string, error) { return owners, nil } -func (s *sinkerCache) GetAll(ownerID string) ([]config.SinkConfig, error) { +func (s *sinkerCache) GetAll(ownerID string) ([]sinkerconfig.SinkConfig, error) { iter := s.client.Scan(context.Background(), 0, fmt.Sprintf("%s-%s:*", keyPrefix, ownerID), 0).Iterator() - var configs []config.SinkConfig + var configs []sinkerconfig.SinkConfig for iter.Next(context.Background()) { keys := strings.Split(strings.TrimPrefix(iter.Val(), fmt.Sprintf("%s-", keyPrefix)), ":") sinkID := "" From 65060cfc97003fae0d0b0ba3bbe0afbed4153f31 Mon Sep 17 00:00:00 2001 From: Luiz Pegoraro Date: Tue, 24 Jan 2023 14:12:30 -0300 Subject: [PATCH 56/56] feat(sinker): fix implementation. Signed-off-by: Luiz Pegoraro --- sinker/redis/producer/streams.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sinker/redis/producer/streams.go b/sinker/redis/producer/streams.go index 2e4af4ae5..53aa29e54 100644 --- a/sinker/redis/producer/streams.go +++ b/sinker/redis/producer/streams.go @@ -21,6 +21,11 @@ type eventStore struct { logger *zap.Logger } +// DeployCollector only used in maestro +func (e eventStore) DeployCollector(ctx context.Context, config config.SinkConfig) error { + return nil +} + func (e eventStore) Exists(ownerID string, sinkID string) bool { return e.sinkCache.Exists(ownerID, sinkID) }