From 908c4f49c2d2ae1ea463aa35862ab7c5fcaa14b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 30 Aug 2024 16:12:01 +0200 Subject: [PATCH 01/41] wip --- pkg/conduit/runtime.go | 11 ++++++++++- pkg/pipeline/lifecycle.go | 13 ++++++++++--- pkg/pipeline/service.go | 5 ++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index 512b7f615..8a2a7c347 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -62,6 +62,7 @@ import ( "github.com/conduitio/conduit/pkg/web/ui" apiv1 "github.com/conduitio/conduit/proto/api/v1" grpcruntime "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/jpillora/backoff" "github.com/piotrkowalczuk/promgrpc/v4" promclient "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -197,7 +198,15 @@ func createServices(r *Runtime) error { tokenService, ) - plService := pipeline.NewService(r.logger, r.DB) + // TODO: remove me once this is coming from configuration + backoffCfg := &backoff.Backoff{ + Min: 100 * time.Millisecond, + Max: 10 * time.Second, + Factor: 2, + Jitter: true, + } + + plService := pipeline.NewService(r.logger, r.DB, backoffCfg) connService := connector.NewService(r.logger, r.DB, r.connectorPersister) procService := processor.NewService(r.logger, r.DB, procPluginService) diff --git a/pkg/pipeline/lifecycle.go b/pkg/pipeline/lifecycle.go index 9eb8b24a3..35f749251 100644 --- a/pkg/pipeline/lifecycle.go +++ b/pkg/pipeline/lifecycle.go @@ -627,9 +627,16 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { pl.SetStatus(StatusUserStopped) } default: - pl.SetStatus(StatusDegraded) - // we use %+v to get the stack trace too - pl.Error = fmt.Sprintf("%+v", err) + if cerrors.IsFatalError(err) { + pl.SetStatus(StatusDegraded) + // we use %+v to get the stack trace too + pl.Error = fmt.Sprintf("%+v", err) + } else { + pl.SetStatus(StatusRecovering) + // TODO: Implement backoff strategy + // check backoff strategy + + } } s.logger. diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index 95f910765..f871a691e 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -24,6 +24,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" + "github.com/jpillora/backoff" ) var idRegex = regexp.MustCompile(`^[A-Za-z0-9-_:.]*$`) @@ -51,15 +52,17 @@ type Service struct { instances map[string]*Instance instanceNames map[string]bool handlers []FailureHandler + backoffCfg *backoff.Backoff } // NewService initializes and returns a pipeline Service. -func NewService(logger log.CtxLogger, db database.DB) *Service { +func NewService(logger log.CtxLogger, db database.DB, backoffCfg *backoff.Backoff) *Service { return &Service{ logger: logger.WithComponent("pipeline.Service"), store: NewStore(db), instances: make(map[string]*Instance), instanceNames: make(map[string]bool), + backoffCfg: backoffCfg, } } From 93863610329ae04c6f013c0456df331aee34d39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 30 Aug 2024 16:42:20 +0200 Subject: [PATCH 02/41] update method calls --- pkg/orchestrator/orchestrator_test.go | 5 ++- pkg/pipeline/lifecycle.go | 2 -- pkg/pipeline/lifecycle_test.go | 27 ++++++++++------ pkg/pipeline/service_test.go | 45 ++++++++++++++++++--------- pkg/provisioning/service_test.go | 8 ++++- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/pkg/orchestrator/orchestrator_test.go b/pkg/orchestrator/orchestrator_test.go index e75deb2c9..c2d336514 100644 --- a/pkg/orchestrator/orchestrator_test.go +++ b/pkg/orchestrator/orchestrator_test.go @@ -37,6 +37,7 @@ import ( proc_builtin "github.com/conduitio/conduit/pkg/plugin/processor/builtin" "github.com/conduitio/conduit/pkg/processor" "github.com/google/go-cmp/cmp" + "github.com/jpillora/backoff" "github.com/matryer/is" "github.com/rs/zerolog" "go.uber.org/mock/gomock" @@ -90,10 +91,12 @@ func TestPipelineSimple(t *testing.T) { nil, ) + b := &backoff.Backoff{} + orc := NewOrchestrator( db, logger, - pipeline.NewService(logger, db), + pipeline.NewService(logger, db, b), connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)), processor.NewService(logger, db, procPluginService), connPluginService, diff --git a/pkg/pipeline/lifecycle.go b/pkg/pipeline/lifecycle.go index 35f749251..86fde345c 100644 --- a/pkg/pipeline/lifecycle.go +++ b/pkg/pipeline/lifecycle.go @@ -634,8 +634,6 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { } else { pl.SetStatus(StatusRecovering) // TODO: Implement backoff strategy - // check backoff strategy - } } diff --git a/pkg/pipeline/lifecycle_test.go b/pkg/pipeline/lifecycle_test.go index a71ae6bf3..124da0e00 100644 --- a/pkg/pipeline/lifecycle_test.go +++ b/pkg/pipeline/lifecycle_test.go @@ -35,6 +35,7 @@ import ( pmock "github.com/conduitio/conduit/pkg/plugin/connector/mock" "github.com/conduitio/conduit/pkg/processor" "github.com/google/uuid" + "github.com/jpillora/backoff" "github.com/matryer/is" "github.com/rs/zerolog" "go.uber.org/mock/gomock" @@ -50,8 +51,9 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) source := dummySource(persister) destination := dummyDestination(persister) @@ -133,8 +135,9 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) wantErr := "can't build pipeline without any source connectors" @@ -180,8 +183,9 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) wantErr := "can't build pipeline without any destination connectors" @@ -228,8 +232,9 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) defer persister.Wait() + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) @@ -287,8 +292,9 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { logger := log.Test(t) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) @@ -371,8 +377,9 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) @@ -472,8 +479,9 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { logger := log.New(zerolog.Nop()) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) @@ -533,8 +541,9 @@ func TestService_Run_Rerun(t *testing.T) { logger := log.Test(t) db := &inmemory.DB{} persister := connector.NewPersister(logger, db, time.Second, 3) + b := &backoff.Backoff{} - ps := NewService(logger, db) + ps := NewService(logger, db, b) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) @@ -571,7 +580,7 @@ func TestService_Run_Rerun(t *testing.T) { is.NoErr(err) // create a new pipeline service and initialize it - ps = NewService(logger, db) + ps = NewService(logger, db, b) err = ps.Init(ctx) is.NoErr(err) err = ps.Run( diff --git a/pkg/pipeline/service_test.go b/pkg/pipeline/service_test.go index a5a80502a..48533df6e 100644 --- a/pkg/pipeline/service_test.go +++ b/pkg/pipeline/service_test.go @@ -25,6 +25,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/google/uuid" + "github.com/jpillora/backoff" "github.com/matryer/is" "go.uber.org/mock/gomock" ) @@ -34,15 +35,16 @@ func TestService_Init_Simple(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) _, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) want := service.List(ctx) // create a new pipeline service and initialize it - service = NewService(logger, db) + service = NewService(logger, db, b) err = service.Init(ctx) is.NoErr(err) @@ -61,6 +63,7 @@ func TestService_Check(t *testing.T) { ctx := context.Background() logger := log.Nop() db := mock.NewDB(gomock.NewController(t)) + b := &backoff.Backoff{} testCases := []struct { name string @@ -80,7 +83,7 @@ func TestService_Check(t *testing.T) { t.Run(tc.name, func(t *testing.T) { is := is.New(t) db.EXPECT().Ping(gomock.Any()).Return(tc.wantErr) - service := NewService(logger, db) + service := NewService(logger, db, b) gotErr := service.Check(ctx) is.Equal(tc.wantErr, gotErr) @@ -92,8 +95,9 @@ func TestService_CreateSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) testCases := []struct { id string @@ -151,8 +155,9 @@ func TestService_Create_ValidateSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) testCases := []struct { name string @@ -193,8 +198,9 @@ func TestService_Create_ValidateError(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) testCases := []struct { name string @@ -262,8 +268,9 @@ func TestService_Create_PipelineNameExists(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) conf := Config{Name: "test-pipeline"} got, err := service.Create(ctx, uuid.NewString(), conf, ProvisionTypeAPI) @@ -279,8 +286,9 @@ func TestService_CreateEmptyName(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) got, err := service.Create(ctx, uuid.NewString(), Config{Name: ""}, ProvisionTypeAPI) is.True(err != nil) is.Equal(got, nil) @@ -291,8 +299,9 @@ func TestService_GetSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) want, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) @@ -306,8 +315,9 @@ func TestService_GetInstanceNotFound(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) // get pipeline instance that does not exist got, err := service.Get(ctx, uuid.NewString()) @@ -321,8 +331,9 @@ func TestService_DeleteSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) instance, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) @@ -339,8 +350,9 @@ func TestService_List(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) want := make(map[string]*Instance) for i := 0; i < 10; i++ { @@ -358,8 +370,9 @@ func TestService_UpdateSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) instance, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) @@ -378,8 +391,9 @@ func TestService_Update_PipelineNameExists(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) _, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) instance2, err2 := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline2"}, ProvisionTypeAPI) @@ -400,8 +414,9 @@ func TestService_UpdateInvalidConfig(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} + b := &backoff.Backoff{} - service := NewService(logger, db) + service := NewService(logger, db, b) instance, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) diff --git a/pkg/provisioning/service_test.go b/pkg/provisioning/service_test.go index 2491cd27a..e1308df99 100644 --- a/pkg/provisioning/service_test.go +++ b/pkg/provisioning/service_test.go @@ -40,6 +40,7 @@ import ( p4 "github.com/conduitio/conduit/pkg/provisioning/test/pipelines4-integration-test" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/jpillora/backoff" "github.com/matryer/is" "github.com/rs/zerolog" "go.uber.org/mock/gomock" @@ -513,8 +514,13 @@ func TestService_IntegrationTestServices(t *testing.T) { proc_builtin.NewRegistry(logger, proc_builtin.DefaultBuiltinProcessors, schemaRegistry), nil, ) + b := &backoff.Backoff{ + Factor: 2, + Min: time.Millisecond * 100, + Max: time.Second, // 8 tries + } - plService := pipeline.NewService(logger, db) + plService := pipeline.NewService(logger, db, b) connService := connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)) procService := processor.NewService(logger, db, procPluginService) From 51043b6731685f7b46db9f7edd7006750b627ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 3 Sep 2024 16:04:52 +0200 Subject: [PATCH 03/41] use configuration --- pkg/conduit/runtime.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index 8a2a7c347..6898288b5 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -198,11 +198,11 @@ func createServices(r *Runtime) error { tokenService, ) - // TODO: remove me once this is coming from configuration + errRecovery := r.Config.Pipelines.ErrorRecovery backoffCfg := &backoff.Backoff{ - Min: 100 * time.Millisecond, - Max: 10 * time.Second, - Factor: 2, + Min: errRecovery.MinDelay, + Max: errRecovery.MaxDelay, + Factor: float64(errRecovery.BackoffFactor), Jitter: true, } From b8602846e3f046b7442f2e307d4038306e542c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 6 Sep 2024 13:42:19 +0200 Subject: [PATCH 04/41] wip refactor --- pkg/{pipeline => lifecycle}/dlq.go | 2 +- pkg/{pipeline => lifecycle}/dlq_test.go | 2 +- .../lifecycle.go => lifecycle/service.go} | 101 +++++++++---- .../service_test.go} | 135 +++++++++--------- pkg/orchestrator/mock/orchestrator.go | 7 +- pkg/orchestrator/orchestrator.go | 3 +- pkg/pipeline/store.go | 2 + pkg/provisioning/interfaces.go | 3 +- pkg/provisioning/mock/provisioning.go | 7 +- 9 files changed, 156 insertions(+), 106 deletions(-) rename pkg/{pipeline => lifecycle}/dlq.go (99%) rename pkg/{pipeline => lifecycle}/dlq_test.go (99%) rename pkg/{pipeline/lifecycle.go => lifecycle/service.go} (87%) rename pkg/{pipeline/lifecycle_test.go => lifecycle/service_test.go} (85%) diff --git a/pkg/pipeline/dlq.go b/pkg/lifecycle/dlq.go similarity index 99% rename from pkg/pipeline/dlq.go rename to pkg/lifecycle/dlq.go index e602f2d44..5371da6a1 100644 --- a/pkg/pipeline/dlq.go +++ b/pkg/lifecycle/dlq.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package lifecycle import ( "bytes" diff --git a/pkg/pipeline/dlq_test.go b/pkg/lifecycle/dlq_test.go similarity index 99% rename from pkg/pipeline/dlq_test.go rename to pkg/lifecycle/dlq_test.go index 569cff8e8..32a54e8f6 100644 --- a/pkg/pipeline/dlq_test.go +++ b/pkg/lifecycle/dlq_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package lifecycle import ( "context" diff --git a/pkg/pipeline/lifecycle.go b/pkg/lifecycle/service.go similarity index 87% rename from pkg/pipeline/lifecycle.go rename to pkg/lifecycle/service.go index 86fde345c..c44e9f1ce 100644 --- a/pkg/pipeline/lifecycle.go +++ b/pkg/lifecycle/service.go @@ -1,4 +1,4 @@ -// Copyright © 2022 Meroxa, Inc. +// Copyright © 2024 Meroxa, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +// Package lifecycle wires up everything under the hood of a Conduit instance +// including metrics, telemetry, logging, and server construction. +// It should only ever interact with the Orchestrator, never individual +// services. All of that responsibility should be left to the Orchestrator. +package lifecycle import ( "context" @@ -23,17 +27,53 @@ import ( "sync/atomic" "time" + "github.com/conduitio/conduit-commons/database" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" + "github.com/conduitio/conduit/pkg/pipeline" "github.com/conduitio/conduit/pkg/pipeline/stream" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" + "github.com/jpillora/backoff" "gopkg.in/tomb.v2" ) +// Store handles the persistence and fetching of pipeline instances. +type Store struct { + db database.DB +} + +func NewStore(db database.DB) *Store { + return &Store{ + db: db, + } +} + +// Service manages pipelines. +type Service struct { + logger log.CtxLogger + + store *Store + + instances map[string]*pipeline.Instance + instanceNames map[string]bool + backoffCfg *backoff.Backoff +} + +// NewService initializes and returns a pipeline Service. +func NewService(logger log.CtxLogger, db database.DB, backoffCfg *backoff.Backoff) *Service { + return &Service{ + logger: logger.WithComponent("pipeline.Service"), + store: NewStore(db), + instances: make(map[string]*pipeline.Instance), + instanceNames: make(map[string]bool), + backoffCfg: backoffCfg, + } +} + // ConnectorFetcher can fetch a connector instance. type ConnectorFetcher interface { Get(ctx context.Context, id string) (*connector.Instance, error) @@ -63,7 +103,7 @@ func (s *Service) Run( // run pipelines that are in the StatusSystemStopped state for _, instance := range s.instances { - if instance.GetStatus() == StatusSystemStopped { + if instance.GetStatus() == pipeline.StatusSystemStopped { err := s.Start(ctx, connFetcher, procService, pluginFetcher, instance.ID) if err != nil { // try to start remaining pipelines and gather errors @@ -88,8 +128,8 @@ func (s *Service) Start( if err != nil { return err } - if pl.GetStatus() == StatusRunning { - return cerrors.Errorf("can't start pipeline %s: %w", pl.ID, ErrPipelineRunning) + if pl.GetStatus() == pipeline.StatusRunning { + return cerrors.Errorf("can't start pipeline %s: %w", pl.ID, pipeline.ErrPipelineRunning) } s.logger.Debug(ctx).Str(log.PipelineIDField, pl.ID).Msg("starting pipeline") @@ -124,8 +164,8 @@ func (s *Service) Stop(ctx context.Context, pipelineID string, force bool) error return err } - if pl.GetStatus() != StatusRunning && pl.GetStatus() != StatusRecovering { - return cerrors.Errorf("can't stop pipeline with status %q: %w", pl.GetStatus(), ErrPipelineNotRunning) + if pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering { + return cerrors.Errorf("can't stop pipeline with status %q: %w", pl.GetStatus(), pipeline.ErrPipelineNotRunning) } switch force { @@ -137,7 +177,7 @@ func (s *Service) Stop(ctx context.Context, pipelineID string, force bool) error panic("unreachable code") } -func (s *Service) stopGraceful(ctx context.Context, pl *Instance, reason error) error { +func (s *Service) stopGraceful(ctx context.Context, pl *pipeline.Instance, reason error) error { s.logger.Info(ctx). Str(log.PipelineIDField, pl.ID). Any(log.PipelineStatusField, pl.GetStatus()). @@ -157,12 +197,12 @@ func (s *Service) stopGraceful(ctx context.Context, pl *Instance, reason error) return cerrors.Join(errs...) } -func (s *Service) stopForceful(ctx context.Context, pl *Instance) error { +func (s *Service) stopForceful(ctx context.Context, pl *pipeline.Instance) error { s.logger.Info(ctx). Str(log.PipelineIDField, pl.ID). Any(log.PipelineStatusField, pl.GetStatus()). Msg("force stopping pipeline") - pl.t.Kill(ErrForceStop) + pl.t.Kill(pipeline.ErrForceStop) for _, n := range pl.n { if node, ok := n.(stream.ForceStoppableNode); ok { // stop all pub nodes @@ -177,7 +217,7 @@ func (s *Service) stopForceful(ctx context.Context, pl *Instance) error { // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { for _, pl := range s.instances { - if pl.GetStatus() != StatusRunning && pl.GetStatus() != StatusRecovering { + if pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering { continue } err := s.stopGraceful(ctx, pl, reason) @@ -211,7 +251,7 @@ func (s *Service) Wait(timeout time.Duration) error { case <-gracefullyStopped: return err case <-time.After(timeout): - return ErrTimeout + return pipeline.ErrTimeout } } @@ -234,7 +274,7 @@ func (s *Service) buildNodes( connFetcher ConnectorFetcher, procService ProcessorService, pluginFetcher PluginDispenserFetcher, - pl *Instance, + pl *pipeline.Instance, ) ([]stream.Node, error) { // setup many to many channels fanIn := stream.FaninNode{Name: "fanin"} @@ -282,7 +322,7 @@ func (s *Service) buildNodes( func (s *Service) buildProcessorNodes( ctx context.Context, procService ProcessorService, - pl *Instance, + pl *pipeline.Instance, processorIDs []string, first stream.PubNode, last stream.SubNode, @@ -319,7 +359,7 @@ func (s *Service) buildProcessorNodes( } func (s *Service) buildParallelProcessorNode( - pl *Instance, + pl *pipeline.Instance, proc *processor.RunnableProcessor, ) *stream.ParallelNode { return &stream.ParallelNode{ @@ -334,7 +374,7 @@ func (s *Service) buildParallelProcessorNode( } func (s *Service) buildProcessorNode( - pl *Instance, + pl *pipeline.Instance, proc *processor.RunnableProcessor, ) *stream.ProcessorNode { return &stream.ProcessorNode{ @@ -349,7 +389,7 @@ func (s *Service) buildSourceNodes( connFetcher ConnectorFetcher, procService ProcessorService, pluginFetcher PluginDispenserFetcher, - pl *Instance, + pl *pipeline.Instance, next stream.SubNode, ) ([]stream.Node, error) { var nodes []stream.Node @@ -417,7 +457,7 @@ func (s *Service) buildDLQHandlerNode( ctx context.Context, connFetcher ConnectorFetcher, pluginFetcher PluginDispenserFetcher, - pl *Instance, + pl *pipeline.Instance, ) (*stream.DLQHandlerNode, error) { conn, err := connFetcher.Create( ctx, @@ -461,7 +501,7 @@ func (s *Service) buildDLQHandlerNode( } func (s *Service) buildMetricsNode( - pl *Instance, + pl *pipeline.Instance, conn *connector.Instance, ) *stream.MetricsNode { return &stream.MetricsNode{ @@ -490,7 +530,7 @@ func (s *Service) buildDestinationNodes( connFetcher ConnectorFetcher, procService ProcessorService, pluginFetcher PluginDispenserFetcher, - pl *Instance, + pl *pipeline.Instance, prev stream.PubNode, ) ([]stream.Node, error) { var nodes []stream.Node @@ -536,9 +576,9 @@ func (s *Service) buildDestinationNodes( return nodes, nil } -func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { +func (s *Service) runPipeline(ctx context.Context, pl *pipeline.Instance) error { if pl.t != nil && pl.t.Alive() { - return ErrPipelineRunning + return pipeline.ErrPipelineRunning } // the tomb is responsible for running goroutines related to the pipeline @@ -577,7 +617,7 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { defer nodesWg.Done() err := node.Run(ctx) - if cerrors.Is(err, ErrGracefulShutdown) { + if cerrors.Is(err, pipeline.ErrGracefulShutdown) { // This node was shutdown because of ErrGracefulShutdown, we // need to stop this goroutine without returning an error to let // other nodes stop gracefully. We set a boolean that lets the @@ -594,7 +634,7 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { } measure.PipelinesGauge.WithValues(strings.ToLower(pl.GetStatus().String())).Dec() - pl.SetStatus(StatusRunning) + pl.SetStatus(pipeline.StatusRunning) pl.Error = "" measure.PipelinesGauge.WithValues(strings.ToLower(pl.GetStatus().String())).Inc() @@ -621,19 +661,22 @@ func (s *Service) runPipeline(ctx context.Context, pl *Instance) error { err = nil if isGracefulShutdown.Load() { // it was triggered by a graceful shutdown of Conduit - pl.SetStatus(StatusSystemStopped) + pl.SetStatus(pipeline.StatusSystemStopped) } else { // it was manually triggered by a user - pl.SetStatus(StatusUserStopped) + pl.SetStatus(pipeline.StatusUserStopped) } default: if cerrors.IsFatalError(err) { - pl.SetStatus(StatusDegraded) + pl.SetStatus(pipeline.StatusDegraded) // we use %+v to get the stack trace too pl.Error = fmt.Sprintf("%+v", err) } else { - pl.SetStatus(StatusRecovering) - // TODO: Implement backoff strategy + pl.SetStatus(pipeline.StatusRecovering) + err = s.store.Set(ctx, pl.ID, pl) + if err != nil { + return cerrors.Errorf("failed to save pipeline with ID %q: %w", pl.ID, err) + } } } diff --git a/pkg/pipeline/lifecycle_test.go b/pkg/lifecycle/service_test.go similarity index 85% rename from pkg/pipeline/lifecycle_test.go rename to pkg/lifecycle/service_test.go index 124da0e00..0d6cb0d61 100644 --- a/pkg/pipeline/lifecycle_test.go +++ b/pkg/lifecycle/service_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package lifecycle import ( "context" @@ -29,6 +29,7 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/pipeline" "github.com/conduitio/conduit/pkg/pipeline/stream" "github.com/conduitio/conduit/pkg/plugin" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" @@ -53,16 +54,16 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ls := NewService(logger, db, b) source := dummySource(persister) destination := dummyDestination(persister) dlq := dummyDestination(persister) - pl := &Instance{ + pl := &pipeline.Instance{ ID: uuid.NewString(), - Config: Config{Name: "test-pipeline"}, - status: StatusUserStopped, - DLQ: DLQ{ + Config: pipeline.Config{Name: "test-pipeline"}, + status: pipeline.StatusUserStopped, + DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, WindowSize: 3, @@ -71,7 +72,7 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { ConnectorIDs: []string{source.ID, destination.ID}, } - got, err := ps.buildNodes( + got, err := ls.buildNodes( ctx, testConnectorFetcher{ source.ID: source, @@ -137,17 +138,17 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ls := NewService(logger, db, b) wantErr := "can't build pipeline without any source connectors" destination := dummyDestination(persister) dlq := dummyDestination(persister) - pl := &Instance{ + pl := &pipeline.Instance{ ID: uuid.NewString(), - Config: Config{Name: "test-pipeline"}, - status: StatusUserStopped, - DLQ: DLQ{ + Config: pipeline.Config{Name: "test-pipeline"}, + status: pipeline.StatusUserStopped, + DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, WindowSize: 3, @@ -156,7 +157,7 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { ConnectorIDs: []string{destination.ID}, } - got, err := ps.buildNodes( + got, err := ls.buildNodes( ctx, testConnectorFetcher{ destination.ID: destination, @@ -185,18 +186,18 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ls := NewService(logger, db, b) wantErr := "can't build pipeline without any destination connectors" source := dummySource(persister) dlq := dummyDestination(persister) - pl := &Instance{ + pl := &pipeline.Instance{ ID: uuid.NewString(), - Config: Config{Name: "test-pipeline"}, - status: StatusUserStopped, - DLQ: DLQ{ + Config: pipeline.Config{Name: "test-pipeline"}, + status: pipeline.StatusUserStopped, + DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, WindowSize: 3, @@ -205,7 +206,7 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { ConnectorIDs: []string{source.ID}, } - got, err := ps.buildNodes( + got, err := ls.buildNodes( ctx, testConnectorFetcher{ source.ID: source, @@ -234,10 +235,10 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { defer persister.Wait() b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ls := NewService(logger, db, b) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ls.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -248,13 +249,13 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { dlq, dlqDispenser := asserterDestination(ctrl, persister, nil) pl.DLQ.Plugin = dlq.Plugin - pl, err = ps.AddConnector(ctx, pl.ID, source.ID) + pl, err = ls.AddConnector(ctx, pl.ID, source.ID) is.NoErr(err) - pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) + pl, err = ls.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) // start the pipeline now that everything is set up - err = ps.Start( + err = ls.Start( ctx, testConnectorFetcher{ source.ID: source, @@ -274,11 +275,11 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { // wait for pipeline to finish consuming records from the source time.Sleep(100 * time.Millisecond) - is.Equal(StatusRunning, pl.GetStatus()) + is.Equal(pipeline.StatusRunning, pl.GetStatus()) is.Equal("", pl.Error) // stop pipeline before ending test - err = ps.Stop(ctx, pl.ID, false) + err = ls.Stop(ctx, pl.ID, false) is.NoErr(err) is.NoErr(pl.Wait()) } @@ -294,10 +295,10 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ps := pipeline.NewService(logger, db, b) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -314,8 +315,8 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - events := make(chan FailureEvent, 1) - ps.OnFailure(func(e FailureEvent) { + events := make(chan pipeline.FailureEvent, 1) + ps.OnFailure(func(e pipeline.FailureEvent) { events <- e }) // start the pipeline now that everything is set up @@ -340,11 +341,11 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { err = pl.Wait() is.True(err != nil) - is.Equal(StatusDegraded, pl.GetStatus()) + is.Equal(pipeline.StatusDegraded, pl.GetStatus()) // pipeline errors contain only string messages, so we can only compare the errors by the messages t.Log(pl.Error) - event, eventReceived, err := cchan.Chan[FailureEvent](events).RecvTimeout(ctx, 200*time.Millisecond) + event, eventReceived, err := cchan.Chan[pipeline.FailureEvent](events).RecvTimeout(ctx, 200*time.Millisecond) is.NoErr(err) is.True(eventReceived) is.Equal(pl.ID, event.ID) @@ -362,11 +363,11 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { type testCase struct { name string - stopFn func(ctx context.Context, is *is.I, pipelineService *Service, pipelineID string) + stopFn func(ctx context.Context, is *is.I, pipelineService *pipeline.Service, pipelineID string) // whether we expect the source plugin's Stop() function to be called // (doesn't happen when force-stopping) wantSourceStop bool - want Status + want pipeline.Status wantErr error } @@ -379,10 +380,10 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ps := pipeline.NewService(logger, db, b) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -421,7 +422,7 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { // wait for pipeline to finish consuming records from the source time.Sleep(100 * time.Millisecond) - pl.SetStatus(StatusRecovering) + pl.SetStatus(pipeline.StatusRecovering) tc.stopFn(ctx, is, ps, pl.ID) // wait for pipeline to finish @@ -439,29 +440,29 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { testCases := []testCase{ { name: "system stop (graceful shutdown err)", - stopFn: func(ctx context.Context, is *is.I, ps *Service, pipelineID string) { - ps.StopAll(ctx, ErrGracefulShutdown) + stopFn: func(ctx context.Context, is *is.I, ps *pipeline.Service, pipelineID string) { + ps.StopAll(ctx, pipeline.ErrGracefulShutdown) }, wantSourceStop: true, - want: StatusSystemStopped, + want: pipeline.StatusSystemStopped, }, { name: "system stop (terrible err)", - stopFn: func(ctx context.Context, is *is.I, ps *Service, pipelineID string) { + stopFn: func(ctx context.Context, is *is.I, ps *pipeline.Service, pipelineID string) { ps.StopAll(ctx, cerrors.New("terrible err")) }, wantSourceStop: true, - want: StatusDegraded, + want: pipeline.StatusDegraded, wantErr: cerrors.New("terrible err"), }, { name: "user stop (graceful)", - stopFn: func(ctx context.Context, is *is.I, ps *Service, pipelineID string) { + stopFn: func(ctx context.Context, is *is.I, ps *pipeline.Service, pipelineID string) { err := ps.Stop(ctx, pipelineID, false) is.NoErr(err) }, wantSourceStop: true, - want: StatusUserStopped, + want: pipeline.StatusUserStopped, }, } @@ -481,10 +482,10 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ps := pipeline.NewService(logger, db, b) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -522,18 +523,18 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { // wait for pipeline to finish consuming records from the source time.Sleep(100 * time.Millisecond) - ps.StopAll(ctx, ErrGracefulShutdown) + ps.StopAll(ctx, pipeline.ErrGracefulShutdown) // wait for pipeline to finish err = pl.Wait() is.NoErr(err) - is.Equal(StatusSystemStopped, pl.GetStatus()) + is.Equal(pipeline.StatusSystemStopped, pl.GetStatus()) is.Equal("", pl.Error) } func TestService_Run_Rerun(t *testing.T) { - runTest := func(t *testing.T, status Status, expected Status) { + runTest := func(t *testing.T, status pipeline.Status, expected pipeline.Status) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -543,10 +544,10 @@ func TestService_Run_Rerun(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := NewService(logger, db, b) + ls := NewService(logger, db, b) // create a host pipeline - pl, err := ps.Create(ctx, uuid.NewString(), Config{Name: "test pipeline"}, ProvisionTypeAPI) + pl, err := ls.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -558,7 +559,7 @@ func TestService_Run_Rerun(t *testing.T) { dlq *connector.Instance dlqDispenser *pmock.Dispenser ) - if expected == StatusRunning { + if expected == pipeline.StatusRunning { // mocked connectors that are expected to be started source, sourceDispenser = generatorSource(ctrl, persister, nil, nil, true) destination, destDispenser = asserterDestination(ctrl, persister, nil) @@ -574,16 +575,16 @@ func TestService_Run_Rerun(t *testing.T) { pl.DLQ.Plugin = dlq.Plugin pl.SetStatus(status) - pl, err = ps.AddConnector(ctx, pl.ID, source.ID) + pl, err = ls.AddConnector(ctx, pl.ID, source.ID) is.NoErr(err) - pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) + pl, err = ls.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) // create a new pipeline service and initialize it - ps = NewService(logger, db, b) - err = ps.Init(ctx) + ls = pipeline.NewService(logger, db, b) + err = ls.Init(ctx) is.NoErr(err) - err = ps.Run( + err = ls.Run( ctx, testConnectorFetcher{ source.ID: source, @@ -602,26 +603,26 @@ func TestService_Run_Rerun(t *testing.T) { // give pipeline a chance to start if needed time.Sleep(time.Millisecond * 100) - got := ps.List(ctx) + got := ls.List(ctx) is.Equal(len(got), 1) is.True(got[pl.ID] != nil) is.Equal(got[pl.ID].GetStatus(), expected) - if expected == StatusRunning { - pl, _ = ps.Get(ctx, pl.ID) - is.NoErr(ps.Stop(ctx, pl.ID, false)) + if expected == pipeline.StatusRunning { + pl, _ = ls.Get(ctx, pl.ID) + is.NoErr(ls.Stop(ctx, pl.ID, false)) is.NoErr(pl.Wait()) } } testCases := []struct { - have Status - want Status + have pipeline.Status + want pipeline.Status }{ - {have: StatusRunning, want: StatusRunning}, - {have: StatusUserStopped, want: StatusUserStopped}, - {have: StatusSystemStopped, want: StatusRunning}, - {have: StatusDegraded, want: StatusDegraded}, + {have: pipeline.StatusRunning, want: pipeline.StatusRunning}, + {have: pipeline.StatusUserStopped, want: pipeline.StatusUserStopped}, + {have: pipeline.StatusSystemStopped, want: pipeline.StatusRunning}, + {have: pipeline.StatusDegraded, want: pipeline.StatusDegraded}, } for _, tt := range testCases { t.Run(fmt.Sprintf("%s->%s", tt.have, tt.want), func(t *testing.T) { diff --git a/pkg/orchestrator/mock/orchestrator.go b/pkg/orchestrator/mock/orchestrator.go index 25a5645f3..b8875a5bd 100644 --- a/pkg/orchestrator/mock/orchestrator.go +++ b/pkg/orchestrator/mock/orchestrator.go @@ -17,6 +17,7 @@ import ( sdk "github.com/conduitio/conduit-processor-sdk" connector "github.com/conduitio/conduit/pkg/connector" log "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" pipeline "github.com/conduitio/conduit/pkg/pipeline" connector0 "github.com/conduitio/conduit/pkg/plugin/connector" processor "github.com/conduitio/conduit/pkg/processor" @@ -357,7 +358,7 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, } // Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 pipeline.ConnectorFetcher, arg2 pipeline.ProcessorService, arg3 pipeline.PluginDispenserFetcher, arg4 string) error { +func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorFetcher, arg2 lifecycle.ProcessorService, arg3 lifecycle.PluginDispenserFetcher, arg4 string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) @@ -383,13 +384,13 @@ func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall } // Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index d53d8390d..72809ad92 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -24,6 +24,7 @@ import ( processorSdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/pipeline" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" @@ -77,7 +78,7 @@ type base struct { } type PipelineService interface { - Start(ctx context.Context, connFetcher pipeline.ConnectorFetcher, procService pipeline.ProcessorService, pluginFetcher pipeline.PluginDispenserFetcher, pipelineID string) error + Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error // Stop initiates a stop of the given pipeline. The method does not wait for // the pipeline (and its nodes) to actually stop. // When force is false the pipeline will try to stop gracefully and drain diff --git a/pkg/pipeline/store.go b/pkg/pipeline/store.go index 41ac88ff8..bd08860db 100644 --- a/pkg/pipeline/store.go +++ b/pkg/pipeline/store.go @@ -31,11 +31,13 @@ const ( storeKeyPrefix = "pipeline:instance:" ) +// TODO: remove? // Store handles the persistence and fetching of pipeline instances. type Store struct { db database.DB } +// TODO: remove? func NewStore(db database.DB) *Store { return &Store{ db: db, diff --git a/pkg/provisioning/interfaces.go b/pkg/provisioning/interfaces.go index d4e8e6f72..09ca663cb 100644 --- a/pkg/provisioning/interfaces.go +++ b/pkg/provisioning/interfaces.go @@ -19,6 +19,7 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/pipeline" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" @@ -34,7 +35,7 @@ type PipelineService interface { Delete(ctx context.Context, pipelineID string) error UpdateDLQ(ctx context.Context, pipelineID string, cfg pipeline.DLQ) (*pipeline.Instance, error) - Start(ctx context.Context, connFetcher pipeline.ConnectorFetcher, procService pipeline.ProcessorService, pluginFetcher pipeline.PluginDispenserFetcher, pipelineID string) error + Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error Stop(ctx context.Context, pipelineID string, force bool) error AddConnector(ctx context.Context, pipelineID string, connectorID string) (*pipeline.Instance, error) diff --git a/pkg/provisioning/mock/provisioning.go b/pkg/provisioning/mock/provisioning.go index e7c8ce85c..d3f9ffcc1 100644 --- a/pkg/provisioning/mock/provisioning.go +++ b/pkg/provisioning/mock/provisioning.go @@ -15,6 +15,7 @@ import ( connector "github.com/conduitio/conduit/pkg/connector" log "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" pipeline "github.com/conduitio/conduit/pkg/pipeline" connector0 "github.com/conduitio/conduit/pkg/plugin/connector" processor "github.com/conduitio/conduit/pkg/processor" @@ -355,7 +356,7 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, } // Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 pipeline.ConnectorFetcher, arg2 pipeline.ProcessorService, arg3 pipeline.PluginDispenserFetcher, arg4 string) error { +func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorFetcher, arg2 lifecycle.ProcessorService, arg3 lifecycle.PluginDispenserFetcher, arg4 string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) @@ -381,13 +382,13 @@ func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall } // Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, pipeline.ConnectorFetcher, pipeline.ProcessorService, pipeline.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { c.Call = c.Call.DoAndReturn(f) return c } From fd66596a0873d09f6cd983cb9a8308c0ad26cdbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 6 Sep 2024 16:51:06 +0200 Subject: [PATCH 05/41] wip --- pkg/lifecycle/service.go | 42 ++++++++++++++++-------- pkg/lifecycle/service_test.go | 55 +++++++++++++++++++------------- pkg/orchestrator/orchestrator.go | 25 +++++++++------ pkg/pipeline/service.go | 7 +++- pkg/provisioning/service.go | 2 ++ 5 files changed, 85 insertions(+), 46 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index c44e9f1ce..0c2f93123 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -58,19 +58,26 @@ type Service struct { store *Store - instances map[string]*pipeline.Instance - instanceNames map[string]bool - backoffCfg *backoff.Backoff + // PipelineService + //instances map[string]*pipeline.Instance + //instanceNames map[string]bool + backoffCfg *backoff.Backoff + + // TODO: + // add the other services + pipelines pipeline.Service + connectors ConnectorService + processors ProcessorService + connectorPlugins ConnectorPluginService + processorPlugins ProcessorPluginService } // NewService initializes and returns a pipeline Service. func NewService(logger log.CtxLogger, db database.DB, backoffCfg *backoff.Backoff) *Service { return &Service{ - logger: logger.WithComponent("pipeline.Service"), - store: NewStore(db), - instances: make(map[string]*pipeline.Instance), - instanceNames: make(map[string]bool), - backoffCfg: backoffCfg, + logger: logger.WithComponent("pipeline.Service"), + store: NewStore(db), + backoffCfg: backoffCfg, } } @@ -91,20 +98,28 @@ type PluginDispenserFetcher interface { NewDispenser(logger log.CtxLogger, name string, connectorID string) (connectorPlugin.Dispenser, error) } +// PipelineService can fetch a pipeline instance. +type PipelineService interface { + Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) + GetInstances() map[string]*pipeline.Instance +} + // Run runs pipelines that had the running state in store. func (s *Service) Run( ctx context.Context, connFetcher ConnectorFetcher, procService ProcessorService, pluginFetcher PluginDispenserFetcher, + pipelineService PipelineService, ) error { var errs []error s.logger.Debug(ctx).Msg("initializing pipelines statuses") // run pipelines that are in the StatusSystemStopped state - for _, instance := range s.instances { + instances := pipelineService.GetInstances() + for _, instance := range instances { if instance.GetStatus() == pipeline.StatusSystemStopped { - err := s.Start(ctx, connFetcher, procService, pluginFetcher, instance.ID) + err := s.Start(ctx, connFetcher, procService, pluginFetcher, pipelineService, instance.ID) if err != nil { // try to start remaining pipelines and gather errors errs = append(errs, err) @@ -122,9 +137,10 @@ func (s *Service) Start( connFetcher ConnectorFetcher, procService ProcessorService, pluginFetcher PluginDispenserFetcher, + ps PipelineService, pipelineID string, ) error { - pl, err := s.Get(ctx, pipelineID) + pl, err := ps.Get(ctx, pipelineID) if err != nil { return err } @@ -158,8 +174,8 @@ func (s *Service) Start( // Stop function. If force is set to true the pipeline won't stop gracefully, // instead the context for all nodes will be canceled which causes them to stop // running as soon as possible. -func (s *Service) Stop(ctx context.Context, pipelineID string, force bool) error { - pl, err := s.Get(ctx, pipelineID) +func (s *Service) Stop(ctx context.Context, pipelineID string, force bool, ps PipelineService) error { + pl, err := ps.Get(ctx, pipelineID) if err != nil { return err } diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 0d6cb0d61..1fb982512 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -62,7 +62,6 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { pl := &pipeline.Instance{ ID: uuid.NewString(), Config: pipeline.Config{Name: "test-pipeline"}, - status: pipeline.StatusUserStopped, DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, @@ -71,6 +70,7 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { }, ConnectorIDs: []string{source.ID, destination.ID}, } + pl.SetStatus(pipeline.StatusUserStopped) got, err := ls.buildNodes( ctx, @@ -147,7 +147,6 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { pl := &pipeline.Instance{ ID: uuid.NewString(), Config: pipeline.Config{Name: "test-pipeline"}, - status: pipeline.StatusUserStopped, DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, @@ -156,6 +155,7 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { }, ConnectorIDs: []string{destination.ID}, } + pl.SetStatus(pipeline.StatusUserStopped) got, err := ls.buildNodes( ctx, @@ -196,7 +196,6 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { pl := &pipeline.Instance{ ID: uuid.NewString(), Config: pipeline.Config{Name: "test-pipeline"}, - status: pipeline.StatusUserStopped, DLQ: pipeline.DLQ{ Plugin: dlq.Plugin, Settings: map[string]string{}, @@ -205,6 +204,7 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { }, ConnectorIDs: []string{source.ID}, } + pl.SetStatus(pipeline.StatusUserStopped) got, err := ls.buildNodes( ctx, @@ -235,10 +235,10 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { defer persister.Wait() b := &backoff.Backoff{} - ls := NewService(logger, db, b) + ps := pipeline.NewService(logger, db, b) // create a host pipeline - pl, err := ls.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -249,11 +249,13 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { dlq, dlqDispenser := asserterDestination(ctrl, persister, nil) pl.DLQ.Plugin = dlq.Plugin - pl, err = ls.AddConnector(ctx, pl.ID, source.ID) + pl, err = ps.AddConnector(ctx, pl.ID, source.ID) is.NoErr(err) - pl, err = ls.AddConnector(ctx, pl.ID, destination.ID) + pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) + ls := NewService(logger, db, b) + // start the pipeline now that everything is set up err = ls.Start( ctx, @@ -319,8 +321,11 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { ps.OnFailure(func(e pipeline.FailureEvent) { events <- e }) + + ls := NewService(logger, db, b) + // start the pipeline now that everything is set up - err = ps.Start( + err = ls.Start( ctx, testConnectorFetcher{ source.ID: source, @@ -363,7 +368,7 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { type testCase struct { name string - stopFn func(ctx context.Context, is *is.I, pipelineService *pipeline.Service, pipelineID string) + stopFn func(ctx context.Context, is *is.I, lifecycleService *Service, pipelineID string) // whether we expect the source plugin's Stop() function to be called // (doesn't happen when force-stopping) wantSourceStop bool @@ -401,8 +406,10 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) + ls := NewService(logger, db, b) + // start the pipeline now that everything is set up - err = ps.Start( + err = ls.Start( ctx, testConnectorFetcher{ source.ID: source, @@ -440,16 +447,16 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { testCases := []testCase{ { name: "system stop (graceful shutdown err)", - stopFn: func(ctx context.Context, is *is.I, ps *pipeline.Service, pipelineID string) { - ps.StopAll(ctx, pipeline.ErrGracefulShutdown) + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + ls.StopAll(ctx, pipeline.ErrGracefulShutdown) }, wantSourceStop: true, want: pipeline.StatusSystemStopped, }, { name: "system stop (terrible err)", - stopFn: func(ctx context.Context, is *is.I, ps *pipeline.Service, pipelineID string) { - ps.StopAll(ctx, cerrors.New("terrible err")) + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + ls.StopAll(ctx, cerrors.New("terrible err")) }, wantSourceStop: true, want: pipeline.StatusDegraded, @@ -544,10 +551,10 @@ func TestService_Run_Rerun(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ls := NewService(logger, db, b) + ps := pipeline.NewService(logger, db, b) // create a host pipeline - pl, err := ls.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) + pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) is.NoErr(err) // create mocked connectors @@ -575,15 +582,17 @@ func TestService_Run_Rerun(t *testing.T) { pl.DLQ.Plugin = dlq.Plugin pl.SetStatus(status) - pl, err = ls.AddConnector(ctx, pl.ID, source.ID) + pl, err = ps.AddConnector(ctx, pl.ID, source.ID) is.NoErr(err) - pl, err = ls.AddConnector(ctx, pl.ID, destination.ID) + pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) // create a new pipeline service and initialize it - ls = pipeline.NewService(logger, db, b) - err = ls.Init(ctx) + ps = pipeline.NewService(logger, db, b) + err = ps.Init(ctx) is.NoErr(err) + + // TODO: create a new instance of the lifecycle service err = ls.Run( ctx, testConnectorFetcher{ @@ -603,13 +612,15 @@ func TestService_Run_Rerun(t *testing.T) { // give pipeline a chance to start if needed time.Sleep(time.Millisecond * 100) - got := ls.List(ctx) + got := ps.List(ctx) is.Equal(len(got), 1) is.True(got[pl.ID] != nil) is.Equal(got[pl.ID].GetStatus(), expected) if expected == pipeline.StatusRunning { - pl, _ = ls.Get(ctx, pl.ID) + pl, _ = ps.Get(ctx, pl.ID) + + // TODO: create a new instance of the lifecycle service is.NoErr(ls.Stop(ctx, pl.ID, false)) is.NoErr(pl.Wait()) } diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index 72809ad92..dd0386a48 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -46,6 +46,7 @@ func NewOrchestrator( processors ProcessorService, connectorPlugins ConnectorPluginService, processorPlugins ProcessorPluginService, + lifecycle LifecycleService, ) *Orchestrator { b := base{ db: db, @@ -55,6 +56,7 @@ func NewOrchestrator( processors: processors, connectorPlugins: connectorPlugins, processorPlugins: processorPlugins, + lifecycle: lifecycle, } return &Orchestrator{ @@ -75,19 +77,10 @@ type base struct { processors ProcessorService connectorPlugins ConnectorPluginService processorPlugins ProcessorPluginService + lifecycle LifecycleService } type PipelineService interface { - Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error - // Stop initiates a stop of the given pipeline. The method does not wait for - // the pipeline (and its nodes) to actually stop. - // When force is false the pipeline will try to stop gracefully and drain - // any in-flight messages that have not yet reached the destination. When - // force is true the pipeline will stop without draining in-flight messages. - // It is allowed to execute a force stop even after a graceful stop was - // requested. - Stop(ctx context.Context, pipelineID string, force bool) error - List(ctx context.Context) map[string]*pipeline.Instance Get(ctx context.Context, id string) (*pipeline.Instance, error) Create(ctx context.Context, id string, cfg pipeline.Config, p pipeline.ProvisionType) (*pipeline.Instance, error) @@ -133,3 +126,15 @@ type ProcessorPluginService interface { NewProcessor(ctx context.Context, pluginName string, id string) (processorSdk.Processor, error) RegisterStandalonePlugin(ctx context.Context, path string) (string, error) } + +type LifecycleService interface { + Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error + // Stop initiates a stop of the given pipeline. The method does not wait for + // the pipeline (and its nodes) to actually stop. + // When force is false the pipeline will try to stop gracefully and drain + // any in-flight messages that have not yet reached the destination. When + // force is true the pipeline will stop without draining in-flight messages. + // It is allowed to execute a force stop even after a graceful stop was + // requested. + Stop(ctx context.Context, pipelineID string, force bool) error +} diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index f871a691e..34f8183b9 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -121,7 +121,7 @@ func (s *Service) Get(_ context.Context, id string) (*Instance, error) { } // Create will create a new pipeline instance with the given config and return -// it if it was successfully saved to the database. +// if it was successfully saved to the database. func (s *Service) Create(ctx context.Context, id string, cfg Config, p ProvisionType) (*Instance, error) { err := s.validatePipeline(cfg, id) if err != nil { @@ -366,3 +366,8 @@ func (s *Service) validatePipeline(cfg Config, id string) error { return cerrors.Join(errs...) } + +// GetInstances returns all pipeline instances. +func (s *Service) GetInstances() map[string]*Instance { + return s.instances +} diff --git a/pkg/provisioning/service.go b/pkg/provisioning/service.go index c0b3bcec6..0e35a0aa3 100644 --- a/pkg/provisioning/service.go +++ b/pkg/provisioning/service.go @@ -253,6 +253,8 @@ func (s *Service) provisionPipeline(ctx context.Context, cfg config.Pipeline) er // check if pipeline should be running if cfg.Status == config.StatusRunning { // TODO set status and let the pipeline service start it + + // TODO: Use the lifecycle service (not the pipeline service) to start the pipeline err := s.pipelineService.Start(ctx, s.connectorService, s.processorService, s.connectorPluginService, cfg.ID) if err != nil { return cerrors.Errorf("could not start the pipeline %q: %w", cfg.ID, err) From 8e2de1b2da124bb2dca123707b82c48e57b646c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 9 Sep 2024 13:38:21 +0200 Subject: [PATCH 06/41] move stream --- pkg/conduit/runtime.go | 7 ++-- pkg/lifecycle/service.go | 42 ++++++++++++++----- pkg/{pipeline => lifecycle}/stream/base.go | 0 .../stream/base_test.go | 0 .../stream/destination.go | 0 .../stream/destination_acker.go | 0 .../stream/destination_acker_test.go | 0 .../stream/destination_test.go | 0 pkg/{pipeline => lifecycle}/stream/dlq.go | 0 .../stream/dlq_test.go | 0 pkg/{pipeline => lifecycle}/stream/doc.go | 0 pkg/{pipeline => lifecycle}/stream/fanin.go | 0 .../stream/fanin_select.go | 0 pkg/{pipeline => lifecycle}/stream/fanout.go | 0 .../stream/fanout_test.go | 0 pkg/{pipeline => lifecycle}/stream/logger.go | 0 pkg/{pipeline => lifecycle}/stream/message.go | 0 .../stream/message_test.go | 0 .../stream/messagestatus_string.go | 0 pkg/{pipeline => lifecycle}/stream/metrics.go | 0 .../stream/mock/destination.go | 0 .../stream/mock/dlq.go | 0 .../stream/mock/processor.go | 0 .../stream/mock/source.go | 0 pkg/{pipeline => lifecycle}/stream/node.go | 0 .../stream/parallel.go | 0 .../stream/parallel_test.go | 0 .../stream/processor.go | 0 .../stream/processor_test.go | 0 pkg/{pipeline => lifecycle}/stream/source.go | 0 .../stream/source_acker.go | 0 .../stream/source_acker_test.go | 0 .../stream/source_test.go | 0 .../stream/stream_test.go | 0 pkg/orchestrator/orchestrator.go | 2 +- pkg/pipeline/service.go | 27 ++---------- pkg/provisioning/interfaces.go | 8 ++-- pkg/provisioning/service.go | 4 +- 38 files changed, 47 insertions(+), 43 deletions(-) rename pkg/{pipeline => lifecycle}/stream/base.go (100%) rename pkg/{pipeline => lifecycle}/stream/base_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/destination.go (100%) rename pkg/{pipeline => lifecycle}/stream/destination_acker.go (100%) rename pkg/{pipeline => lifecycle}/stream/destination_acker_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/destination_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/dlq.go (100%) rename pkg/{pipeline => lifecycle}/stream/dlq_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/doc.go (100%) rename pkg/{pipeline => lifecycle}/stream/fanin.go (100%) rename pkg/{pipeline => lifecycle}/stream/fanin_select.go (100%) rename pkg/{pipeline => lifecycle}/stream/fanout.go (100%) rename pkg/{pipeline => lifecycle}/stream/fanout_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/logger.go (100%) rename pkg/{pipeline => lifecycle}/stream/message.go (100%) rename pkg/{pipeline => lifecycle}/stream/message_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/messagestatus_string.go (100%) rename pkg/{pipeline => lifecycle}/stream/metrics.go (100%) rename pkg/{pipeline => lifecycle}/stream/mock/destination.go (100%) rename pkg/{pipeline => lifecycle}/stream/mock/dlq.go (100%) rename pkg/{pipeline => lifecycle}/stream/mock/processor.go (100%) rename pkg/{pipeline => lifecycle}/stream/mock/source.go (100%) rename pkg/{pipeline => lifecycle}/stream/node.go (100%) rename pkg/{pipeline => lifecycle}/stream/parallel.go (100%) rename pkg/{pipeline => lifecycle}/stream/parallel_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/processor.go (100%) rename pkg/{pipeline => lifecycle}/stream/processor_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/source.go (100%) rename pkg/{pipeline => lifecycle}/stream/source_acker.go (100%) rename pkg/{pipeline => lifecycle}/stream/source_acker_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/source_test.go (100%) rename pkg/{pipeline => lifecycle}/stream/stream_test.go (100%) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index 6898288b5..acea21e79 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -44,6 +44,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/metrics" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" "github.com/conduitio/conduit/pkg/foundation/metrics/prometheus" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/orchestrator" "github.com/conduitio/conduit/pkg/pipeline" conn_plugin "github.com/conduitio/conduit/pkg/plugin/connector" @@ -209,10 +210,10 @@ func createServices(r *Runtime) error { plService := pipeline.NewService(r.logger, r.DB, backoffCfg) connService := connector.NewService(r.logger, r.DB, r.connectorPersister) procService := processor.NewService(r.logger, r.DB, procPluginService) - + lifecycleService := lifecycle.NewService(r.logger, r.DB, backoffCfg) provisionService := provisioning.NewService(r.DB, r.logger, plService, connService, procService, connPluginService, r.Config.Pipelines.Path) - orc := orchestrator.NewOrchestrator(r.DB, r.logger, plService, connService, procService, connPluginService, procPluginService) + orc := orchestrator.NewOrchestrator(r.DB, r.logger, plService, connService, procService, connPluginService, procPluginService, lifecycleService) r.Orchestrator = orc r.ProvisionService = provisionService @@ -762,7 +763,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } if r.Config.Pipelines.ExitOnError { - r.pipelineService.OnFailure(func(e pipeline.FailureEvent) { + r.pipelineService.OnFailure(func(e lifecycle.FailureEvent) { r.logger.Warn(ctx). Err(e.Error). Str(log.PipelineIDField, e.ID). diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 0c2f93123..a52ec6e18 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -33,14 +33,22 @@ import ( "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" + "github.com/conduitio/conduit/pkg/lifecycle/stream" "github.com/conduitio/conduit/pkg/pipeline" - "github.com/conduitio/conduit/pkg/pipeline/stream" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" "github.com/jpillora/backoff" "gopkg.in/tomb.v2" ) +type FailureEvent struct { + // ID is the ID of the pipeline which failed. + ID string + Error error +} + +type FailureHandler func(FailureEvent) + // Store handles the persistence and fetching of pipeline instances. type Store struct { db database.DB @@ -63,13 +71,11 @@ type Service struct { //instanceNames map[string]bool backoffCfg *backoff.Backoff - // TODO: - // add the other services - pipelines pipeline.Service - connectors ConnectorService - processors ProcessorService - connectorPlugins ConnectorPluginService - processorPlugins ProcessorPluginService + pipelines pipeline.Service + //connectors ConnectorService + processors ProcessorService + //connectorPlugins ConnectorPluginService + //processorPlugins ProcessorPluginService } // NewService initializes and returns a pipeline Service. @@ -232,7 +238,8 @@ func (s *Service) stopForceful(ctx context.Context, pl *pipeline.Instance) error // StopAll will ask all the pipelines to stop gracefully // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { - for _, pl := range s.instances { + instances := s.pipelines.GetInstances() + for _, pl := range instances { if pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering { continue } @@ -275,7 +282,8 @@ func (s *Service) Wait(timeout time.Duration) error { // the pipelines failed to stop gracefully. func (s *Service) waitInternal() error { var errs []error - for _, pl := range s.instances { + instances := s.pipelines.GetInstances() + for _, pl := range instances { err := pl.Wait() if err != nil { errs = append(errs, err) @@ -716,3 +724,17 @@ func (s *Service) runPipeline(ctx context.Context, pl *pipeline.Instance) error return nil } + +// notify notifies all registered FailureHandlers about an error. +func (s *Service) notify(pipelineID string, err error) { + if err == nil { + return + } + e := FailureEvent{ + ID: pipelineID, + Error: err, + } + for _, handler := range s.handlers { + handler(e) + } +} diff --git a/pkg/pipeline/stream/base.go b/pkg/lifecycle/stream/base.go similarity index 100% rename from pkg/pipeline/stream/base.go rename to pkg/lifecycle/stream/base.go diff --git a/pkg/pipeline/stream/base_test.go b/pkg/lifecycle/stream/base_test.go similarity index 100% rename from pkg/pipeline/stream/base_test.go rename to pkg/lifecycle/stream/base_test.go diff --git a/pkg/pipeline/stream/destination.go b/pkg/lifecycle/stream/destination.go similarity index 100% rename from pkg/pipeline/stream/destination.go rename to pkg/lifecycle/stream/destination.go diff --git a/pkg/pipeline/stream/destination_acker.go b/pkg/lifecycle/stream/destination_acker.go similarity index 100% rename from pkg/pipeline/stream/destination_acker.go rename to pkg/lifecycle/stream/destination_acker.go diff --git a/pkg/pipeline/stream/destination_acker_test.go b/pkg/lifecycle/stream/destination_acker_test.go similarity index 100% rename from pkg/pipeline/stream/destination_acker_test.go rename to pkg/lifecycle/stream/destination_acker_test.go diff --git a/pkg/pipeline/stream/destination_test.go b/pkg/lifecycle/stream/destination_test.go similarity index 100% rename from pkg/pipeline/stream/destination_test.go rename to pkg/lifecycle/stream/destination_test.go diff --git a/pkg/pipeline/stream/dlq.go b/pkg/lifecycle/stream/dlq.go similarity index 100% rename from pkg/pipeline/stream/dlq.go rename to pkg/lifecycle/stream/dlq.go diff --git a/pkg/pipeline/stream/dlq_test.go b/pkg/lifecycle/stream/dlq_test.go similarity index 100% rename from pkg/pipeline/stream/dlq_test.go rename to pkg/lifecycle/stream/dlq_test.go diff --git a/pkg/pipeline/stream/doc.go b/pkg/lifecycle/stream/doc.go similarity index 100% rename from pkg/pipeline/stream/doc.go rename to pkg/lifecycle/stream/doc.go diff --git a/pkg/pipeline/stream/fanin.go b/pkg/lifecycle/stream/fanin.go similarity index 100% rename from pkg/pipeline/stream/fanin.go rename to pkg/lifecycle/stream/fanin.go diff --git a/pkg/pipeline/stream/fanin_select.go b/pkg/lifecycle/stream/fanin_select.go similarity index 100% rename from pkg/pipeline/stream/fanin_select.go rename to pkg/lifecycle/stream/fanin_select.go diff --git a/pkg/pipeline/stream/fanout.go b/pkg/lifecycle/stream/fanout.go similarity index 100% rename from pkg/pipeline/stream/fanout.go rename to pkg/lifecycle/stream/fanout.go diff --git a/pkg/pipeline/stream/fanout_test.go b/pkg/lifecycle/stream/fanout_test.go similarity index 100% rename from pkg/pipeline/stream/fanout_test.go rename to pkg/lifecycle/stream/fanout_test.go diff --git a/pkg/pipeline/stream/logger.go b/pkg/lifecycle/stream/logger.go similarity index 100% rename from pkg/pipeline/stream/logger.go rename to pkg/lifecycle/stream/logger.go diff --git a/pkg/pipeline/stream/message.go b/pkg/lifecycle/stream/message.go similarity index 100% rename from pkg/pipeline/stream/message.go rename to pkg/lifecycle/stream/message.go diff --git a/pkg/pipeline/stream/message_test.go b/pkg/lifecycle/stream/message_test.go similarity index 100% rename from pkg/pipeline/stream/message_test.go rename to pkg/lifecycle/stream/message_test.go diff --git a/pkg/pipeline/stream/messagestatus_string.go b/pkg/lifecycle/stream/messagestatus_string.go similarity index 100% rename from pkg/pipeline/stream/messagestatus_string.go rename to pkg/lifecycle/stream/messagestatus_string.go diff --git a/pkg/pipeline/stream/metrics.go b/pkg/lifecycle/stream/metrics.go similarity index 100% rename from pkg/pipeline/stream/metrics.go rename to pkg/lifecycle/stream/metrics.go diff --git a/pkg/pipeline/stream/mock/destination.go b/pkg/lifecycle/stream/mock/destination.go similarity index 100% rename from pkg/pipeline/stream/mock/destination.go rename to pkg/lifecycle/stream/mock/destination.go diff --git a/pkg/pipeline/stream/mock/dlq.go b/pkg/lifecycle/stream/mock/dlq.go similarity index 100% rename from pkg/pipeline/stream/mock/dlq.go rename to pkg/lifecycle/stream/mock/dlq.go diff --git a/pkg/pipeline/stream/mock/processor.go b/pkg/lifecycle/stream/mock/processor.go similarity index 100% rename from pkg/pipeline/stream/mock/processor.go rename to pkg/lifecycle/stream/mock/processor.go diff --git a/pkg/pipeline/stream/mock/source.go b/pkg/lifecycle/stream/mock/source.go similarity index 100% rename from pkg/pipeline/stream/mock/source.go rename to pkg/lifecycle/stream/mock/source.go diff --git a/pkg/pipeline/stream/node.go b/pkg/lifecycle/stream/node.go similarity index 100% rename from pkg/pipeline/stream/node.go rename to pkg/lifecycle/stream/node.go diff --git a/pkg/pipeline/stream/parallel.go b/pkg/lifecycle/stream/parallel.go similarity index 100% rename from pkg/pipeline/stream/parallel.go rename to pkg/lifecycle/stream/parallel.go diff --git a/pkg/pipeline/stream/parallel_test.go b/pkg/lifecycle/stream/parallel_test.go similarity index 100% rename from pkg/pipeline/stream/parallel_test.go rename to pkg/lifecycle/stream/parallel_test.go diff --git a/pkg/pipeline/stream/processor.go b/pkg/lifecycle/stream/processor.go similarity index 100% rename from pkg/pipeline/stream/processor.go rename to pkg/lifecycle/stream/processor.go diff --git a/pkg/pipeline/stream/processor_test.go b/pkg/lifecycle/stream/processor_test.go similarity index 100% rename from pkg/pipeline/stream/processor_test.go rename to pkg/lifecycle/stream/processor_test.go diff --git a/pkg/pipeline/stream/source.go b/pkg/lifecycle/stream/source.go similarity index 100% rename from pkg/pipeline/stream/source.go rename to pkg/lifecycle/stream/source.go diff --git a/pkg/pipeline/stream/source_acker.go b/pkg/lifecycle/stream/source_acker.go similarity index 100% rename from pkg/pipeline/stream/source_acker.go rename to pkg/lifecycle/stream/source_acker.go diff --git a/pkg/pipeline/stream/source_acker_test.go b/pkg/lifecycle/stream/source_acker_test.go similarity index 100% rename from pkg/pipeline/stream/source_acker_test.go rename to pkg/lifecycle/stream/source_acker_test.go diff --git a/pkg/pipeline/stream/source_test.go b/pkg/lifecycle/stream/source_test.go similarity index 100% rename from pkg/pipeline/stream/source_test.go rename to pkg/lifecycle/stream/source_test.go diff --git a/pkg/pipeline/stream/stream_test.go b/pkg/lifecycle/stream/stream_test.go similarity index 100% rename from pkg/pipeline/stream/stream_test.go rename to pkg/lifecycle/stream/stream_test.go diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index dd0386a48..7c38bb4e8 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -128,7 +128,7 @@ type ProcessorPluginService interface { } type LifecycleService interface { - Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error + Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, lifecycleService lifecycle.Service, pipelineID string) error // Stop initiates a stop of the given pipeline. The method does not wait for // the pipeline (and its nodes) to actually stop. // When force is false the pipeline will try to stop gracefully and drain diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index 34f8183b9..7bca244dd 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -24,6 +24,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/jpillora/backoff" ) @@ -35,14 +36,6 @@ const ( DescriptionLengthLimit = 8192 ) -type FailureEvent struct { - // ID is the ID of the pipeline which failed. - ID string - Error error -} - -type FailureHandler func(FailureEvent) - // Service manages pipelines. type Service struct { logger log.CtxLogger @@ -51,7 +44,7 @@ type Service struct { instances map[string]*Instance instanceNames map[string]bool - handlers []FailureHandler + handlers []lifecycle.FailureHandler backoffCfg *backoff.Backoff } @@ -319,24 +312,10 @@ func (s *Service) Delete(ctx context.Context, pipelineID string) error { // OnFailure registers a handler for a pipeline.FailureEvent. // Only errors which happen after a pipeline has been started // are being sent. -func (s *Service) OnFailure(handler FailureHandler) { +func (s *Service) OnFailure(handler lifecycle.FailureHandler) { s.handlers = append(s.handlers, handler) } -// notify notifies all registered FailureHandlers about an error. -func (s *Service) notify(pipelineID string, err error) { - if err == nil { - return - } - e := FailureEvent{ - ID: pipelineID, - Error: err, - } - for _, handler := range s.handlers { - handler(e) - } -} - func (s *Service) validatePipeline(cfg Config, id string) error { // contains all the errors occurred while provisioning configuration files. var errs []error diff --git a/pkg/provisioning/interfaces.go b/pkg/provisioning/interfaces.go index 09ca663cb..6b959b9e6 100644 --- a/pkg/provisioning/interfaces.go +++ b/pkg/provisioning/interfaces.go @@ -35,9 +35,6 @@ type PipelineService interface { Delete(ctx context.Context, pipelineID string) error UpdateDLQ(ctx context.Context, pipelineID string, cfg pipeline.DLQ) (*pipeline.Instance, error) - Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error - Stop(ctx context.Context, pipelineID string, force bool) error - AddConnector(ctx context.Context, pipelineID string, connectorID string) (*pipeline.Instance, error) RemoveConnector(ctx context.Context, pipelineID string, connectorID string) (*pipeline.Instance, error) AddProcessor(ctx context.Context, pipelineID string, processorID string) (*pipeline.Instance, error) @@ -73,3 +70,8 @@ type ProcessorService interface { type ConnectorPluginService interface { NewDispenser(ctx log.CtxLogger, name string, connectorID string) (connectorPlugin.Dispenser, error) } + +type LifecycleService interface { + Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, lifecycleService lifecycle.Service, pipelineID string) error + Stop(ctx context.Context, pipelineID string, force bool) error +} diff --git a/pkg/provisioning/service.go b/pkg/provisioning/service.go index 0e35a0aa3..217a7c193 100644 --- a/pkg/provisioning/service.go +++ b/pkg/provisioning/service.go @@ -40,6 +40,7 @@ type Service struct { connectorService ConnectorService processorService ProcessorService connectorPluginService ConnectorPluginService + lifecycleService LifecycleService pipelinesPath string } @@ -254,8 +255,7 @@ func (s *Service) provisionPipeline(ctx context.Context, cfg config.Pipeline) er if cfg.Status == config.StatusRunning { // TODO set status and let the pipeline service start it - // TODO: Use the lifecycle service (not the pipeline service) to start the pipeline - err := s.pipelineService.Start(ctx, s.connectorService, s.processorService, s.connectorPluginService, cfg.ID) + err := s.lifecycleService.Start(ctx, s.connectorService, s.processorService, s.connectorPluginService, cfg.ID) if err != nil { return cerrors.Errorf("could not start the pipeline %q: %w", cfg.ID, err) } From f21f90156330644d1cd135cbdbbbb65c14007898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 9 Sep 2024 15:34:57 +0200 Subject: [PATCH 07/41] add lifecycleService to runtime --- pkg/conduit/runtime.go | 10 ++++++---- pkg/orchestrator/orchestrator.go | 4 ++-- pkg/provisioning/interfaces.go | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index acea21e79..8604fbc1d 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -95,6 +95,7 @@ type Runtime struct { pipelineService *pipeline.Service connectorService *connector.Service processorService *processor.Service + lifecycleService *lifecycle.Service connectorPluginService *conn_plugin.PluginService processorPluginService *proc_plugin.PluginService @@ -226,6 +227,7 @@ func createServices(r *Runtime) error { r.processorPluginService = procPluginService r.connSchemaService = connSchemaService r.procSchemaService = procSchemaService + r.lifecycleService = lifecycleService return nil } @@ -416,12 +418,12 @@ func (r *Runtime) registerCleanup(t *tomb.Tomb) { // t.Err() can be nil, when we had a call: t.Kill(nil) // t.Err() will be context.Canceled, if the tomb's context was canceled if t.Err() == nil || cerrors.Is(t.Err(), context.Canceled) { - r.pipelineService.StopAll(ctx, pipeline.ErrGracefulShutdown) + r.lifecycleService.StopAll(ctx, pipeline.ErrGracefulShutdown) } else { // tomb died due to a real error - r.pipelineService.StopAll(ctx, cerrors.Errorf("conduit experienced an error: %w", t.Err())) + r.lifecycleService.StopAll(ctx, cerrors.Errorf("conduit experienced an error: %w", t.Err())) } - err := r.pipelineService.Wait(exitTimeout) + err := r.lifecycleService.Wait(exitTimeout) t.Go(func() error { r.connectorPersister.Wait() return r.DB.Close() @@ -790,7 +792,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } } - err = r.pipelineService.Run(ctx, r.connectorService, r.processorService, r.connectorPluginService) + err = r.lifecycleService.Run(ctx, r.connectorService, r.processorService, r.connectorPluginService) if err != nil { cerrors.ForEach(err, func(err error) { r.logger.Err(ctx, err).Msg("pipeline failed to be started") diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index 7c38bb4e8..992b15393 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService +//go:generate mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService package orchestrator @@ -128,7 +128,7 @@ type ProcessorPluginService interface { } type LifecycleService interface { - Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, lifecycleService lifecycle.Service, pipelineID string) error + Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error // Stop initiates a stop of the given pipeline. The method does not wait for // the pipeline (and its nodes) to actually stop. // When force is false the pipeline will try to stop gracefully and drain diff --git a/pkg/provisioning/interfaces.go b/pkg/provisioning/interfaces.go index 6b959b9e6..7a77b693a 100644 --- a/pkg/provisioning/interfaces.go +++ b/pkg/provisioning/interfaces.go @@ -72,6 +72,6 @@ type ConnectorPluginService interface { } type LifecycleService interface { - Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, lifecycleService lifecycle.Service, pipelineID string) error + Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error Stop(ctx context.Context, pipelineID string, force bool) error } From 9867f235ff1bb4ebcdac94ee37a0298127f1a385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 9 Sep 2024 15:39:23 +0200 Subject: [PATCH 08/41] fix --- pkg/conduit/runtime.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index 8604fbc1d..fddc5d829 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -792,7 +792,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } } - err = r.lifecycleService.Run(ctx, r.connectorService, r.processorService, r.connectorPluginService) + err = r.lifecycleService.Run(ctx, r.connectorService, r.processorService, r.connectorPluginService, r.pipelineService) if err != nil { cerrors.ForEach(err, func(err error) { r.logger.Err(ctx, err).Msg("pipeline failed to be started") From 0ef514291e8434d58cfcb2a2294db20c284161a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Mon, 9 Sep 2024 15:41:06 +0200 Subject: [PATCH 09/41] fix imports --- pkg/lifecycle/dlq.go | 2 +- pkg/lifecycle/service_test.go | 2 +- pkg/lifecycle/stream/stream_test.go | 4 ++-- pkg/pipeline/instance.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/lifecycle/dlq.go b/pkg/lifecycle/dlq.go index 5371da6a1..8590fffde 100644 --- a/pkg/lifecycle/dlq.go +++ b/pkg/lifecycle/dlq.go @@ -21,7 +21,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/pipeline/stream" + "github.com/conduitio/conduit/pkg/lifecycle/stream" ) // DLQDestination is a DLQ handler that forwards DLQ records to a destination diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 1fb982512..2d5a1ec0c 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -29,8 +29,8 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle/stream" "github.com/conduitio/conduit/pkg/pipeline" - "github.com/conduitio/conduit/pkg/pipeline/stream" "github.com/conduitio/conduit/pkg/plugin" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" pmock "github.com/conduitio/conduit/pkg/plugin/connector/mock" diff --git a/pkg/lifecycle/stream/stream_test.go b/pkg/lifecycle/stream/stream_test.go index 44f3e061e..d04d46621 100644 --- a/pkg/lifecycle/stream/stream_test.go +++ b/pkg/lifecycle/stream/stream_test.go @@ -28,8 +28,8 @@ import ( "github.com/conduitio/conduit/pkg/foundation/ctxutil" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream" - streammock "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream" + streammock "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/rs/zerolog" "go.uber.org/mock/gomock" diff --git a/pkg/pipeline/instance.go b/pkg/pipeline/instance.go index 35654eb98..078da1330 100644 --- a/pkg/pipeline/instance.go +++ b/pkg/pipeline/instance.go @@ -20,7 +20,7 @@ import ( "sync" "time" - "github.com/conduitio/conduit/pkg/pipeline/stream" + "github.com/conduitio/conduit/pkg/lifecycle/stream" "gopkg.in/tomb.v2" ) From 80cd33b0380079caee4c2839889c31f5beb15419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 10 Sep 2024 13:43:49 +0200 Subject: [PATCH 10/41] no errors on lifecycle service --- pkg/lifecycle/service.go | 245 ++++++++++++++++------------------ pkg/lifecycle/service_test.go | 6 +- pkg/pipeline/instance.go | 6 - pkg/pipeline/service.go | 27 ++-- pkg/pipeline/store.go | 2 - 5 files changed, 139 insertions(+), 147 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index a52ec6e18..1f2748679 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -27,7 +27,6 @@ import ( "sync/atomic" "time" - "github.com/conduitio/conduit-commons/database" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" @@ -49,44 +48,47 @@ type FailureEvent struct { type FailureHandler func(FailureEvent) -// Store handles the persistence and fetching of pipeline instances. -type Store struct { - db database.DB -} - -func NewStore(db database.DB) *Store { - return &Store{ - db: db, - } -} - // Service manages pipelines. type Service struct { logger log.CtxLogger - store *Store - - // PipelineService - //instances map[string]*pipeline.Instance - //instanceNames map[string]bool backoffCfg *backoff.Backoff - pipelines pipeline.Service - //connectors ConnectorService - processors ProcessorService - //connectorPlugins ConnectorPluginService - //processorPlugins ProcessorPluginService + pipelines PipelineService + connectors ConnectorFetcher + + processors ProcessorService + connectorPlugins PluginDispenserFetcher + + handlers []FailureHandler + runningPipelines map[string]*runnablePipeline } // NewService initializes and returns a pipeline Service. -func NewService(logger log.CtxLogger, db database.DB, backoffCfg *backoff.Backoff) *Service { +func NewService( + logger log.CtxLogger, + backoffCfg *backoff.Backoff, + connectors ConnectorFetcher, + processors ProcessorService, + connectorPlugins PluginDispenserFetcher, + pipelines PipelineService, +) *Service { return &Service{ - logger: logger.WithComponent("pipeline.Service"), - store: NewStore(db), - backoffCfg: backoffCfg, + logger: logger.WithComponent("pipeline.Service"), + backoffCfg: backoffCfg, + connectors: connectors, + processors: processors, + connectorPlugins: connectorPlugins, + pipelines: pipelines, } } +type runnablePipeline struct { + pipeline *pipeline.Instance + n []stream.Node + t *tomb.Tomb +} + // ConnectorFetcher can fetch a connector instance. type ConnectorFetcher interface { Get(ctx context.Context, id string) (*connector.Instance, error) @@ -108,24 +110,28 @@ type PluginDispenserFetcher interface { type PipelineService interface { Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) GetInstances() map[string]*pipeline.Instance + UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status) error +} + +// OnFailure registers a handler for a pipeline.FailureEvent. +// Only errors which happen after a pipeline has been started +// are being sent. +func (s *Service) OnFailure(handler FailureHandler) { + s.handlers = append(s.handlers, handler) } // Run runs pipelines that had the running state in store. func (s *Service) Run( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, - pipelineService PipelineService, ) error { var errs []error s.logger.Debug(ctx).Msg("initializing pipelines statuses") // run pipelines that are in the StatusSystemStopped state - instances := pipelineService.GetInstances() + instances := s.pipelines.GetInstances() for _, instance := range instances { if instance.GetStatus() == pipeline.StatusSystemStopped { - err := s.Start(ctx, connFetcher, procService, pluginFetcher, pipelineService, instance.ID) + err := s.Start(ctx, instance.ID) if err != nil { // try to start remaining pipelines and gather errors errs = append(errs, err) @@ -140,16 +146,13 @@ func (s *Service) Run( // If the pipeline is already running, Start returns ErrPipelineRunning. func (s *Service) Start( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, - ps PipelineService, pipelineID string, ) error { - pl, err := ps.Get(ctx, pipelineID) + pl, err := s.pipelines.Get(ctx, pipelineID) if err != nil { return err } + if pl.GetStatus() == pipeline.StatusRunning { return cerrors.Errorf("can't start pipeline %s: %w", pl.ID, pipeline.ErrPipelineRunning) } @@ -157,22 +160,22 @@ func (s *Service) Start( s.logger.Debug(ctx).Str(log.PipelineIDField, pl.ID).Msg("starting pipeline") s.logger.Trace(ctx).Str(log.PipelineIDField, pl.ID).Msg("building nodes") - nodes, err := s.buildNodes(ctx, connFetcher, procService, pluginFetcher, pl) + + rp, err := s.buildRunnablePipeline(ctx, pl) if err != nil { return cerrors.Errorf("could not build nodes for pipeline %s: %w", pl.ID, err) } - pl.n = make(map[string]stream.Node) - for _, node := range nodes { - pl.n[node.ID()] = node - } + s.runningPipelines[pl.ID] = rp s.logger.Trace(ctx).Str(log.PipelineIDField, pl.ID).Msg("running nodes") - if err := s.runPipeline(ctx, pl); err != nil { + if err := s.runPipeline(ctx, rp); err != nil { return cerrors.Errorf("failed to run pipeline %s: %w", pl.ID, err) } s.logger.Info(ctx).Str(log.PipelineIDField, pl.ID).Msg("pipeline started") + // TODO: store which pipelines are actually running + return nil } @@ -180,32 +183,32 @@ func (s *Service) Start( // Stop function. If force is set to true the pipeline won't stop gracefully, // instead the context for all nodes will be canceled which causes them to stop // running as soon as possible. -func (s *Service) Stop(ctx context.Context, pipelineID string, force bool, ps PipelineService) error { - pl, err := ps.Get(ctx, pipelineID) - if err != nil { - return err +func (s *Service) Stop(ctx context.Context, pipelineID string, force bool) error { + rp, ok := s.runningPipelines[pipelineID] + if !ok { + return cerrors.Errorf("pipeline %s is not running: %w", pipelineID, pipeline.ErrPipelineNotRunning) } - if pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering { - return cerrors.Errorf("can't stop pipeline with status %q: %w", pl.GetStatus(), pipeline.ErrPipelineNotRunning) + if rp.pipeline.GetStatus() != pipeline.StatusRunning && rp.pipeline.GetStatus() != pipeline.StatusRecovering { + return cerrors.Errorf("can't stop pipeline with status %q: %w", rp.pipeline.GetStatus(), pipeline.ErrPipelineNotRunning) } switch force { case false: - return s.stopGraceful(ctx, pl, nil) + return s.stopGraceful(ctx, rp, nil) case true: - return s.stopForceful(ctx, pl) + return s.stopForceful(ctx, rp) } panic("unreachable code") } -func (s *Service) stopGraceful(ctx context.Context, pl *pipeline.Instance, reason error) error { +func (s *Service) stopGraceful(ctx context.Context, rp *runnablePipeline, reason error) error { s.logger.Info(ctx). - Str(log.PipelineIDField, pl.ID). - Any(log.PipelineStatusField, pl.GetStatus()). + Str(log.PipelineIDField, rp.pipeline.ID). + Any(log.PipelineStatusField, rp.pipeline.GetStatus()). Msg("gracefully stopping pipeline") var errs []error - for _, n := range pl.n { + for _, n := range rp.n { if node, ok := n.(stream.StoppableNode); ok { // stop all pub nodes s.logger.Trace(ctx).Str(log.NodeIDField, n.ID()).Msg("stopping node") @@ -216,22 +219,30 @@ func (s *Service) stopGraceful(ctx context.Context, pl *pipeline.Instance, reaso } } } + + if len(errs) == 0 { + s.runningPipelines[rp.pipeline.ID] = nil + return nil + } + return cerrors.Join(errs...) } -func (s *Service) stopForceful(ctx context.Context, pl *pipeline.Instance) error { +func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error { s.logger.Info(ctx). - Str(log.PipelineIDField, pl.ID). - Any(log.PipelineStatusField, pl.GetStatus()). + Str(log.PipelineIDField, rp.pipeline.ID). + Any(log.PipelineStatusField, rp.pipeline.GetStatus()). Msg("force stopping pipeline") - pl.t.Kill(pipeline.ErrForceStop) - for _, n := range pl.n { + rp.t.Kill(pipeline.ErrForceStop) + for _, n := range rp.n { if node, ok := n.(stream.ForceStoppableNode); ok { // stop all pub nodes s.logger.Trace(ctx).Str(log.NodeIDField, n.ID()).Msg("force stopping node") node.ForceStop(ctx) } } + + s.runningPipelines[rp.pipeline.ID] = nil return nil } @@ -240,10 +251,11 @@ func (s *Service) stopForceful(ctx context.Context, pl *pipeline.Instance) error func (s *Service) StopAll(ctx context.Context, reason error) { instances := s.pipelines.GetInstances() for _, pl := range instances { - if pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering { + rp, ok := s.runningPipelines[pl.ID] + if !ok || (pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering) { continue } - err := s.stopGraceful(ctx, pl, reason) + err := s.stopGraceful(ctx, rp, reason) if err != nil { s.logger.Warn(ctx). Err(err). @@ -292,19 +304,16 @@ func (s *Service) waitInternal() error { return cerrors.Join(errs...) } -// buildsNodes will build and connect all nodes configured in the pipeline. -func (s *Service) buildNodes( +// buildRunnablePipeline will build and connect all nodes configured in the pipeline. +func (s *Service) buildRunnablePipeline( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, pl *pipeline.Instance, -) ([]stream.Node, error) { +) (*runnablePipeline, error) { // setup many to many channels fanIn := stream.FaninNode{Name: "fanin"} fanOut := stream.FanoutNode{Name: "fanout"} - sourceNodes, err := s.buildSourceNodes(ctx, connFetcher, procService, pluginFetcher, pl, &fanIn) + sourceNodes, err := s.buildSourceNodes(ctx, pl, &fanIn) if err != nil { return nil, cerrors.Errorf("could not build source nodes: %w", err) } @@ -312,12 +321,12 @@ func (s *Service) buildNodes( return nil, cerrors.New("can't build pipeline without any source connectors") } - processorNodes, err := s.buildProcessorNodes(ctx, procService, pl, pl.ProcessorIDs, &fanIn, &fanOut) + processorNodes, err := s.buildProcessorNodes(ctx, pl, pl.ProcessorIDs, &fanIn, &fanOut) if err != nil { return nil, cerrors.Errorf("could not build processor nodes: %w", err) } - destinationNodes, err := s.buildDestinationNodes(ctx, connFetcher, procService, pluginFetcher, pl, &fanOut) + destinationNodes, err := s.buildDestinationNodes(ctx, pl, &fanOut) if err != nil { return nil, cerrors.Errorf("could not build destination nodes: %w", err) } @@ -340,12 +349,14 @@ func (s *Service) buildNodes( stream.SetLogger(n, nodeLogger) } - return nodes, nil + return &runnablePipeline{ + pipeline: pl, + n: nodes, + }, nil } func (s *Service) buildProcessorNodes( ctx context.Context, - procService ProcessorService, pl *pipeline.Instance, processorIDs []string, first stream.PubNode, @@ -355,12 +366,12 @@ func (s *Service) buildProcessorNodes( prev := first for _, procID := range processorIDs { - instance, err := procService.Get(ctx, procID) + instance, err := s.processors.Get(ctx, procID) if err != nil { return nil, cerrors.Errorf("could not fetch processor: %w", err) } - runnableProc, err := procService.MakeRunnableProcessor(ctx, instance) + runnableProc, err := s.processors.MakeRunnableProcessor(ctx, instance) if err != nil { return nil, err } @@ -410,21 +421,18 @@ func (s *Service) buildProcessorNode( func (s *Service) buildSourceNodes( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, pl *pipeline.Instance, next stream.SubNode, ) ([]stream.Node, error) { var nodes []stream.Node - dlqHandlerNode, err := s.buildDLQHandlerNode(ctx, connFetcher, pluginFetcher, pl) + dlqHandlerNode, err := s.buildDLQHandlerNode(ctx, pl) if err != nil { return nil, err } for _, connID := range pl.ConnectorIDs { - instance, err := connFetcher.Get(ctx, connID) + instance, err := s.connectors.Get(ctx, connID) if err != nil { return nil, cerrors.Errorf("could not fetch connector: %w", err) } @@ -433,7 +441,7 @@ func (s *Service) buildSourceNodes( continue // skip any connector that's not a source } - src, err := instance.Connector(ctx, pluginFetcher) + src, err := instance.Connector(ctx, s.connectorPlugins) if err != nil { return nil, err } @@ -451,7 +459,7 @@ func (s *Service) buildSourceNodes( metricsNode := s.buildMetricsNode(pl, instance) metricsNode.Sub(ackerNode.Pub()) - procNodes, err := s.buildProcessorNodes(ctx, procService, pl, instance.ProcessorIDs, metricsNode, next) + procNodes, err := s.buildProcessorNodes(ctx, pl, instance.ProcessorIDs, metricsNode, next) if err != nil { return nil, cerrors.Errorf("could not build processor nodes for connector %s: %w", instance.ID, err) } @@ -479,11 +487,9 @@ func (s *Service) buildSourceAckerNode( func (s *Service) buildDLQHandlerNode( ctx context.Context, - connFetcher ConnectorFetcher, - pluginFetcher PluginDispenserFetcher, pl *pipeline.Instance, ) (*stream.DLQHandlerNode, error) { - conn, err := connFetcher.Create( + conn, err := s.connectors.Create( ctx, pl.ID+"-dlq", connector.TypeDestination, @@ -499,7 +505,7 @@ func (s *Service) buildDLQHandlerNode( return nil, cerrors.Errorf("failed to create DLQ destination: %w", err) } - dest, err := conn.Connector(ctx, pluginFetcher) + dest, err := conn.Connector(ctx, s.connectorPlugins) if err != nil { return nil, err } @@ -551,16 +557,13 @@ func (s *Service) buildDestinationAckerNode( func (s *Service) buildDestinationNodes( ctx context.Context, - connFetcher ConnectorFetcher, - procService ProcessorService, - pluginFetcher PluginDispenserFetcher, pl *pipeline.Instance, prev stream.PubNode, ) ([]stream.Node, error) { var nodes []stream.Node for _, connID := range pl.ConnectorIDs { - instance, err := connFetcher.Get(ctx, connID) + instance, err := s.connectors.Get(ctx, connID) if err != nil { return nil, cerrors.Errorf("could not fetch connector: %w", err) } @@ -569,7 +572,7 @@ func (s *Service) buildDestinationNodes( continue // skip any connector that's not a destination } - dest, err := instance.Connector(ctx, pluginFetcher) + dest, err := instance.Connector(ctx, s.connectorPlugins) if err != nil { return nil, err } @@ -588,7 +591,7 @@ func (s *Service) buildDestinationNodes( destinationNode.Sub(metricsNode.Pub()) ackerNode.Sub(destinationNode.Pub()) - connNodes, err := s.buildProcessorNodes(ctx, procService, pl, instance.ProcessorIDs, prev, metricsNode) + connNodes, err := s.buildProcessorNodes(ctx, pl, instance.ProcessorIDs, prev, metricsNode) if err != nil { return nil, cerrors.Errorf("could not build processor nodes for connector %s: %w", instance.ID, err) } @@ -600,18 +603,19 @@ func (s *Service) buildDestinationNodes( return nodes, nil } -func (s *Service) runPipeline(ctx context.Context, pl *pipeline.Instance) error { - if pl.t != nil && pl.t.Alive() { +func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { + // TODO: Handle the tomb outside and after maybe checking the status + if rp.t != nil && rp.t.Alive() { return pipeline.ErrPipelineRunning } // the tomb is responsible for running goroutines related to the pipeline - pl.t = &tomb.Tomb{} + rp.t = &tomb.Tomb{} // keep tomb alive until the end of this function, this way we guarantee we // can run the cleanup goroutine even if all nodes stop before we get to it keepAlive := make(chan struct{}) - pl.t.Go(func() error { + rp.t.Go(func() error { <-keepAlive return nil }) @@ -620,16 +624,15 @@ func (s *Service) runPipeline(ctx context.Context, pl *pipeline.Instance) error // nodesWg is done once all nodes stop running var nodesWg sync.WaitGroup var isGracefulShutdown atomic.Bool - for id := range pl.n { + for _, node := range rp.n { nodesWg.Add(1) - node := pl.n[id] - pl.t.Go(func() (errOut error) { + rp.t.Go(func() (errOut error) { // If any of the nodes stop, the tomb will be put into a dying state // and ctx will be cancelled. // This way, the other nodes will be notified that they need to stop too. //nolint:staticcheck // nil used to use the default (parent provided via WithContext) - ctx := pl.t.Context(nil) + ctx := rp.t.Context(nil) s.logger.Trace(ctx).Str(log.NodeIDField, node.ID()).Msg("running node") defer func() { e := s.logger.Trace(ctx) @@ -657,27 +660,22 @@ func (s *Service) runPipeline(ctx context.Context, pl *pipeline.Instance) error }) } - measure.PipelinesGauge.WithValues(strings.ToLower(pl.GetStatus().String())).Dec() - pl.SetStatus(pipeline.StatusRunning) - pl.Error = "" - measure.PipelinesGauge.WithValues(strings.ToLower(pl.GetStatus().String())).Inc() - - err := s.store.Set(ctx, pl.ID, pl) + err := s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRunning) if err != nil { - return cerrors.Errorf("pipeline not updated: %w", err) + return err } // cleanup function updates the metrics and pipeline status once all nodes // stop running - pl.t.Go(func() error { + rp.t.Go(func() error { // use fresh context for cleanup function, otherwise the updated status // won't be stored ctx := context.Background() nodesWg.Wait() - err := pl.t.Err() + err := rp.t.Err() - measure.PipelinesGauge.WithValues(strings.ToLower(pl.GetStatus().String())).Dec() + measure.PipelinesGauge.WithValues(strings.ToLower(rp.pipeline.GetStatus().String())).Dec() switch err { case tomb.ErrStillAlive: @@ -685,43 +683,36 @@ func (s *Service) runPipeline(ctx context.Context, pl *pipeline.Instance) error err = nil if isGracefulShutdown.Load() { // it was triggered by a graceful shutdown of Conduit - pl.SetStatus(pipeline.StatusSystemStopped) + rp.pipeline.SetStatus(pipeline.StatusSystemStopped) } else { // it was manually triggered by a user - pl.SetStatus(pipeline.StatusUserStopped) + rp.pipeline.SetStatus(pipeline.StatusUserStopped) } default: if cerrors.IsFatalError(err) { - pl.SetStatus(pipeline.StatusDegraded) + rp.pipeline.SetStatus(pipeline.StatusDegraded) // we use %+v to get the stack trace too - pl.Error = fmt.Sprintf("%+v", err) + rp.pipeline.Error = fmt.Sprintf("%+v", err) } else { - pl.SetStatus(pipeline.StatusRecovering) - err = s.store.Set(ctx, pl.ID, pl) + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRecovering) if err != nil { - return cerrors.Errorf("failed to save pipeline with ID %q: %w", pl.ID, err) + return err } } } s.logger. Err(ctx, err). - Str(log.PipelineIDField, pl.ID). + Str(log.PipelineIDField, rp.pipeline.ID). Msg("pipeline stopped") - s.notify(pl.ID, err) + s.notify(rp.pipeline.ID, err) // It's important to update the metrics before we handle the error from s.Store.Set() (if any), // since the source of the truth is the actual pipeline (stored in memory). - measure.PipelinesGauge.WithValues(strings.ToLower(pl.GetStatus().String())).Inc() + measure.PipelinesGauge.WithValues(strings.ToLower(rp.pipeline.GetStatus().String())).Inc() - storeErr := s.store.Set(ctx, pl.ID, pl) - if storeErr != nil { - return cerrors.Errorf("pipeline not updated: %w", storeErr) - } - - return err + return s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, rp.pipeline.GetStatus()) }) - return nil } diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 2d5a1ec0c..2797d97c9 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -72,7 +72,7 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { } pl.SetStatus(pipeline.StatusUserStopped) - got, err := ls.buildNodes( + got, err := ls.buildRunnablePipeline( ctx, testConnectorFetcher{ source.ID: source, @@ -157,7 +157,7 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { } pl.SetStatus(pipeline.StatusUserStopped) - got, err := ls.buildNodes( + got, err := ls.buildRunnablePipeline( ctx, testConnectorFetcher{ destination.ID: destination, @@ -206,7 +206,7 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { } pl.SetStatus(pipeline.StatusUserStopped) - got, err := ls.buildNodes( + got, err := ls.buildRunnablePipeline( ctx, testConnectorFetcher{ source.ID: source, diff --git a/pkg/pipeline/instance.go b/pkg/pipeline/instance.go index 078da1330..0b37d1813 100644 --- a/pkg/pipeline/instance.go +++ b/pkg/pipeline/instance.go @@ -19,9 +19,6 @@ package pipeline import ( "sync" "time" - - "github.com/conduitio/conduit/pkg/lifecycle/stream" - "gopkg.in/tomb.v2" ) const ( @@ -61,9 +58,6 @@ type Instance struct { status Status statusLock sync.RWMutex - - n map[string]stream.Node - t *tomb.Tomb } // encodableInstance is an encodable "view" of Instance diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index 7bca244dd..a75ca9d62 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -24,7 +24,6 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" - "github.com/conduitio/conduit/pkg/lifecycle" "github.com/jpillora/backoff" ) @@ -44,7 +43,6 @@ type Service struct { instances map[string]*Instance instanceNames map[string]bool - handlers []lifecycle.FailureHandler backoffCfg *backoff.Backoff } @@ -309,13 +307,6 @@ func (s *Service) Delete(ctx context.Context, pipelineID string) error { return nil } -// OnFailure registers a handler for a pipeline.FailureEvent. -// Only errors which happen after a pipeline has been started -// are being sent. -func (s *Service) OnFailure(handler lifecycle.FailureHandler) { - s.handlers = append(s.handlers, handler) -} - func (s *Service) validatePipeline(cfg Config, id string) error { // contains all the errors occurred while provisioning configuration files. var errs []error @@ -350,3 +341,21 @@ func (s *Service) validatePipeline(cfg Config, id string) error { func (s *Service) GetInstances() map[string]*Instance { return s.instances } + +func (s *Service) UpdateStatus(ctx context.Context, id string, status Status) error { + pipeline, err := s.Get(ctx, id) + if err != nil { + return err + } + measure.PipelinesGauge.WithValues(strings.ToLower(pipeline.GetStatus().String())).Dec() + pipeline.SetStatus(status) + + pipeline.Error = "" + measure.PipelinesGauge.WithValues(strings.ToLower(pipeline.GetStatus().String())).Inc() + + err = s.store.Set(ctx, pipeline.ID, pipeline) + if err != nil { + return cerrors.Errorf("pipeline not updated: %w", err) + } + return nil +} diff --git a/pkg/pipeline/store.go b/pkg/pipeline/store.go index bd08860db..41ac88ff8 100644 --- a/pkg/pipeline/store.go +++ b/pkg/pipeline/store.go @@ -31,13 +31,11 @@ const ( storeKeyPrefix = "pipeline:instance:" ) -// TODO: remove? // Store handles the persistence and fetching of pipeline instances. type Store struct { db database.DB } -// TODO: remove? func NewStore(db database.DB) *Store { return &Store{ db: db, From a9f30b3883328cdc30b95b1d5a0f85776ab13445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 10 Sep 2024 15:30:17 +0200 Subject: [PATCH 11/41] fix wait --- pkg/lifecycle/service.go | 13 ++++++------- pkg/pipeline/instance.go | 7 ------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 1f2748679..2a142f5f8 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -166,15 +166,12 @@ func (s *Service) Start( return cerrors.Errorf("could not build nodes for pipeline %s: %w", pl.ID, err) } - s.runningPipelines[pl.ID] = rp - s.logger.Trace(ctx).Str(log.PipelineIDField, pl.ID).Msg("running nodes") if err := s.runPipeline(ctx, rp); err != nil { return cerrors.Errorf("failed to run pipeline %s: %w", pl.ID, err) } s.logger.Info(ctx).Str(log.PipelineIDField, pl.ID).Msg("pipeline started") - - // TODO: store which pipelines are actually running + s.runningPipelines[pl.ID] = rp return nil } @@ -294,9 +291,11 @@ func (s *Service) Wait(timeout time.Duration) error { // the pipelines failed to stop gracefully. func (s *Service) waitInternal() error { var errs []error - instances := s.pipelines.GetInstances() - for _, pl := range instances { - err := pl.Wait() + for _, rp := range s.runningPipelines { + if rp.t == nil { + continue + } + err := rp.t.Wait() if err != nil { errs = append(errs, err) } diff --git a/pkg/pipeline/instance.go b/pkg/pipeline/instance.go index 0b37d1813..e6b35620e 100644 --- a/pkg/pipeline/instance.go +++ b/pkg/pipeline/instance.go @@ -91,13 +91,6 @@ var DefaultDLQ = DLQ{ WindowNackThreshold: 0, } -func (p *Instance) Wait() error { - if p.t == nil { - return nil - } - return p.t.Wait() -} - func (p *Instance) SetStatus(s Status) { p.statusLock.Lock() defer p.statusLock.Unlock() From 599cd811ad8421830a0a6aec803e4eb7e346347a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 10 Sep 2024 15:38:51 +0200 Subject: [PATCH 12/41] update status through its method --- pkg/lifecycle/service.go | 22 +++++++++++++--------- pkg/pipeline/service.go | 5 +++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 2a142f5f8..e516439de 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -110,7 +110,7 @@ type PluginDispenserFetcher interface { type PipelineService interface { Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) GetInstances() map[string]*pipeline.Instance - UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status) error + UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status, errMsg string) error } // OnFailure registers a handler for a pipeline.FailureEvent. @@ -659,7 +659,7 @@ func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { }) } - err := s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRunning) + err := s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRunning, "") if err != nil { return err } @@ -682,18 +682,23 @@ func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { err = nil if isGracefulShutdown.Load() { // it was triggered by a graceful shutdown of Conduit - rp.pipeline.SetStatus(pipeline.StatusSystemStopped) + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusSystemStopped, "") } else { // it was manually triggered by a user - rp.pipeline.SetStatus(pipeline.StatusUserStopped) + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusUserStopped, "") + } + if err != nil { + return err } default: if cerrors.IsFatalError(err) { - rp.pipeline.SetStatus(pipeline.StatusDegraded) // we use %+v to get the stack trace too - rp.pipeline.Error = fmt.Sprintf("%+v", err) + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusDegraded, fmt.Sprintf("%+v", err)) + if err != nil { + return err + } } else { - err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRecovering) + err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusRecovering, "") if err != nil { return err } @@ -709,8 +714,7 @@ func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { // It's important to update the metrics before we handle the error from s.Store.Set() (if any), // since the source of the truth is the actual pipeline (stored in memory). measure.PipelinesGauge.WithValues(strings.ToLower(rp.pipeline.GetStatus().String())).Inc() - - return s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, rp.pipeline.GetStatus()) + return s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, rp.pipeline.GetStatus(), "") }) return nil } diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index a75ca9d62..fd128642e 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -342,7 +342,8 @@ func (s *Service) GetInstances() map[string]*Instance { return s.instances } -func (s *Service) UpdateStatus(ctx context.Context, id string, status Status) error { +// UpdateStatus updates the status of a pipeline by the ID. +func (s *Service) UpdateStatus(ctx context.Context, id string, status Status, errMsg string) error { pipeline, err := s.Get(ctx, id) if err != nil { return err @@ -350,7 +351,7 @@ func (s *Service) UpdateStatus(ctx context.Context, id string, status Status) er measure.PipelinesGauge.WithValues(strings.ToLower(pipeline.GetStatus().String())).Dec() pipeline.SetStatus(status) - pipeline.Error = "" + pipeline.Error = errMsg measure.PipelinesGauge.WithValues(strings.ToLower(pipeline.GetStatus().String())).Inc() err = s.store.Set(ctx, pipeline.ID, pipeline) From 871fc0b2680bf983d415f6f8c86b2c151318fd7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 10 Sep 2024 15:47:22 +0200 Subject: [PATCH 13/41] it compiles --- pkg/conduit/runtime.go | 6 +++--- pkg/orchestrator/orchestrator.go | 3 +-- pkg/orchestrator/pipelines.go | 4 ++-- pkg/provisioning/interfaces.go | 3 +-- pkg/provisioning/service.go | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index fddc5d829..160929369 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -211,7 +211,7 @@ func createServices(r *Runtime) error { plService := pipeline.NewService(r.logger, r.DB, backoffCfg) connService := connector.NewService(r.logger, r.DB, r.connectorPersister) procService := processor.NewService(r.logger, r.DB, procPluginService) - lifecycleService := lifecycle.NewService(r.logger, r.DB, backoffCfg) + lifecycleService := lifecycle.NewService(r.logger, backoffCfg, connService, procService, connPluginService, plService) provisionService := provisioning.NewService(r.DB, r.logger, plService, connService, procService, connPluginService, r.Config.Pipelines.Path) orc := orchestrator.NewOrchestrator(r.DB, r.logger, plService, connService, procService, connPluginService, procPluginService, lifecycleService) @@ -765,7 +765,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } if r.Config.Pipelines.ExitOnError { - r.pipelineService.OnFailure(func(e lifecycle.FailureEvent) { + r.lifecycleService.OnFailure(func(e lifecycle.FailureEvent) { r.logger.Warn(ctx). Err(e.Error). Str(log.PipelineIDField, e.ID). @@ -792,7 +792,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } } - err = r.lifecycleService.Run(ctx, r.connectorService, r.processorService, r.connectorPluginService, r.pipelineService) + err = r.lifecycleService.Run(ctx) if err != nil { cerrors.ForEach(err, func(err error) { r.logger.Err(ctx, err).Msg("pipeline failed to be started") diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index 992b15393..35e4badda 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -24,7 +24,6 @@ import ( processorSdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/pipeline" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" @@ -128,7 +127,7 @@ type ProcessorPluginService interface { } type LifecycleService interface { - Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error + Start(ctx context.Context, pipelineID string) error // Stop initiates a stop of the given pipeline. The method does not wait for // the pipeline (and its nodes) to actually stop. // When force is false the pipeline will try to stop gracefully and drain diff --git a/pkg/orchestrator/pipelines.go b/pkg/orchestrator/pipelines.go index 839e0e5bd..e00a3a772 100644 --- a/pkg/orchestrator/pipelines.go +++ b/pkg/orchestrator/pipelines.go @@ -27,12 +27,12 @@ type PipelineOrchestrator base func (po *PipelineOrchestrator) Start(ctx context.Context, id string) error { // TODO lock pipeline - return po.pipelines.Start(ctx, po.connectors, po.processors, po.connectorPlugins, id) + return po.lifecycle.Start(ctx, id) } func (po *PipelineOrchestrator) Stop(ctx context.Context, id string, force bool) error { // TODO lock pipeline - return po.pipelines.Stop(ctx, id, force) + return po.lifecycle.Stop(ctx, id, force) } func (po *PipelineOrchestrator) List(ctx context.Context) map[string]*pipeline.Instance { diff --git a/pkg/provisioning/interfaces.go b/pkg/provisioning/interfaces.go index 7a77b693a..a11f96db7 100644 --- a/pkg/provisioning/interfaces.go +++ b/pkg/provisioning/interfaces.go @@ -19,7 +19,6 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/pipeline" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/processor" @@ -72,6 +71,6 @@ type ConnectorPluginService interface { } type LifecycleService interface { - Start(ctx context.Context, connFetcher lifecycle.ConnectorFetcher, procService lifecycle.ProcessorService, pluginFetcher lifecycle.PluginDispenserFetcher, pipelineID string) error + Start(ctx context.Context, pipelineID string) error Stop(ctx context.Context, pipelineID string, force bool) error } diff --git a/pkg/provisioning/service.go b/pkg/provisioning/service.go index 217a7c193..7c883c224 100644 --- a/pkg/provisioning/service.go +++ b/pkg/provisioning/service.go @@ -255,7 +255,7 @@ func (s *Service) provisionPipeline(ctx context.Context, cfg config.Pipeline) er if cfg.Status == config.StatusRunning { // TODO set status and let the pipeline service start it - err := s.lifecycleService.Start(ctx, s.connectorService, s.processorService, s.connectorPluginService, cfg.ID) + err := s.lifecycleService.Start(ctx, cfg.ID) if err != nil { return cerrors.Errorf("could not start the pipeline %q: %w", cfg.ID, err) } From 60547de3703eca1573188e6b1937fb1665854e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Tue, 10 Sep 2024 16:57:44 +0200 Subject: [PATCH 14/41] update interfaces --- pkg/lifecycle/service.go | 16 ++++++++-------- pkg/lifecycle/service_test.go | 27 +++++++++++++-------------- pkg/orchestrator/mock/orchestrator.go | 6 +++--- pkg/orchestrator/orchestrator.go | 3 +-- pkg/provisioning/mock/provisioning.go | 6 +++--- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index e516439de..1025269d5 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -55,10 +55,10 @@ type Service struct { backoffCfg *backoff.Backoff pipelines PipelineService - connectors ConnectorFetcher + connectors ConnectorService processors ProcessorService - connectorPlugins PluginDispenserFetcher + connectorPlugins ConnectorPluginService handlers []FailureHandler runningPipelines map[string]*runnablePipeline @@ -68,9 +68,9 @@ type Service struct { func NewService( logger log.CtxLogger, backoffCfg *backoff.Backoff, - connectors ConnectorFetcher, + connectors ConnectorService, processors ProcessorService, - connectorPlugins PluginDispenserFetcher, + connectorPlugins ConnectorPluginService, pipelines PipelineService, ) *Service { return &Service{ @@ -89,8 +89,8 @@ type runnablePipeline struct { t *tomb.Tomb } -// ConnectorFetcher can fetch a connector instance. -type ConnectorFetcher interface { +// ConnectorService can fetch a connector instance. +type ConnectorService interface { Get(ctx context.Context, id string) (*connector.Instance, error) Create(ctx context.Context, id string, t connector.Type, plugin string, pipelineID string, cfg connector.Config, p connector.ProvisionType) (*connector.Instance, error) } @@ -101,8 +101,8 @@ type ProcessorService interface { MakeRunnableProcessor(ctx context.Context, i *processor.Instance) (*processor.RunnableProcessor, error) } -// PluginDispenserFetcher can fetch a plugin. -type PluginDispenserFetcher interface { +// ConnectorPluginService can fetch a plugin. +type ConnectorPluginService interface { NewDispenser(logger log.CtxLogger, name string, connectorID string) (connectorPlugin.Dispenser, error) } diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 2797d97c9..d9aa6a764 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -54,8 +54,6 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ls := NewService(logger, db, b) - source := dummySource(persister) destination := dummyDestination(persister) dlq := dummyDestination(persister) @@ -72,9 +70,10 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { } pl.SetStatus(pipeline.StatusUserStopped) + ls := NewService(logger, db, b) got, err := ls.buildRunnablePipeline( ctx, - testConnectorFetcher{ + testConnectorService{ source.ID: source, destination.ID: destination, testDLQID: dlq, @@ -159,7 +158,7 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { got, err := ls.buildRunnablePipeline( ctx, - testConnectorFetcher{ + testConnectorService{ destination.ID: destination, testDLQID: dlq, }, @@ -208,7 +207,7 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { got, err := ls.buildRunnablePipeline( ctx, - testConnectorFetcher{ + testConnectorService{ source.ID: source, testDLQID: dlq, }, @@ -259,7 +258,7 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { // start the pipeline now that everything is set up err = ls.Start( ctx, - testConnectorFetcher{ + testConnectorService{ source.ID: source, destination.ID: destination, testDLQID: dlq, @@ -327,7 +326,7 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { // start the pipeline now that everything is set up err = ls.Start( ctx, - testConnectorFetcher{ + testConnectorService{ source.ID: source, destination.ID: destination, testDLQID: dlq, @@ -411,7 +410,7 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { // start the pipeline now that everything is set up err = ls.Start( ctx, - testConnectorFetcher{ + testConnectorService{ source.ID: source, destination.ID: destination, testDLQID: dlq, @@ -513,7 +512,7 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { // start the pipeline now that everything is set up err = ps.Start( ctx, - testConnectorFetcher{ + testConnectorService{ source.ID: source, destination.ID: destination, testDLQID: dlq, @@ -595,7 +594,7 @@ func TestService_Run_Rerun(t *testing.T) { // TODO: create a new instance of the lifecycle service err = ls.Run( ctx, - testConnectorFetcher{ + testConnectorService{ source.ID: source, destination.ID: destination, testDLQID: dlq, @@ -752,10 +751,10 @@ func dummyDestination(persister *connector.Persister) *connector.Instance { return destination } -// testConnectorFetcher fulfills the ConnectorFetcher interface. -type testConnectorFetcher map[string]*connector.Instance +// testConnectorService fulfills the ConnectorService interface. +type testConnectorService map[string]*connector.Instance -func (tcf testConnectorFetcher) Get(_ context.Context, id string) (*connector.Instance, error) { +func (tcf testConnectorService) Get(_ context.Context, id string) (*connector.Instance, error) { conn, ok := tcf[id] if !ok { return nil, connector.ErrInstanceNotFound @@ -763,7 +762,7 @@ func (tcf testConnectorFetcher) Get(_ context.Context, id string) (*connector.In return conn, nil } -func (tcf testConnectorFetcher) Create(context.Context, string, connector.Type, string, string, connector.Config, connector.ProvisionType) (*connector.Instance, error) { +func (tcf testConnectorService) Create(context.Context, string, connector.Type, string, string, connector.Config, connector.ProvisionType) (*connector.Instance, error) { return tcf[testDLQID], nil } diff --git a/pkg/orchestrator/mock/orchestrator.go b/pkg/orchestrator/mock/orchestrator.go index b8875a5bd..78b971bb5 100644 --- a/pkg/orchestrator/mock/orchestrator.go +++ b/pkg/orchestrator/mock/orchestrator.go @@ -358,7 +358,7 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, } // Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorFetcher, arg2 lifecycle.ProcessorService, arg3 lifecycle.PluginDispenserFetcher, arg4 string) error { +func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorService, arg2 lifecycle.ProcessorService, arg3 lifecycle.ConnectorPluginService, arg4 string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) @@ -384,13 +384,13 @@ func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall } // Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { c.Call = c.Call.DoAndReturn(f) return c } diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index 35e4badda..24513076b 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:generate mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService - +//go:generate mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService package orchestrator import ( diff --git a/pkg/provisioning/mock/provisioning.go b/pkg/provisioning/mock/provisioning.go index d3f9ffcc1..c10374a45 100644 --- a/pkg/provisioning/mock/provisioning.go +++ b/pkg/provisioning/mock/provisioning.go @@ -356,7 +356,7 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, } // Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorFetcher, arg2 lifecycle.ProcessorService, arg3 lifecycle.PluginDispenserFetcher, arg4 string) error { +func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorService, arg2 lifecycle.ProcessorService, arg3 lifecycle.ConnectorPluginService, arg4 string) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) @@ -382,13 +382,13 @@ func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall } // Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { c.Call = c.Call.Do(f) return c } // DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorFetcher, lifecycle.ProcessorService, lifecycle.PluginDispenserFetcher, string) error) *PipelineServiceStartCall { +func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { c.Call = c.Call.DoAndReturn(f) return c } From b2a41ee181cafd93401a4276e19bde6a05f71308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 10:28:49 +0200 Subject: [PATCH 15/41] more test fixes --- pkg/lifecycle/dlq_test.go | 2 +- pkg/lifecycle/service.go | 7 + pkg/lifecycle/service_test.go | 253 ++++++++++-------- .../stream/destination_acker_test.go | 2 +- pkg/lifecycle/stream/destination_test.go | 2 +- pkg/lifecycle/stream/dlq_test.go | 2 +- pkg/lifecycle/stream/parallel_test.go | 2 +- pkg/lifecycle/stream/processor_test.go | 2 +- pkg/lifecycle/stream/source_acker_test.go | 2 +- pkg/lifecycle/stream/source_test.go | 2 +- 10 files changed, 156 insertions(+), 120 deletions(-) diff --git a/pkg/lifecycle/dlq_test.go b/pkg/lifecycle/dlq_test.go index 32a54e8f6..815146041 100644 --- a/pkg/lifecycle/dlq_test.go +++ b/pkg/lifecycle/dlq_test.go @@ -22,7 +22,7 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" - streammock "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + streammock "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 1025269d5..f0d5a9f86 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -303,6 +303,13 @@ func (s *Service) waitInternal() error { return cerrors.Join(errs...) } +func (s *Service) WaitPipeline(id string) error { + if s.runningPipelines[id].t == nil { + return nil + } + return s.runningPipelines[id].t.Wait() +} + // buildRunnablePipeline will build and connect all nodes configured in the pipeline. func (s *Service) buildRunnablePipeline( ctx context.Context, diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index d9aa6a764..1b2d9c6e7 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -44,7 +44,7 @@ import ( const testDLQID = "test-dlq" -func TestServiceLifecycle_buildNodes(t *testing.T) { +func TestServiceLifecycle_buildRunnablePipeline(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -70,9 +70,9 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { } pl.SetStatus(pipeline.StatusUserStopped) - ls := NewService(logger, db, b) - got, err := ls.buildRunnablePipeline( - ctx, + ls := NewService( + logger, + b, testConnectorService{ source.ID: source, destination.ID: destination, @@ -84,25 +84,35 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { destination.Plugin: pmock.NewDispenser(ctrl), dlq.Plugin: pmock.NewDispenser(ctrl), }, + testPipelineService{}, + ) + + got, err := ls.buildRunnablePipeline( + ctx, pl, ) + is.NoErr(err) - want := []stream.Node{ - &stream.SourceNode{}, - &stream.SourceAckerNode{}, - &stream.MetricsNode{}, - &stream.DLQHandlerNode{}, - &stream.FaninNode{}, - &stream.FanoutNode{}, - &stream.MetricsNode{}, - &stream.DestinationNode{}, - &stream.DestinationAckerNode{}, + want := runnablePipeline{ + pipeline: pl, + n: []stream.Node{ + &stream.SourceNode{}, + &stream.SourceAckerNode{}, + &stream.MetricsNode{}, + &stream.DLQHandlerNode{}, + &stream.FaninNode{}, + &stream.FanoutNode{}, + &stream.MetricsNode{}, + &stream.DestinationNode{}, + &stream.DestinationAckerNode{}, + }, } - is.Equal(len(want), len(got)) - for i := range want { - want := want[i] - got := got[i] + + is.Equal(len(want.n), len(got.n)) + for i := range want.n { + want := want.n[i] + got := got.n[i] is.Equal(reflect.TypeOf(want), reflect.TypeOf(got)) // unexpected node type switch got := got.(type) { @@ -127,7 +137,7 @@ func TestServiceLifecycle_buildNodes(t *testing.T) { } } -func TestService_buildNodes_NoSourceNode(t *testing.T) { +func TestService_buildRunnablePipeline_NoSourceNode(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -137,10 +147,6 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ls := NewService(logger, db, b) - - wantErr := "can't build pipeline without any source connectors" - destination := dummyDestination(persister) dlq := dummyDestination(persister) pl := &pipeline.Instance{ @@ -156,17 +162,19 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { } pl.SetStatus(pipeline.StatusUserStopped) - got, err := ls.buildRunnablePipeline( - ctx, - testConnectorService{ - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, + ls := NewService(logger, b, testConnectorService{ + destination.ID: destination, + testDLQID: dlq, + }, testProcessorFetcher{}, testPluginFetcher{ destination.Plugin: pmock.NewDispenser(ctrl), dlq.Plugin: pmock.NewDispenser(ctrl), - }, + }, testPipelineService{}) + + wantErr := "can't build pipeline without any source connectors" + + got, err := ls.buildRunnablePipeline( + ctx, pl, ) @@ -175,7 +183,7 @@ func TestService_buildNodes_NoSourceNode(t *testing.T) { is.Equal(got, nil) } -func TestService_buildNodes_NoDestinationNode(t *testing.T) { +func TestService_buildRunnablePipeline_NoDestinationNode(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) defer killAll() @@ -185,13 +193,21 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ls := NewService(logger, db, b) - - wantErr := "can't build pipeline without any destination connectors" - source := dummySource(persister) dlq := dummyDestination(persister) + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + testDLQID: dlq, + }, + testProcessorFetcher{}, + testPluginFetcher{ + source.Plugin: pmock.NewDispenser(ctrl), + dlq.Plugin: pmock.NewDispenser(ctrl), + }, testPipelineService{}) + + wantErr := "can't build pipeline without any destination connectors" + pl := &pipeline.Instance{ ID: uuid.NewString(), Config: pipeline.Config{Name: "test-pipeline"}, @@ -207,15 +223,6 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { got, err := ls.buildRunnablePipeline( ctx, - testConnectorService{ - source.ID: source, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ - source.Plugin: pmock.NewDispenser(ctrl), - dlq.Plugin: pmock.NewDispenser(ctrl), - }, pl, ) @@ -224,6 +231,11 @@ func TestService_buildNodes_NoDestinationNode(t *testing.T) { is.Equal(got, nil) } +// TODO: Fix +// destination.go:117: err: context deadline exceeded // run didn't finish +// destination.go:117: err: context deadline exceeded // run didn't finish +// source.go:145: 0 != 10 // number of expected acks don't match +// source.go:116: not true: done.Load() // run didn't finish func TestServiceLifecycle_PipelineSuccess(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) @@ -253,22 +265,21 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - ls := NewService(logger, db, b) - - // start the pipeline now that everything is set up - err = ls.Start( - ctx, - testConnectorService{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, testProcessorFetcher{}, testPluginFetcher{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, + }, testPipelineService{}) + + // start the pipeline now that everything is set up + err = ls.Start( + ctx, pl.ID, ) is.NoErr(err) @@ -282,7 +293,7 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { // stop pipeline before ending test err = ls.Stop(ctx, pl.ID, false) is.NoErr(err) - is.NoErr(pl.Wait()) + is.NoErr(ls.WaitPipeline(pl.ID)) } func TestServiceLifecycle_PipelineError(t *testing.T) { @@ -316,40 +327,39 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - events := make(chan pipeline.FailureEvent, 1) - ps.OnFailure(func(e pipeline.FailureEvent) { + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, + testProcessorFetcher{}, + testPluginFetcher{ + source.Plugin: sourceDispenser, + destination.Plugin: destDispenser, + dlq.Plugin: dlqDispenser, + }, testPipelineService{}) + + events := make(chan FailureEvent, 1) + ls.OnFailure(func(e FailureEvent) { events <- e }) - ls := NewService(logger, db, b) - // start the pipeline now that everything is set up err = ls.Start( ctx, - testConnectorService{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, - testProcessorFetcher{}, - testPluginFetcher{ - source.Plugin: sourceDispenser, - destination.Plugin: destDispenser, - dlq.Plugin: dlqDispenser, - }, pl.ID, ) is.NoErr(err) // wait for pipeline to finish - err = pl.Wait() + err = ls.WaitPipeline(pl.ID) is.True(err != nil) is.Equal(pipeline.StatusDegraded, pl.GetStatus()) // pipeline errors contain only string messages, so we can only compare the errors by the messages t.Log(pl.Error) - event, eventReceived, err := cchan.Chan[pipeline.FailureEvent](events).RecvTimeout(ctx, 200*time.Millisecond) + event, eventReceived, err := cchan.Chan[FailureEvent](events).RecvTimeout(ctx, 200*time.Millisecond) is.NoErr(err) is.True(eventReceived) is.Equal(pl.ID, event.ID) @@ -364,6 +374,10 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { is.True(cerrors.Is(event.Error, wantErr)) } +// TODO: Fix +// destination.go:117: err: context deadline exceeded // run didn't finish +// destination.go:117: err: context deadline exceeded // run didn't finish +// source.go:116: not true: done.Load() // run didn't finish func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { type testCase struct { name string @@ -405,22 +419,21 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - ls := NewService(logger, db, b) - - // start the pipeline now that everything is set up - err = ls.Start( - ctx, - testConnectorService{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, testProcessorFetcher{}, testPluginFetcher{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, + }, testPipelineService{}) + + // start the pipeline now that everything is set up + err = ls.Start( + ctx, pl.ID, ) is.NoErr(err) @@ -429,10 +442,10 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { time.Sleep(100 * time.Millisecond) pl.SetStatus(pipeline.StatusRecovering) - tc.stopFn(ctx, is, ps, pl.ID) + tc.stopFn(ctx, is, ls, pl.ID) // wait for pipeline to finish - err = pl.Wait() + err = ls.WaitPipeline(pl.ID) if tc.wantErr != nil { is.True(err != nil) } else { @@ -463,8 +476,8 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { }, { name: "user stop (graceful)", - stopFn: func(ctx context.Context, is *is.I, ps *pipeline.Service, pipelineID string) { - err := ps.Stop(ctx, pipelineID, false) + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + err := ls.Stop(ctx, pipelineID, false) is.NoErr(err) }, wantSourceStop: true, @@ -479,6 +492,7 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { } } +// TODO: Fix errors func TestServiceLifecycle_PipelineStop(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) @@ -509,36 +523,38 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { pl, err = ps.AddConnector(ctx, pl.ID, destination.ID) is.NoErr(err) - // start the pipeline now that everything is set up - err = ps.Start( - ctx, - testConnectorService{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, testProcessorFetcher{}, testPluginFetcher{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, + }, ps) + + // start the pipeline now that everything is set up + err = ls.Start( + ctx, pl.ID, ) is.NoErr(err) // wait for pipeline to finish consuming records from the source time.Sleep(100 * time.Millisecond) - ps.StopAll(ctx, pipeline.ErrGracefulShutdown) + ls.StopAll(ctx, pipeline.ErrGracefulShutdown) // wait for pipeline to finish - err = pl.Wait() + err = ls.WaitPipeline(pl.ID) is.NoErr(err) is.Equal(pipeline.StatusSystemStopped, pl.GetStatus()) is.Equal("", pl.Error) } +// TODO: Fix errors func TestService_Run_Rerun(t *testing.T) { runTest := func(t *testing.T, status pipeline.Status, expected pipeline.Status) { is := is.New(t) @@ -591,21 +607,18 @@ func TestService_Run_Rerun(t *testing.T) { err = ps.Init(ctx) is.NoErr(err) - // TODO: create a new instance of the lifecycle service - err = ls.Run( - ctx, - testConnectorService{ - source.ID: source, - destination.ID: destination, - testDLQID: dlq, - }, + ls := NewService(logger, b, testConnectorService{ + source.ID: source, + destination.ID: destination, + testDLQID: dlq, + }, testProcessorFetcher{}, testPluginFetcher{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, - ) + }, ps) + err = ls.Run(ctx) is.NoErr(err) // give pipeline a chance to start if needed @@ -619,9 +632,8 @@ func TestService_Run_Rerun(t *testing.T) { if expected == pipeline.StatusRunning { pl, _ = ps.Get(ctx, pl.ID) - // TODO: create a new instance of the lifecycle service is.NoErr(ls.Stop(ctx, pl.ID, false)) - is.NoErr(pl.Wait()) + is.NoErr(ls.WaitPipeline(pl.ID)) } } @@ -784,6 +796,23 @@ func (tpf testProcessorFetcher) Get(_ context.Context, id string) (*processor.In // testPluginFetcher fulfills the PluginFetcher interface. type testPluginFetcher map[string]connectorPlugin.Dispenser +type testPipelineService map[string]*pipeline.Instance + +func (t testPipelineService) Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) { + //TODO implement me + panic("implement me") +} + +func (t testPipelineService) GetInstances() map[string]*pipeline.Instance { + //TODO implement me + panic("implement me") +} + +func (t testPipelineService) UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status, errMsg string) error { + //TODO implement me + panic("implement me") +} + func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string, _ string) (connectorPlugin.Dispenser, error) { plug, ok := tpf[name] if !ok { diff --git a/pkg/lifecycle/stream/destination_acker_test.go b/pkg/lifecycle/stream/destination_acker_test.go index 21a030564..6b7c7188f 100644 --- a/pkg/lifecycle/stream/destination_acker_test.go +++ b/pkg/lifecycle/stream/destination_acker_test.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/lifecycle/stream/destination_test.go b/pkg/lifecycle/stream/destination_test.go index 77f2d7f02..e4b17126a 100644 --- a/pkg/lifecycle/stream/destination_test.go +++ b/pkg/lifecycle/stream/destination_test.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/lifecycle/stream/dlq_test.go b/pkg/lifecycle/stream/dlq_test.go index 2ee6fcbfc..c0e5e25ca 100644 --- a/pkg/lifecycle/stream/dlq_test.go +++ b/pkg/lifecycle/stream/dlq_test.go @@ -27,7 +27,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/google/uuid" "github.com/matryer/is" "go.uber.org/mock/gomock" diff --git a/pkg/lifecycle/stream/parallel_test.go b/pkg/lifecycle/stream/parallel_test.go index 09db1ebc7..e7c8c7039 100644 --- a/pkg/lifecycle/stream/parallel_test.go +++ b/pkg/lifecycle/stream/parallel_test.go @@ -29,7 +29,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/lifecycle/stream/processor_test.go b/pkg/lifecycle/stream/processor_test.go index cdc4168d4..66233e5e6 100644 --- a/pkg/lifecycle/stream/processor_test.go +++ b/pkg/lifecycle/stream/processor_test.go @@ -23,7 +23,7 @@ import ( sdk "github.com/conduitio/conduit-processor-sdk" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/google/uuid" "github.com/matryer/is" "go.uber.org/mock/gomock" diff --git a/pkg/lifecycle/stream/source_acker_test.go b/pkg/lifecycle/stream/source_acker_test.go index 53d910d1b..151bfc4d2 100644 --- a/pkg/lifecycle/stream/source_acker_test.go +++ b/pkg/lifecycle/stream/source_acker_test.go @@ -26,7 +26,7 @@ import ( "github.com/conduitio/conduit-commons/csync" "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" "github.com/matryer/is" "go.uber.org/mock/gomock" ) diff --git a/pkg/lifecycle/stream/source_test.go b/pkg/lifecycle/stream/source_test.go index a910aea74..e3b8d63e4 100644 --- a/pkg/lifecycle/stream/source_test.go +++ b/pkg/lifecycle/stream/source_test.go @@ -26,7 +26,7 @@ import ( "github.com/conduitio/conduit-commons/opencdc" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/metrics/noop" - "github.com/conduitio/conduit/pkg/pipeline/stream/mock" + "github.com/conduitio/conduit/pkg/lifecycle/stream/mock" connectorPlugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/google/uuid" "github.com/matryer/is" From d6a958bc4e71e0cd8d11d98c2bde968d7682a0a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 10:32:52 +0200 Subject: [PATCH 16/41] update mocks --- pkg/lifecycle/stream/mock/destination.go | 2 +- pkg/lifecycle/stream/mock/dlq.go | 2 +- pkg/lifecycle/stream/mock/processor.go | 2 +- pkg/lifecycle/stream/mock/source.go | 2 +- pkg/orchestrator/mock/orchestrator.go | 180 +++++++++++++---------- pkg/provisioning/mock/provisioning.go | 77 ---------- 6 files changed, 105 insertions(+), 160 deletions(-) diff --git a/pkg/lifecycle/stream/mock/destination.go b/pkg/lifecycle/stream/mock/destination.go index 8efe897bc..ff75a9a58 100644 --- a/pkg/lifecycle/stream/mock/destination.go +++ b/pkg/lifecycle/stream/mock/destination.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: Destination) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: Destination) // // Generated by this command: // diff --git a/pkg/lifecycle/stream/mock/dlq.go b/pkg/lifecycle/stream/mock/dlq.go index 7de89a3c2..fc753be11 100644 --- a/pkg/lifecycle/stream/mock/dlq.go +++ b/pkg/lifecycle/stream/mock/dlq.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: DLQHandler) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: DLQHandler) // // Generated by this command: // diff --git a/pkg/lifecycle/stream/mock/processor.go b/pkg/lifecycle/stream/mock/processor.go index 1b6ed851b..8f2a64e04 100644 --- a/pkg/lifecycle/stream/mock/processor.go +++ b/pkg/lifecycle/stream/mock/processor.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: Processor) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: Processor) // // Generated by this command: // diff --git a/pkg/lifecycle/stream/mock/source.go b/pkg/lifecycle/stream/mock/source.go index f41c864f0..c339f901f 100644 --- a/pkg/lifecycle/stream/mock/source.go +++ b/pkg/lifecycle/stream/mock/source.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/pipeline/stream (interfaces: Source) +// Source: github.com/conduitio/conduit/pkg/lifecycle/stream (interfaces: Source) // // Generated by this command: // diff --git a/pkg/orchestrator/mock/orchestrator.go b/pkg/orchestrator/mock/orchestrator.go index 78b971bb5..78d6f71a5 100644 --- a/pkg/orchestrator/mock/orchestrator.go +++ b/pkg/orchestrator/mock/orchestrator.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/orchestrator (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService) +// Source: github.com/conduitio/conduit/pkg/orchestrator (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService) // // Generated by this command: // -// mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService +// mockgen -typed -destination=mock/orchestrator.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,ProcessorPluginService=ProcessorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,ProcessorPluginService,LifecycleService // // Package mock is a generated GoMock package. @@ -17,7 +17,6 @@ import ( sdk "github.com/conduitio/conduit-processor-sdk" connector "github.com/conduitio/conduit/pkg/connector" log "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/lifecycle" pipeline "github.com/conduitio/conduit/pkg/pipeline" connector0 "github.com/conduitio/conduit/pkg/plugin/connector" processor "github.com/conduitio/conduit/pkg/processor" @@ -357,82 +356,6 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, return c } -// Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorService, arg2 lifecycle.ProcessorService, arg3 lifecycle.ConnectorPluginService, arg4 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(error) - return ret0 -} - -// Start indicates an expected call of Start. -func (mr *PipelineServiceMockRecorder) Start(arg0, arg1, arg2, arg3, arg4 any) *PipelineServiceStartCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*PipelineService)(nil).Start), arg0, arg1, arg2, arg3, arg4) - return &PipelineServiceStartCall{Call: call} -} - -// PipelineServiceStartCall wrap *gomock.Call -type PipelineServiceStartCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Stop mocks base method. -func (m *PipelineService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Stop indicates an expected call of Stop. -func (mr *PipelineServiceMockRecorder) Stop(arg0, arg1, arg2 any) *PipelineServiceStopCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*PipelineService)(nil).Stop), arg0, arg1, arg2) - return &PipelineServiceStopCall{Call: call} -} - -// PipelineServiceStopCall wrap *gomock.Call -type PipelineServiceStopCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStopCall) Return(arg0 error) *PipelineServiceStopCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStopCall) Do(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Update mocks base method. func (m *PipelineService) Update(arg0 context.Context, arg1 string, arg2 pipeline.Config) (*pipeline.Instance, error) { m.ctrl.T.Helper() @@ -1376,3 +1299,102 @@ func (c *ProcessorPluginServiceRegisterStandalonePluginCall) DoAndReturn(f func( c.Call = c.Call.DoAndReturn(f) return c } + +// LifecycleService is a mock of LifecycleService interface. +type LifecycleService struct { + ctrl *gomock.Controller + recorder *LifecycleServiceMockRecorder +} + +// LifecycleServiceMockRecorder is the mock recorder for LifecycleService. +type LifecycleServiceMockRecorder struct { + mock *LifecycleService +} + +// NewLifecycleService creates a new mock instance. +func NewLifecycleService(ctrl *gomock.Controller) *LifecycleService { + mock := &LifecycleService{ctrl: ctrl} + mock.recorder = &LifecycleServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *LifecycleService) EXPECT() *LifecycleServiceMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *LifecycleService) Start(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *LifecycleServiceMockRecorder) Start(arg0, arg1 any) *LifecycleServiceStartCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*LifecycleService)(nil).Start), arg0, arg1) + return &LifecycleServiceStartCall{Call: call} +} + +// LifecycleServiceStartCall wrap *gomock.Call +type LifecycleServiceStartCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStartCall) Return(arg0 error) *LifecycleServiceStartCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStartCall) Do(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStartCall) DoAndReturn(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Stop mocks base method. +func (m *LifecycleService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *LifecycleServiceMockRecorder) Stop(arg0, arg1, arg2 any) *LifecycleServiceStopCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*LifecycleService)(nil).Stop), arg0, arg1, arg2) + return &LifecycleServiceStopCall{Call: call} +} + +// LifecycleServiceStopCall wrap *gomock.Call +type LifecycleServiceStopCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStopCall) Return(arg0 error) *LifecycleServiceStopCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStopCall) Do(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/pkg/provisioning/mock/provisioning.go b/pkg/provisioning/mock/provisioning.go index c10374a45..f4305d536 100644 --- a/pkg/provisioning/mock/provisioning.go +++ b/pkg/provisioning/mock/provisioning.go @@ -15,7 +15,6 @@ import ( connector "github.com/conduitio/conduit/pkg/connector" log "github.com/conduitio/conduit/pkg/foundation/log" - "github.com/conduitio/conduit/pkg/lifecycle" pipeline "github.com/conduitio/conduit/pkg/pipeline" connector0 "github.com/conduitio/conduit/pkg/plugin/connector" processor "github.com/conduitio/conduit/pkg/processor" @@ -355,82 +354,6 @@ func (c *PipelineServiceRemoveProcessorCall) DoAndReturn(f func(context.Context, return c } -// Start mocks base method. -func (m *PipelineService) Start(arg0 context.Context, arg1 lifecycle.ConnectorService, arg2 lifecycle.ProcessorService, arg3 lifecycle.ConnectorPluginService, arg4 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Start", arg0, arg1, arg2, arg3, arg4) - ret0, _ := ret[0].(error) - return ret0 -} - -// Start indicates an expected call of Start. -func (mr *PipelineServiceMockRecorder) Start(arg0, arg1, arg2, arg3, arg4 any) *PipelineServiceStartCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*PipelineService)(nil).Start), arg0, arg1, arg2, arg3, arg4) - return &PipelineServiceStartCall{Call: call} -} - -// PipelineServiceStartCall wrap *gomock.Call -type PipelineServiceStartCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStartCall) Return(arg0 error) *PipelineServiceStartCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStartCall) Do(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStartCall) DoAndReturn(f func(context.Context, lifecycle.ConnectorService, lifecycle.ProcessorService, lifecycle.ConnectorPluginService, string) error) *PipelineServiceStartCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - -// Stop mocks base method. -func (m *PipelineService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Stop indicates an expected call of Stop. -func (mr *PipelineServiceMockRecorder) Stop(arg0, arg1, arg2 any) *PipelineServiceStopCall { - mr.mock.ctrl.T.Helper() - call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*PipelineService)(nil).Stop), arg0, arg1, arg2) - return &PipelineServiceStopCall{Call: call} -} - -// PipelineServiceStopCall wrap *gomock.Call -type PipelineServiceStopCall struct { - *gomock.Call -} - -// Return rewrite *gomock.Call.Return -func (c *PipelineServiceStopCall) Return(arg0 error) *PipelineServiceStopCall { - c.Call = c.Call.Return(arg0) - return c -} - -// Do rewrite *gomock.Call.Do -func (c *PipelineServiceStopCall) Do(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.Do(f) - return c -} - -// DoAndReturn rewrite *gomock.Call.DoAndReturn -func (c *PipelineServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *PipelineServiceStopCall { - c.Call = c.Call.DoAndReturn(f) - return c -} - // Update mocks base method. func (m *PipelineService) Update(arg0 context.Context, arg1 string, arg2 pipeline.Config) (*pipeline.Instance, error) { m.ctrl.T.Helper() From fbdbfa3bb0b356a637482227031c01da3c84336e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 14:23:23 +0200 Subject: [PATCH 17/41] add mocks and update tests --- pkg/conduit/runtime.go | 2 +- pkg/orchestrator/connectors_test.go | 64 ++++++++-------- pkg/orchestrator/orchestrator_test.go | 10 ++- pkg/orchestrator/pipelines_test.go | 80 ++++++++++---------- pkg/orchestrator/processors_test.go | 80 ++++++++++---------- pkg/provisioning/import_test.go | 45 +++++------ pkg/provisioning/interfaces.go | 2 +- pkg/provisioning/mock/provisioning.go | 103 +++++++++++++++++++++++++- pkg/provisioning/service.go | 2 + pkg/provisioning/service_test.go | 40 +++++----- pkg/provisioning/test/dest-file.txt | 1 + 11 files changed, 269 insertions(+), 160 deletions(-) create mode 100644 pkg/provisioning/test/dest-file.txt diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index 160929369..ac45131e6 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -212,7 +212,7 @@ func createServices(r *Runtime) error { connService := connector.NewService(r.logger, r.DB, r.connectorPersister) procService := processor.NewService(r.logger, r.DB, procPluginService) lifecycleService := lifecycle.NewService(r.logger, backoffCfg, connService, procService, connPluginService, plService) - provisionService := provisioning.NewService(r.DB, r.logger, plService, connService, procService, connPluginService, r.Config.Pipelines.Path) + provisionService := provisioning.NewService(r.DB, r.logger, plService, connService, procService, connPluginService, lifecycleService, r.Config.Pipelines.Path) orc := orchestrator.NewOrchestrator(r.DB, r.logger, plService, connService, procService, connPluginService, procPluginService, lifecycleService) diff --git a/pkg/orchestrator/connectors_test.go b/pkg/orchestrator/connectors_test.go index e26a76158..7a448b6da 100644 --- a/pkg/orchestrator/connectors_test.go +++ b/pkg/orchestrator/connectors_test.go @@ -33,7 +33,7 @@ func TestConnectorOrchestrator_Create_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -79,7 +79,7 @@ func TestConnectorOrchestrator_Create_Success(t *testing.T) { AddConnector(gomock.AssignableToTypeOf(ctxType), pl.ID, want.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, want.Type, want.Plugin, want.PipelineID, want.Config) is.NoErr(err) is.Equal(want, got) @@ -89,7 +89,7 @@ func TestConnectorOrchestrator_Create_PipelineNotExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pipelineID := uuid.NewString() wantErr := pipeline.ErrInstanceNotFound @@ -97,7 +97,7 @@ func TestConnectorOrchestrator_Create_PipelineNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pipelineID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pipelineID, connector.Config{}) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -108,7 +108,7 @@ func TestConnectorOrchestrator_Create_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -119,7 +119,7 @@ func TestConnectorOrchestrator_Create_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pl.ID, connector.Config{}) is.True(err != nil) is.True(cerrors.Is(err, pipeline.ErrPipelineRunning)) @@ -130,7 +130,7 @@ func TestConnectorOrchestrator_Create_PipelineProvisionByConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -142,7 +142,7 @@ func TestConnectorOrchestrator_Create_PipelineProvisionByConfig(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pl.ID, connector.Config{}) is.Equal(got, nil) is.True(err != nil) @@ -153,7 +153,7 @@ func TestConnectorOrchestrator_Create_CreateConnectorError(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -183,7 +183,7 @@ func TestConnectorOrchestrator_Create_CreateConnectorError(t *testing.T) { ). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, "test-plugin", pl.ID, config) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -194,7 +194,7 @@ func TestConnectorOrchestrator_Create_AddConnectorError(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -246,7 +246,7 @@ func TestConnectorOrchestrator_Create_AddConnectorError(t *testing.T) { Delete(gomock.AssignableToTypeOf(ctxType), conn.ID, connPluginMock). Return(nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Create(ctx, connector.TypeSource, conn.Plugin, pl.ID, conn.Config) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -257,7 +257,7 @@ func TestConnectorOrchestrator_Delete_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -282,7 +282,7 @@ func TestConnectorOrchestrator_Delete_Success(t *testing.T) { RemoveConnector(gomock.AssignableToTypeOf(ctxType), pl.ID, conn.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.NoErr(err) } @@ -291,7 +291,7 @@ func TestConnectorOrchestrator_Delete_ConnectorNotExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) id := uuid.NewString() wantErr := cerrors.New("connector doesn't exist") @@ -299,7 +299,7 @@ func TestConnectorOrchestrator_Delete_ConnectorNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), id). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, id) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -309,7 +309,7 @@ func TestConnectorOrchestrator_Delete_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -328,7 +328,7 @@ func TestConnectorOrchestrator_Delete_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -338,7 +338,7 @@ func TestConnectorOrchestrator_Delete_ProcessorAttached(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -355,7 +355,7 @@ func TestConnectorOrchestrator_Delete_ProcessorAttached(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), conn.ID). Return(conn, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.True(cerrors.Is(err, ErrConnectorHasProcessorsAttached)) @@ -365,7 +365,7 @@ func TestConnectorOrchestrator_Delete_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -388,7 +388,7 @@ func TestConnectorOrchestrator_Delete_Fail(t *testing.T) { Delete(gomock.AssignableToTypeOf(ctxType), conn.ID, connPluginMock). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -398,7 +398,7 @@ func TestConnectorOrchestrator_Delete_RemoveConnectorFailed(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -437,7 +437,7 @@ func TestConnectorOrchestrator_Delete_RemoveConnectorFailed(t *testing.T) { ). Return(conn, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Connectors.Delete(ctx, conn.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) @@ -447,7 +447,7 @@ func TestConnectorOrchestrator_Update_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -488,7 +488,7 @@ func TestConnectorOrchestrator_Update_Success(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), conn.ID, newConfig). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, conn.ID, newConfig) is.NoErr(err) is.Equal(got, want) @@ -498,7 +498,7 @@ func TestConnectorOrchestrator_Update_ConnectorNotExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) id := uuid.NewString() wantErr := cerrors.New("connector doesn't exist") @@ -506,7 +506,7 @@ func TestConnectorOrchestrator_Update_ConnectorNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), id). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, id, connector.Config{}) is.True(got == nil) is.True(err != nil) @@ -517,7 +517,7 @@ func TestConnectorOrchestrator_Update_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -536,7 +536,7 @@ func TestConnectorOrchestrator_Update_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, conn.ID, connector.Config{}) is.True(got == nil) is.True(err != nil) @@ -547,7 +547,7 @@ func TestConnectorOrchestrator_Update_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -574,7 +574,7 @@ func TestConnectorOrchestrator_Update_Fail(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), conn.ID, connector.Config{}). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Connectors.Update(ctx, conn.ID, connector.Config{}) is.True(got == nil) is.True(err != nil) diff --git a/pkg/orchestrator/orchestrator_test.go b/pkg/orchestrator/orchestrator_test.go index c2d336514..cb5b6df83 100644 --- a/pkg/orchestrator/orchestrator_test.go +++ b/pkg/orchestrator/orchestrator_test.go @@ -47,14 +47,15 @@ import ( // a context is passed to a function. var ctxType = reflect.TypeOf((*context.Context)(nil)).Elem() -func newMockServices(t *testing.T) (*mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService, *mock.ProcessorPluginService) { +func newMockServices(t *testing.T) (*mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService, *mock.ProcessorPluginService, *mock.LifecycleService) { ctrl := gomock.NewController(t) return mock.NewPipelineService(ctrl), mock.NewConnectorService(ctrl), mock.NewProcessorService(ctrl), mock.NewConnectorPluginService(ctrl), - mock.NewProcessorPluginService(ctrl) + mock.NewProcessorPluginService(ctrl), + mock.NewLifecycleService(ctrl) } func TestPipelineSimple(t *testing.T) { @@ -91,6 +92,8 @@ func TestPipelineSimple(t *testing.T) { nil, ) + lifecycleService := mock.NewLifecycleService(gomock.NewController(t)) + b := &backoff.Backoff{} orc := NewOrchestrator( @@ -101,6 +104,7 @@ func TestPipelineSimple(t *testing.T) { processor.NewService(logger, db, procPluginService), connPluginService, procPluginService, + lifecycleService, ) // create a host pipeline @@ -161,7 +165,7 @@ func TestPipelineSimple(t *testing.T) { err = orc.Pipelines.Stop(ctx, pl.ID, false) is.NoErr(err) t.Log("waiting") - err = pl.Wait() + err = lifecycleService.Wait() is.NoErr(err) t.Log("successfully stopped pipeline") diff --git a/pkg/orchestrator/pipelines_test.go b/pkg/orchestrator/pipelines_test.go index 19a229317..1ca657047 100644 --- a/pkg/orchestrator/pipelines_test.go +++ b/pkg/orchestrator/pipelines_test.go @@ -31,17 +31,17 @@ func TestPipelineOrchestrator_Start_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) - plsMock.EXPECT(). - Start(gomock.AssignableToTypeOf(ctxType), orc.Pipelines.connectors, orc.Pipelines.processors, orc.Pipelines.connectorPlugins, plBefore.ID). + lifecycleMock.EXPECT(). + Start(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(nil) err := orc.Pipelines.Start(ctx, plBefore.ID) @@ -52,7 +52,7 @@ func TestPipelineOrchestrator_Start_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -60,11 +60,11 @@ func TestPipelineOrchestrator_Start_Fail(t *testing.T) { plBefore.SetStatus(pipeline.StatusSystemStopped) wantErr := cerrors.New("pipeline doesn't exist") - plsMock.EXPECT(). - Start(gomock.AssignableToTypeOf(ctxType), consMock, procsMock, connPluginMock, gomock.AssignableToTypeOf("")). + lifecycleMock.EXPECT(). + Start(gomock.AssignableToTypeOf(ctxType), gomock.AssignableToTypeOf("")). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Pipelines.Start(ctx, plBefore.ID) is.True(cerrors.Is(err, wantErr)) } @@ -73,15 +73,15 @@ func TestPipelineOrchestrator_Stop_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusRunning) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) - plsMock.EXPECT(). + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) + lifecycleMock.EXPECT(). Stop(gomock.AssignableToTypeOf(ctxType), plBefore.ID, false). Return(nil) @@ -93,7 +93,7 @@ func TestPipelineOrchestrator_Stop_Fail(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -101,11 +101,11 @@ func TestPipelineOrchestrator_Stop_Fail(t *testing.T) { plBefore.SetStatus(pipeline.StatusRunning) wantErr := cerrors.New("pipeline doesn't exist") - plsMock.EXPECT(). + lifecycleMock.EXPECT(). Stop(gomock.AssignableToTypeOf(ctxType), gomock.AssignableToTypeOf(""), true). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Pipelines.Stop(ctx, plBefore.ID, true) is.True(cerrors.Is(err, wantErr)) } @@ -114,7 +114,7 @@ func TestPipelineOrchestrator_Update_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -129,7 +129,7 @@ func TestPipelineOrchestrator_Update_Success(t *testing.T) { } want.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -146,7 +146,7 @@ func TestPipelineOrchestrator_Update_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -156,7 +156,7 @@ func TestPipelineOrchestrator_Update_PipelineRunning(t *testing.T) { newConfig := pipeline.Config{Name: "new pipeline"} - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -170,7 +170,7 @@ func TestPipelineOrchestrator_Update_PipelineProvisionedByConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -181,7 +181,7 @@ func TestPipelineOrchestrator_Update_PipelineProvisionedByConfig(t *testing.T) { newConfig := pipeline.Config{Name: "new pipeline"} - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -195,14 +195,14 @@ func TestPipelineOrchestrator_Delete_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -218,14 +218,14 @@ func TestPipelineOrchestrator_Delete_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusRunning) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -238,7 +238,7 @@ func TestPipelineOrchestrator_Delete_PipelineProvisionedByConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -246,7 +246,7 @@ func TestPipelineOrchestrator_Delete_PipelineProvisionedByConfig(t *testing.T) { } plBefore.SetStatus(pipeline.StatusUserStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -259,7 +259,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasProcessorsAttached(t *testing.T) is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -267,7 +267,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasProcessorsAttached(t *testing.T) } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -280,7 +280,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasConnectorsAttached(t *testing.T) is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -288,7 +288,7 @@ func TestPipelineOrchestrator_Delete_PipelineHasConnectorsAttached(t *testing.T) } plBefore.SetStatus(pipeline.StatusSystemStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -301,10 +301,10 @@ func TestPipelineOrchestrator_Delete_PipelineDoesntExist(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) wantErr := cerrors.New("pipeline doesn't exist") - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), gomock.AssignableToTypeOf("")). Return(nil, wantErr) @@ -317,7 +317,7 @@ func TestPipelineOrchestrator_UpdateDLQ_Success(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -344,7 +344,7 @@ func TestPipelineOrchestrator_UpdateDLQ_Success(t *testing.T) { } want.SetStatus(plBefore.GetStatus()) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -364,14 +364,14 @@ func TestPipelineOrchestrator_UpdateDLQ_PipelineRunning(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), } plBefore.SetStatus(pipeline.StatusRunning) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -385,7 +385,7 @@ func TestPipelineOrchestrator_UpdateDLQ_PipelineProvisionedByConfig(t *testing.T is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -393,7 +393,7 @@ func TestPipelineOrchestrator_UpdateDLQ_PipelineProvisionedByConfig(t *testing.T } plBefore.SetStatus(pipeline.StatusUserStopped) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) @@ -407,7 +407,7 @@ func TestConnectorOrchestrator_UpdateDLQ_InvalidConfig(t *testing.T) { is := is.New(t) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) plBefore := &pipeline.Instance{ ID: uuid.NewString(), @@ -423,7 +423,7 @@ func TestConnectorOrchestrator_UpdateDLQ_InvalidConfig(t *testing.T) { } wantErr := cerrors.New("invalid plugin config") - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) plsMock.EXPECT(). Get(gomock.AssignableToTypeOf(ctxType), plBefore.ID). Return(plBefore, nil) diff --git a/pkg/orchestrator/processors_test.go b/pkg/orchestrator/processors_test.go index fdee6acdd..8fc9868d9 100644 --- a/pkg/orchestrator/processors_test.go +++ b/pkg/orchestrator/processors_test.go @@ -34,7 +34,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -72,7 +72,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_Success(t *testing.T) { AddProcessor(gomock.AssignableToTypeOf(ctxType), pl.ID, want.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, want.Plugin, want.Parent, want.Config, want.Condition) is.NoErr(err) is.Equal(want, got) @@ -83,7 +83,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineNotExist(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) parent := processor.Parent{ ID: uuid.NewString(), @@ -94,7 +94,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineNotExist(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), parent.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -106,7 +106,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineRunning(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -121,7 +121,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -133,7 +133,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineProvisionedByConfig(t *t ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -149,7 +149,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_PipelineProvisionedByConfig(t *t Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.Equal(got, nil) is.True(err != nil) @@ -161,7 +161,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_CreateProcessorError(t *testing. ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -189,7 +189,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_CreateProcessorError(t *testing. ). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -201,7 +201,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_AddProcessorError(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -244,7 +244,7 @@ func TestProcessorOrchestrator_CreateOnPipeline_AddProcessorError(t *testing.T) Delete(gomock.AssignableToTypeOf(ctxType), proc.ID). Return(nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, proc.Plugin, proc.Parent, proc.Config, proc.Condition) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -256,7 +256,7 @@ func TestProcessorOrchestrator_CreateOnConnector_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -301,7 +301,7 @@ func TestProcessorOrchestrator_CreateOnConnector_Success(t *testing.T) { AddProcessor(gomock.AssignableToTypeOf(ctxType), conn.ID, want.ID). Return(conn, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, want.Plugin, want.Parent, want.Config, want.Condition) is.NoErr(err) is.Equal(want, got) @@ -312,7 +312,7 @@ func TestProcessorOrchestrator_CreateOnConnector_ConnectorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) parent := processor.Parent{ ID: uuid.NewString(), @@ -323,7 +323,7 @@ func TestProcessorOrchestrator_CreateOnConnector_ConnectorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), parent.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Create(ctx, "test-processor", parent, processor.Config{}, "") is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -335,7 +335,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -378,7 +378,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_Success(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), want.ID, want.Config). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.NoErr(err) is.Equal(want, got) @@ -389,7 +389,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -416,7 +416,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), before.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match") @@ -428,7 +428,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_PipelineRunning(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -457,7 +457,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -469,7 +469,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorProvisionedByConfig(t * ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -497,7 +497,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_ProcessorProvisionedByConfig(t * Get(gomock.AssignableToTypeOf(ctxType), before.ID). Return(before, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.Equal(got, nil) is.True(err != nil) @@ -509,7 +509,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_UpdateFail(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -553,7 +553,7 @@ func TestProcessorOrchestrator_UpdateOnPipeline_UpdateFail(t *testing.T) { Update(gomock.AssignableToTypeOf(ctxType), want.ID, want.Config). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, before.ID, newConfig) is.True(err != nil) is.Equal(wantErr, err) @@ -565,7 +565,7 @@ func TestProcessorOrchestrator_UpdateOnConnector_ConnectorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) connID := uuid.NewString() want := &processor.Instance{ @@ -585,7 +585,7 @@ func TestProcessorOrchestrator_UpdateOnConnector_ConnectorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), connID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) got, err := orc.Processors.Update(ctx, want.ID, processor.Config{}) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -597,7 +597,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Success(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -629,7 +629,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Success(t *testing.T) { RemoveProcessor(gomock.AssignableToTypeOf(ctxType), pl.ID, want.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.NoErr(err) } @@ -639,7 +639,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_ProcessorNotExist(t *testing.T) ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -663,7 +663,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_ProcessorNotExist(t *testing.T) Get(gomock.AssignableToTypeOf(ctxType), want.ID). Return(nil, wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -674,7 +674,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_PipelineRunning(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -699,7 +699,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_PipelineRunning(t *testing.T) { Get(gomock.AssignableToTypeOf(ctxType), pl.ID). Return(pl, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) is.Equal(pipeline.ErrPipelineRunning, err) @@ -710,7 +710,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Fail(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -740,7 +740,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_Fail(t *testing.T) { Delete(gomock.AssignableToTypeOf(ctxType), want.ID). Return(wantErr) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) is.True(cerrors.Is(err, wantErr)) // errors did not match @@ -751,7 +751,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_RemoveProcessorFail(t *testing.T ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -797,7 +797,7 @@ func TestProcessorOrchestrator_DeleteOnPipeline_RemoveProcessorFail(t *testing.T ). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) } @@ -807,7 +807,7 @@ func TestProcessorOrchestrator_DeleteOnConnector_Fail(t *testing.T) { ctx := context.Background() db := &inmemory.DB{} - plsMock, consMock, procsMock, connPluginMock, procPluginMock := newMockServices(t) + plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock := newMockServices(t) pl := &pipeline.Instance{ ID: uuid.NewString(), @@ -860,7 +860,7 @@ func TestProcessorOrchestrator_DeleteOnConnector_Fail(t *testing.T) { ). Return(want, nil) - orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock) + orc := NewOrchestrator(db, log.Nop(), plsMock, consMock, procsMock, connPluginMock, procPluginMock, lifecycleMock) err := orc.Processors.Delete(ctx, want.ID) is.True(err != nil) } diff --git a/pkg/provisioning/import_test.go b/pkg/provisioning/import_test.go index b32603b8f..bd85ab461 100644 --- a/pkg/provisioning/import_test.go +++ b/pkg/provisioning/import_test.go @@ -35,7 +35,7 @@ func TestService_ExecuteActions_Success(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{do: func(ctx context.Context) error { @@ -57,7 +57,7 @@ func TestService_ExecuteActions_Fail(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{do: func(ctx context.Context) error { @@ -85,7 +85,7 @@ func TestService_RollbackActions_Success(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{rollback: func(ctx context.Context) error { @@ -106,7 +106,7 @@ func TestService_RollbackActions_Fail(t *testing.T) { ctrl := gomock.NewController(t) ctx := context.Background() - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) var actionCount int countingAction := fakeAction{rollback: func(ctx context.Context) error { @@ -131,7 +131,7 @@ func TestActionBuilder_Build(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, connSrv, procSrv, connPlugSrv := newTestService(ctrl, logger) + srv, pipSrv, connSrv, procSrv, connPlugSrv, _ := newTestService(ctrl, logger) oldConfig := config.Pipeline{ ID: "config-id", @@ -353,7 +353,7 @@ func TestActionsBuilder_PreparePipelineActions_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, _, _, _ := newTestService(ctrl, logger) + srv, pipSrv, _, _, _, _ := newTestService(ctrl, logger) oldConfig := config.Pipeline{} newConfig := config.Pipeline{ID: "config-id"} @@ -372,7 +372,7 @@ func TestActionsBuilder_PreparePipelineActions_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, _, _, _ := newTestService(ctrl, logger) + srv, pipSrv, _, _, _, _ := newTestService(ctrl, logger) oldConfig := config.Pipeline{ID: "config-id"} newConfig := config.Pipeline{} @@ -390,7 +390,7 @@ func TestActionsBuilder_PreparePipelineActions_NoAction(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -451,7 +451,7 @@ func TestActionsBuilder_PreparePipelineActions_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, pipSrv, _, _, _ := newTestService(ctrl, logger) + srv, pipSrv, _, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -503,7 +503,7 @@ func TestActionsBuilder_PrepareConnectorActions_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, connPlugSrv := newTestService(ctrl, logger) + srv, _, connSrv, _, connPlugSrv, _ := newTestService(ctrl, logger) oldConfig := config.Connector{} newConfig := config.Connector{ID: "config-id"} @@ -525,7 +525,7 @@ func TestActionsBuilder_PrepareConnectorActions_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, connPlugSrv := newTestService(ctrl, logger) + srv, _, connSrv, _, connPlugSrv, _ := newTestService(ctrl, logger) oldConfig := config.Connector{ID: "config-id"} newConfig := config.Connector{} @@ -546,7 +546,7 @@ func TestActionsBuilder_PrepareConnectorActions_NoAction(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -585,7 +585,7 @@ func TestActionsBuilder_PrepareConnectorActions_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, _ := newTestService(ctrl, logger) + srv, _, connSrv, _, _, _ := newTestService(ctrl, logger) testCases := []struct { name string @@ -623,7 +623,7 @@ func TestActionsBuilder_PrepareConnectorActions_Recreate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, connSrv, _, connPlugSrv := newTestService(ctrl, logger) + srv, _, connSrv, _, connPlugSrv, _ := newTestService(ctrl, logger) pipelineID := uuid.NewString() testCases := []struct { @@ -665,7 +665,7 @@ func TestActionsBuilder_PrepareProcessorActions_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) oldConfig := config.Processor{} newConfig := config.Processor{ID: "config-id"} @@ -689,7 +689,7 @@ func TestActionsBuilder_PrepareProcessorActions_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) oldConfig := config.Processor{ID: "config-id"} newConfig := config.Processor{} @@ -713,7 +713,7 @@ func TestActionsBuilder_PrepareProcessorActions_NoAction(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, _, _ := newTestService(ctrl, logger) + srv, _, _, _, _, _ := newTestService(ctrl, logger) oldConfig := config.Processor{ID: "config-id"} newConfig := config.Processor{ID: "config-id"} @@ -730,7 +730,7 @@ func TestActionsBuilder_PrepareProcessorActions_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) parent := processor.Parent{ ID: uuid.NewString(), Type: processor.ParentTypeConnector, @@ -768,7 +768,7 @@ func TestActionsBuilder_PrepareProcessorActions_Recreate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - srv, _, _, procSrv, _ := newTestService(ctrl, logger) + srv, _, _, procSrv, _, _ := newTestService(ctrl, logger) parent := processor.Parent{ ID: uuid.NewString(), Type: processor.ParentTypePipeline, @@ -808,16 +808,17 @@ func TestActionsBuilder_PrepareProcessorActions_Recreate(t *testing.T) { func intPtr(i int) *int { return &i } -func newTestService(ctrl *gomock.Controller, logger log.CtxLogger) (*Service, *mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService) { +func newTestService(ctrl *gomock.Controller, logger log.CtxLogger) (*Service, *mock.PipelineService, *mock.ConnectorService, *mock.ProcessorService, *mock.ConnectorPluginService, *mock.LifecycleService) { db := &inmemory.DB{} pipSrv := mock.NewPipelineService(ctrl) connSrv := mock.NewConnectorService(ctrl) procSrv := mock.NewProcessorService(ctrl) connPlugSrv := mock.NewConnectorPluginService(ctrl) + lifecycleSrv := mock.NewLifecycleService(ctrl) - srv := NewService(db, logger, pipSrv, connSrv, procSrv, connPlugSrv, "") + srv := NewService(db, logger, pipSrv, connSrv, procSrv, connPlugSrv, lifecycleSrv, "") - return srv, pipSrv, connSrv, procSrv, connPlugSrv + return srv, pipSrv, connSrv, procSrv, connPlugSrv, lifecycleSrv } type fakeAction struct { diff --git a/pkg/provisioning/interfaces.go b/pkg/provisioning/interfaces.go index a11f96db7..21f2142e2 100644 --- a/pkg/provisioning/interfaces.go +++ b/pkg/provisioning/interfaces.go @@ -24,7 +24,7 @@ import ( "github.com/conduitio/conduit/pkg/processor" ) -//go:generate mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService +//go:generate mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,LifecycleService type PipelineService interface { Get(ctx context.Context, id string) (*pipeline.Instance, error) diff --git a/pkg/provisioning/mock/provisioning.go b/pkg/provisioning/mock/provisioning.go index f4305d536..ae298ff97 100644 --- a/pkg/provisioning/mock/provisioning.go +++ b/pkg/provisioning/mock/provisioning.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/conduitio/conduit/pkg/provisioning (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService) +// Source: github.com/conduitio/conduit/pkg/provisioning (interfaces: PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,LifecycleService) // // Generated by this command: // -// mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService +// mockgen -typed -destination=mock/provisioning.go -package=mock -mock_names=PipelineService=PipelineService,ConnectorService=ConnectorService,ProcessorService=ProcessorService,ConnectorPluginService=ConnectorPluginService,LifecycleService=LifecycleService . PipelineService,ConnectorService,ProcessorService,ConnectorPluginService,LifecycleService // // Package mock is a generated GoMock package. @@ -966,3 +966,102 @@ func (c *ConnectorPluginServiceNewDispenserCall) DoAndReturn(f func(log.CtxLogge c.Call = c.Call.DoAndReturn(f) return c } + +// LifecycleService is a mock of LifecycleService interface. +type LifecycleService struct { + ctrl *gomock.Controller + recorder *LifecycleServiceMockRecorder +} + +// LifecycleServiceMockRecorder is the mock recorder for LifecycleService. +type LifecycleServiceMockRecorder struct { + mock *LifecycleService +} + +// NewLifecycleService creates a new mock instance. +func NewLifecycleService(ctrl *gomock.Controller) *LifecycleService { + mock := &LifecycleService{ctrl: ctrl} + mock.recorder = &LifecycleServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *LifecycleService) EXPECT() *LifecycleServiceMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *LifecycleService) Start(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *LifecycleServiceMockRecorder) Start(arg0, arg1 any) *LifecycleServiceStartCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*LifecycleService)(nil).Start), arg0, arg1) + return &LifecycleServiceStartCall{Call: call} +} + +// LifecycleServiceStartCall wrap *gomock.Call +type LifecycleServiceStartCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStartCall) Return(arg0 error) *LifecycleServiceStartCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStartCall) Do(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStartCall) DoAndReturn(f func(context.Context, string) error) *LifecycleServiceStartCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + +// Stop mocks base method. +func (m *LifecycleService) Stop(arg0 context.Context, arg1 string, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *LifecycleServiceMockRecorder) Stop(arg0, arg1, arg2 any) *LifecycleServiceStopCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*LifecycleService)(nil).Stop), arg0, arg1, arg2) + return &LifecycleServiceStopCall{Call: call} +} + +// LifecycleServiceStopCall wrap *gomock.Call +type LifecycleServiceStopCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *LifecycleServiceStopCall) Return(arg0 error) *LifecycleServiceStopCall { + c.Call = c.Call.Return(arg0) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *LifecycleServiceStopCall) Do(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *LifecycleServiceStopCall) DoAndReturn(f func(context.Context, string, bool) error) *LifecycleServiceStopCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/pkg/provisioning/service.go b/pkg/provisioning/service.go index 7c883c224..803e50fa7 100644 --- a/pkg/provisioning/service.go +++ b/pkg/provisioning/service.go @@ -51,6 +51,7 @@ func NewService( connService ConnectorService, procService ProcessorService, connPluginService ConnectorPluginService, + lifecycleService LifecycleService, pipelinesDir string, ) *Service { return &Service{ @@ -61,6 +62,7 @@ func NewService( connectorService: connService, processorService: procService, connectorPluginService: connPluginService, + lifecycleService: lifecycleService, pipelinesPath: pipelinesDir, } } diff --git a/pkg/provisioning/service_test.go b/pkg/provisioning/service_test.go index e1308df99..544b2628e 100644 --- a/pkg/provisioning/service_test.go +++ b/pkg/provisioning/service_test.go @@ -26,6 +26,7 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/ctxutil" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/pipeline" conn_plugin "github.com/conduitio/conduit/pkg/plugin/connector" "github.com/conduitio/conduit/pkg/plugin/connector/builtin" @@ -117,7 +118,7 @@ func TestService_Init_Create(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -141,7 +142,7 @@ func TestService_Init_Create(t *testing.T) { procService.EXPECT().CreateWithInstance(anyCtx, p1.P1P1) // start pipeline - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID) err := service.Init(context.Background()) is.NoErr(err) @@ -152,7 +153,7 @@ func TestService_Init_Update(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" pipelineService.EXPECT().List(anyCtx) @@ -176,7 +177,7 @@ func TestService_Init_Update(t *testing.T) { procService.EXPECT().Update(anyCtx, p1.P1P1.ID, p1.P1P1.Config) // start pipeline - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID) err := service.Init(context.Background()) is.NoErr(err) @@ -187,7 +188,7 @@ func TestService_Init_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, plugService, _ := newTestService(ctrl, logger) service.pipelinesPath = "" // don't init any pipelines, delete existing ones // old pipeline exists @@ -216,7 +217,7 @@ func TestService_Init_NoRollbackOnFailedStart(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -241,7 +242,7 @@ func TestService_Init_NoRollbackOnFailedStart(t *testing.T) { // returns an error, no rollback needed wantErr := cerrors.New("error") - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID).Return(wantErr) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID).Return(wantErr) err := service.Init(context.Background()) is.True(cerrors.Is(err, wantErr)) @@ -252,7 +253,7 @@ func TestService_Init_RollbackCreate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, plugService, _ := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -291,7 +292,7 @@ func TestService_Init_RollbackUpdate(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, _ := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, _ := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // return a pipeline not provisioned by API @@ -331,7 +332,7 @@ func TestService_Init_MultiplePipelinesDuplicatedPipelineID(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, _, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines2" // return a pipeline not provisioned by API @@ -349,7 +350,7 @@ func TestService_Init_MultiplePipelinesDuplicatedPipelineID(t *testing.T) { connService.EXPECT().CreateWithInstance(anyCtx, p2.P2C1) connService.EXPECT().CreateWithInstance(anyCtx, p2.P2C2) - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p2.P2.ID) + lifecycleService.EXPECT().Start(anyCtx, p2.P2.ID) err := service.Init(context.Background()) is.True(cerrors.Is(err, ErrDuplicatedPipelineID)) // duplicated pipeline id @@ -360,7 +361,7 @@ func TestService_Init_MultiplePipelines(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, _, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines3" // return a pipeline not provisioned by API @@ -380,7 +381,7 @@ func TestService_Init_MultiplePipelines(t *testing.T) { connService.EXPECT().CreateWithInstance(anyCtx, p3.P1C1) connService.EXPECT().CreateWithInstance(anyCtx, p3.P1C2) - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p3.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p3.P1.ID) // create pipeline2 pipelineService.EXPECT().CreateWithInstance(anyCtx, p3.P2) @@ -391,7 +392,7 @@ func TestService_Init_MultiplePipelines(t *testing.T) { connService.EXPECT().CreateWithInstance(anyCtx, p3.P2C1) connService.EXPECT().CreateWithInstance(anyCtx, p3.P2C2) - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p3.P2.ID) + lifecycleService.EXPECT().Start(anyCtx, p3.P2.ID) err := service.Init(context.Background()) is.NoErr(err) @@ -407,7 +408,7 @@ func TestService_Init_PipelineProvisionedFromAPI(t *testing.T) { ProvisionedBy: pipeline.ProvisionTypeAPI, } - service, pipelineService, _, _, _ := newTestService(ctrl, logger) + service, pipelineService, _, _, _, _ := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // pipeline provisioned by API @@ -425,7 +426,7 @@ func TestService_Init_PipelineProvisionedFromAPI_Error(t *testing.T) { ctrl := gomock.NewController(t) otherErr := cerrors.New("GetError") - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, _, lifecycleService := newTestService(ctrl, logger) service.pipelinesPath = "./test/pipelines1" // error from calling Get @@ -449,7 +450,7 @@ func TestService_Init_PipelineProvisionedFromAPI_Error(t *testing.T) { procService.EXPECT().CreateWithInstance(anyCtx, p1.P1P1) // start pipeline - pipelineService.EXPECT().Start(anyCtx, connService, procService, plugService, p1.P1.ID) + lifecycleService.EXPECT().Start(anyCtx, p1.P1.ID) err := service.Init(context.Background()) is.True(cerrors.Is(err, otherErr)) @@ -460,7 +461,7 @@ func TestService_Delete(t *testing.T) { logger := log.Nop() ctrl := gomock.NewController(t) - service, pipelineService, connService, procService, plugService := newTestService(ctrl, logger) + service, pipelineService, connService, procService, plugService, _ := newTestService(ctrl, logger) // export pipeline pipelineService.EXPECT().Get(anyCtx, oldPipelineInstance.ID).Return(oldPipelineInstance, nil).Times(2) @@ -523,6 +524,7 @@ func TestService_IntegrationTestServices(t *testing.T) { plService := pipeline.NewService(logger, db, b) connService := connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)) procService := processor.NewService(logger, db, procPluginService) + lifecycleService := lifecycle.NewService(logger, b, connService, procService, connPluginService, plService) // create destination file destFile := "./test/dest-file.txt" @@ -533,7 +535,7 @@ func TestService_IntegrationTestServices(t *testing.T) { is.NoErr(err) }) - service := NewService(db, logger, plService, connService, procService, connPluginService, "./test/pipelines4-integration-test") + service := NewService(db, logger, plService, connService, procService, connPluginService, lifecycleService, "./test/pipelines4-integration-test") err = service.Init(context.Background()) is.NoErr(err) diff --git a/pkg/provisioning/test/dest-file.txt b/pkg/provisioning/test/dest-file.txt new file mode 100644 index 000000000..0c48018cc --- /dev/null +++ b/pkg/provisioning/test/dest-file.txt @@ -0,0 +1 @@ +{"position":"MjU=","operation":"create","metadata":{"conduit.source.connector.id":"pipeline1:con1","file.path":"./test/source-file.txt"},"key":"MQ==","payload":{"before":null,"after":"anVzdCBzb21lIHRlc3RpbmcgZGF0YSA6RA=="}} From 9952975800ee38ca099b033c59a0104461fbb109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 16:19:47 +0200 Subject: [PATCH 18/41] fix lint --- pkg/lifecycle/service.go | 1 - pkg/lifecycle/service_test.go | 6 +++--- pkg/orchestrator/orchestrator_test.go | 17 +++++++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index f0d5a9f86..97b667fe5 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -686,7 +686,6 @@ func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { switch err { case tomb.ErrStillAlive: // not an actual error, the pipeline stopped gracefully - err = nil if isGracefulShutdown.Load() { // it was triggered by a graceful shutdown of Conduit err = s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, pipeline.StatusSystemStopped, "") diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 1b2d9c6e7..85201b408 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -799,17 +799,17 @@ type testPluginFetcher map[string]connectorPlugin.Dispenser type testPipelineService map[string]*pipeline.Instance func (t testPipelineService) Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) { - //TODO implement me + // TODO implement me panic("implement me") } func (t testPipelineService) GetInstances() map[string]*pipeline.Instance { - //TODO implement me + // TODO implement me panic("implement me") } func (t testPipelineService) UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status, errMsg string) error { - //TODO implement me + // TODO implement me panic("implement me") } diff --git a/pkg/orchestrator/orchestrator_test.go b/pkg/orchestrator/orchestrator_test.go index cb5b6df83..5c8e9310e 100644 --- a/pkg/orchestrator/orchestrator_test.go +++ b/pkg/orchestrator/orchestrator_test.go @@ -27,6 +27,7 @@ import ( "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/ctxutil" "github.com/conduitio/conduit/pkg/foundation/log" + "github.com/conduitio/conduit/pkg/lifecycle" "github.com/conduitio/conduit/pkg/orchestrator/mock" "github.com/conduitio/conduit/pkg/pipeline" conn_plugin "github.com/conduitio/conduit/pkg/plugin/connector" @@ -92,16 +93,20 @@ func TestPipelineSimple(t *testing.T) { nil, ) - lifecycleService := mock.NewLifecycleService(gomock.NewController(t)) - b := &backoff.Backoff{} + connectorService := connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)) + processorService := processor.NewService(logger, db, procPluginService) + pipelineService := pipeline.NewService(logger, db, b) + + lifecycleService := lifecycle.NewService(logger, b, connectorService, processorService, connPluginService, pipelineService) + orc := NewOrchestrator( db, logger, - pipeline.NewService(logger, db, b), - connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)), - processor.NewService(logger, db, procPluginService), + pipelineService, + connectorService, + processorService, connPluginService, procPluginService, lifecycleService, @@ -165,7 +170,7 @@ func TestPipelineSimple(t *testing.T) { err = orc.Pipelines.Stop(ctx, pl.ID, false) is.NoErr(err) t.Log("waiting") - err = lifecycleService.Wait() + err = lifecycleService.WaitPipeline(pl.ID) is.NoErr(err) t.Log("successfully stopped pipeline") From 49b90f2e8a43feea9c09d61b78699ea4f80eedd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 17:45:25 +0200 Subject: [PATCH 19/41] remove GetInstances --- pkg/lifecycle/service.go | 6 +++--- pkg/lifecycle/service_test.go | 2 +- pkg/pipeline/service.go | 5 ----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 97b667fe5..c9fe8c423 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -109,7 +109,7 @@ type ConnectorPluginService interface { // PipelineService can fetch a pipeline instance. type PipelineService interface { Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) - GetInstances() map[string]*pipeline.Instance + List(ctx context.Context) map[string]*pipeline.Instance UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status, errMsg string) error } @@ -128,7 +128,7 @@ func (s *Service) Run( s.logger.Debug(ctx).Msg("initializing pipelines statuses") // run pipelines that are in the StatusSystemStopped state - instances := s.pipelines.GetInstances() + instances := s.pipelines.List(ctx) for _, instance := range instances { if instance.GetStatus() == pipeline.StatusSystemStopped { err := s.Start(ctx, instance.ID) @@ -246,7 +246,7 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error // StopAll will ask all the pipelines to stop gracefully // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { - instances := s.pipelines.GetInstances() + instances := s.pipelines.List(ctx) for _, pl := range instances { rp, ok := s.runningPipelines[pl.ID] if !ok || (pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering) { diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 85201b408..0e4406ee8 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -803,7 +803,7 @@ func (t testPipelineService) Get(ctx context.Context, pipelineID string) (*pipel panic("implement me") } -func (t testPipelineService) GetInstances() map[string]*pipeline.Instance { +func (t testPipelineService) List(ctx context.Context) map[string]*pipeline.Instance { // TODO implement me panic("implement me") } diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index fd128642e..470684d26 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -337,11 +337,6 @@ func (s *Service) validatePipeline(cfg Config, id string) error { return cerrors.Join(errs...) } -// GetInstances returns all pipeline instances. -func (s *Service) GetInstances() map[string]*Instance { - return s.instances -} - // UpdateStatus updates the status of a pipeline by the ID. func (s *Service) UpdateStatus(ctx context.Context, id string, status Status, errMsg string) error { pipeline, err := s.Get(ctx, id) From 58dd785a521b51f8a6e0d278a6248df8ab36e3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 17:50:15 +0200 Subject: [PATCH 20/41] delete key instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Lovro Mažgon --- pkg/lifecycle/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index c9fe8c423..683bcf1e8 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -218,7 +218,7 @@ func (s *Service) stopGraceful(ctx context.Context, rp *runnablePipeline, reason } if len(errs) == 0 { - s.runningPipelines[rp.pipeline.ID] = nil + delete(s.runningPipelines, rp.pipeline.ID) return nil } @@ -239,7 +239,7 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error } } - s.runningPipelines[rp.pipeline.ID] = nil + delete(s.runningPipelines, rp.pipeline.ID) return nil } From ae85b9d562fa7c73f907f3ef7b1719221dda56ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 20:53:03 +0200 Subject: [PATCH 21/41] ignore test files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 63ebfdf26..bbae96b65 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,6 @@ escape_analysis.txt # Compiled test wasm processors pkg/plugin/processor/standalone/test/wasm_processors/*/processor.wasm + +# Test data +**/test/*.txt From 542c895bfaec0561736549e0c16686828117b5f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Wed, 11 Sep 2024 20:58:33 +0200 Subject: [PATCH 22/41] delete test files --- pkg/provisioning/test/dest-file.txt | 1 - pkg/provisioning/test/source-file.txt | 1 - 2 files changed, 2 deletions(-) delete mode 100644 pkg/provisioning/test/dest-file.txt delete mode 100644 pkg/provisioning/test/source-file.txt diff --git a/pkg/provisioning/test/dest-file.txt b/pkg/provisioning/test/dest-file.txt deleted file mode 100644 index 0c48018cc..000000000 --- a/pkg/provisioning/test/dest-file.txt +++ /dev/null @@ -1 +0,0 @@ -{"position":"MjU=","operation":"create","metadata":{"conduit.source.connector.id":"pipeline1:con1","file.path":"./test/source-file.txt"},"key":"MQ==","payload":{"before":null,"after":"anVzdCBzb21lIHRlc3RpbmcgZGF0YSA6RA=="}} diff --git a/pkg/provisioning/test/source-file.txt b/pkg/provisioning/test/source-file.txt deleted file mode 100644 index cc17f9ca3..000000000 --- a/pkg/provisioning/test/source-file.txt +++ /dev/null @@ -1 +0,0 @@ -just some testing data :D \ No newline at end of file From 0c932fecbe996c60135b822ac0480ca9f14011fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Thu, 12 Sep 2024 13:46:12 +0200 Subject: [PATCH 23/41] ensures runningPipeline is locked --- pkg/lifecycle/service.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index fffefdd82..71cd137f9 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -62,6 +62,7 @@ type Service struct { handlers []FailureHandler runningPipelines map[string]*runnablePipeline + m sync.Mutex } // NewService initializes and returns a pipeline Service. @@ -171,6 +172,9 @@ func (s *Service) Start( return cerrors.Errorf("failed to run pipeline %s: %w", pl.ID, err) } s.logger.Info(ctx).Str(log.PipelineIDField, pl.ID).Msg("pipeline started") + + s.m.Lock() + defer s.m.Unlock() s.runningPipelines[pl.ID] = rp return nil @@ -218,6 +222,8 @@ func (s *Service) stopGraceful(ctx context.Context, rp *runnablePipeline, reason } if len(errs) == 0 { + s.m.Lock() + defer s.m.Unlock() delete(s.runningPipelines, rp.pipeline.ID) return nil } @@ -239,6 +245,8 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error } } + s.m.Lock() + defer s.m.Unlock() delete(s.runningPipelines, rp.pipeline.ID) return nil } @@ -307,6 +315,8 @@ func (s *Service) WaitPipeline(id string) error { if s.runningPipelines[id].t == nil { return nil } + s.m.Lock() + defer s.m.Unlock() return s.runningPipelines[id].t.Wait() } From 17140fb335c64d2632215fd92e5488243c13281b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Thu, 12 Sep 2024 16:15:01 +0200 Subject: [PATCH 24/41] fix not initialized map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Lovro Mažgon --- pkg/lifecycle/service.go | 19 +++++++------------ pkg/lifecycle/service_test.go | 33 ++++++++++++++++----------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 71cd137f9..03ec12f62 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -81,6 +81,7 @@ func NewService( processors: processors, connectorPlugins: connectorPlugins, pipelines: pipelines, + runningPipelines: make(map[string]*runnablePipeline), } } @@ -174,8 +175,8 @@ func (s *Service) Start( s.logger.Info(ctx).Str(log.PipelineIDField, pl.ID).Msg("pipeline started") s.m.Lock() - defer s.m.Unlock() s.runningPipelines[pl.ID] = rp + s.m.Unlock() return nil } @@ -221,13 +222,6 @@ func (s *Service) stopGraceful(ctx context.Context, rp *runnablePipeline, reason } } - if len(errs) == 0 { - s.m.Lock() - defer s.m.Unlock() - delete(s.runningPipelines, rp.pipeline.ID) - return nil - } - return cerrors.Join(errs...) } @@ -244,10 +238,6 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error node.ForceStop(ctx) } } - - s.m.Lock() - defer s.m.Unlock() - delete(s.runningPipelines, rp.pipeline.ID) return nil } @@ -724,6 +714,11 @@ func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { Str(log.PipelineIDField, rp.pipeline.ID). Msg("pipeline stopped") + // confirmed that all nodes stopped, we can now remove the pipeline from the running pipelines + s.m.Lock() + delete(s.runningPipelines, rp.pipeline.ID) + s.m.Unlock() + s.notify(rp.pipeline.ID, err) // It's important to update the metrics before we handle the error from s.Store.Set() (if any), // since the source of the truth is the actual pipeline (stored in memory). diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 0e4406ee8..4ed097c53 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -231,11 +231,6 @@ func TestService_buildRunnablePipeline_NoDestinationNode(t *testing.T) { is.Equal(got, nil) } -// TODO: Fix -// destination.go:117: err: context deadline exceeded // run didn't finish -// destination.go:117: err: context deadline exceeded // run didn't finish -// source.go:145: 0 != 10 // number of expected acks don't match -// source.go:116: not true: done.Load() // run didn't finish func TestServiceLifecycle_PipelineSuccess(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) @@ -275,7 +270,7 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, testPipelineService{}) + }, ps) // start the pipeline now that everything is set up err = ls.Start( @@ -796,11 +791,23 @@ func (tpf testProcessorFetcher) Get(_ context.Context, id string) (*processor.In // testPluginFetcher fulfills the PluginFetcher interface. type testPluginFetcher map[string]connectorPlugin.Dispenser +func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string, _ string) (connectorPlugin.Dispenser, error) { + plug, ok := tpf[name] + if !ok { + return nil, plugin.ErrPluginNotFound + } + return plug, nil +} + +// testPipelineService fulfills the PipelineService interface. type testPipelineService map[string]*pipeline.Instance -func (t testPipelineService) Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) { - // TODO implement me - panic("implement me") +func (t testPipelineService) Get(_ context.Context, pipelineID string) (*pipeline.Instance, error) { + pi, ok := t[pipelineID] + if !ok { + return nil, processor.ErrInstanceNotFound + } + return pi, nil } func (t testPipelineService) List(ctx context.Context) map[string]*pipeline.Instance { @@ -812,11 +819,3 @@ func (t testPipelineService) UpdateStatus(ctx context.Context, pipelineID string // TODO implement me panic("implement me") } - -func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string, _ string) (connectorPlugin.Dispenser, error) { - plug, ok := tpf[name] - if !ok { - return nil, plugin.ErrPluginNotFound - } - return plug, nil -} From 1de69c46275a127c3738ea534341dc77b5c224fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 09:42:34 +0200 Subject: [PATCH 25/41] fix tests --- pkg/lifecycle/service.go | 14 ++--- pkg/lifecycle/service_test.go | 107 ++++++++++++++++++---------------- 2 files changed, 65 insertions(+), 56 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 03ec12f62..ddd6048d5 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -65,7 +65,7 @@ type Service struct { m sync.Mutex } -// NewService initializes and returns a pipeline Service. +// NewService initializes and returns a lifecycle.Service. func NewService( logger log.CtxLogger, backoffCfg *backoff.Backoff, @@ -75,7 +75,7 @@ func NewService( pipelines PipelineService, ) *Service { return &Service{ - logger: logger.WithComponent("pipeline.Service"), + logger: logger.WithComponent("lifecycle.Service"), backoffCfg: backoffCfg, connectors: connectors, processors: processors, @@ -91,7 +91,7 @@ type runnablePipeline struct { t *tomb.Tomb } -// ConnectorService can fetch a connector instance. +// ConnectorService can fetch and create a connector instance. type ConnectorService interface { Get(ctx context.Context, id string) (*connector.Instance, error) Create(ctx context.Context, id string, t connector.Type, plugin string, pipelineID string, cfg connector.Config, p connector.ProvisionType) (*connector.Instance, error) @@ -108,14 +108,14 @@ type ConnectorPluginService interface { NewDispenser(logger log.CtxLogger, name string, connectorID string) (connectorPlugin.Dispenser, error) } -// PipelineService can fetch a pipeline instance. +// PipelineService can fetch, list and update the status of a pipeline instance. type PipelineService interface { Get(ctx context.Context, pipelineID string) (*pipeline.Instance, error) List(ctx context.Context) map[string]*pipeline.Instance UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status, errMsg string) error } -// OnFailure registers a handler for a pipeline.FailureEvent. +// OnFailure registers a handler for a lifecycle.FailureEvent. // Only errors which happen after a pipeline has been started // are being sent. func (s *Service) OnFailure(handler FailureHandler) { @@ -301,12 +301,12 @@ func (s *Service) waitInternal() error { return cerrors.Join(errs...) } +// WaitPipeline blocks until the pipeline with the given ID is stopped. +// This is only used in tests to ensure that a pipeline is stopped before func (s *Service) WaitPipeline(id string) error { if s.runningPipelines[id].t == nil { return nil } - s.m.Lock() - defer s.m.Unlock() return s.runningPipelines[id].t.Wait() } diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 4ed097c53..34cc8c249 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -78,8 +78,8 @@ func TestServiceLifecycle_buildRunnablePipeline(t *testing.T) { destination.ID: destination, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: pmock.NewDispenser(ctrl), destination.Plugin: pmock.NewDispenser(ctrl), dlq.Plugin: pmock.NewDispenser(ctrl), @@ -165,8 +165,8 @@ func TestService_buildRunnablePipeline_NoSourceNode(t *testing.T) { ls := NewService(logger, b, testConnectorService{ destination.ID: destination, testDLQID: dlq, - }, testProcessorFetcher{}, - testPluginFetcher{ + }, testProcessorService{}, + testConnectorPluginService{ destination.Plugin: pmock.NewDispenser(ctrl), dlq.Plugin: pmock.NewDispenser(ctrl), }, testPipelineService{}) @@ -200,8 +200,8 @@ func TestService_buildRunnablePipeline_NoDestinationNode(t *testing.T) { source.ID: source, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: pmock.NewDispenser(ctrl), dlq.Plugin: pmock.NewDispenser(ctrl), }, testPipelineService{}) @@ -265,8 +265,8 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { destination.ID: destination, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, @@ -288,6 +288,7 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { // stop pipeline before ending test err = ls.Stop(ctx, pl.ID, false) is.NoErr(err) + is.NoErr(ls.WaitPipeline(pl.ID)) } @@ -327,12 +328,12 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { destination.ID: destination, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, testPipelineService{}) + }, ps) events := make(chan FailureEvent, 1) ls.OnFailure(func(e FailureEvent) { @@ -369,10 +370,6 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { is.True(cerrors.Is(event.Error, wantErr)) } -// TODO: Fix -// destination.go:117: err: context deadline exceeded // run didn't finish -// destination.go:117: err: context deadline exceeded // run didn't finish -// source.go:116: not true: done.Load() // run didn't finish func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { type testCase struct { name string @@ -419,12 +416,12 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { destination.ID: destination, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, - }, testPipelineService{}) + }, ps) // start the pipeline now that everything is set up err = ls.Start( @@ -461,14 +458,23 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { want: pipeline.StatusSystemStopped, }, { - name: "system stop (terrible err)", + name: "system stop (fatal err)", stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { - ls.StopAll(ctx, cerrors.New("terrible err")) + ls.StopAll(ctx, cerrors.FatalError(cerrors.New("terrible err"))) }, wantSourceStop: true, want: pipeline.StatusDegraded, wantErr: cerrors.New("terrible err"), }, + { + name: "connection error", + stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { + ls.StopAll(ctx, cerrors.New("lost connection to database")) + }, + wantSourceStop: true, + want: pipeline.StatusRecovering, + wantErr: cerrors.New("lost connection to database"), + }, { name: "user stop (graceful)", stopFn: func(ctx context.Context, is *is.I, ls *Service, pipelineID string) { @@ -487,7 +493,6 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { } } -// TODO: Fix errors func TestServiceLifecycle_PipelineStop(t *testing.T) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) @@ -523,8 +528,8 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { destination.ID: destination, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, @@ -549,8 +554,7 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { is.Equal("", pl.Error) } -// TODO: Fix errors -func TestService_Run_Rerun(t *testing.T) { +func TestServiceLifecycle_Run_Rerun(t *testing.T) { runTest := func(t *testing.T, status pipeline.Status, expected pipeline.Status) { is := is.New(t) ctx, killAll := context.WithCancel(context.Background()) @@ -607,8 +611,8 @@ func TestService_Run_Rerun(t *testing.T) { destination.ID: destination, testDLQID: dlq, }, - testProcessorFetcher{}, - testPluginFetcher{ + testProcessorService{}, + testConnectorPluginService{ source.Plugin: sourceDispenser, destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, @@ -761,38 +765,38 @@ func dummyDestination(persister *connector.Persister) *connector.Instance { // testConnectorService fulfills the ConnectorService interface. type testConnectorService map[string]*connector.Instance -func (tcf testConnectorService) Get(_ context.Context, id string) (*connector.Instance, error) { - conn, ok := tcf[id] +func (s testConnectorService) Get(_ context.Context, id string) (*connector.Instance, error) { + conn, ok := s[id] if !ok { return nil, connector.ErrInstanceNotFound } return conn, nil } -func (tcf testConnectorService) Create(context.Context, string, connector.Type, string, string, connector.Config, connector.ProvisionType) (*connector.Instance, error) { - return tcf[testDLQID], nil +func (s testConnectorService) Create(context.Context, string, connector.Type, string, string, connector.Config, connector.ProvisionType) (*connector.Instance, error) { + return s[testDLQID], nil } -// testProcessorFetcher fulfills the ProcessorService interface. -type testProcessorFetcher map[string]*processor.Instance +// testProcessorService fulfills the ProcessorService interface. +type testProcessorService map[string]*processor.Instance -func (tpf testProcessorFetcher) MakeRunnableProcessor(context.Context, *processor.Instance) (*processor.RunnableProcessor, error) { +func (s testProcessorService) MakeRunnableProcessor(context.Context, *processor.Instance) (*processor.RunnableProcessor, error) { return nil, cerrors.New("not implemented") } -func (tpf testProcessorFetcher) Get(_ context.Context, id string) (*processor.Instance, error) { - proc, ok := tpf[id] +func (s testProcessorService) Get(_ context.Context, id string) (*processor.Instance, error) { + proc, ok := s[id] if !ok { return nil, processor.ErrInstanceNotFound } return proc, nil } -// testPluginFetcher fulfills the PluginFetcher interface. -type testPluginFetcher map[string]connectorPlugin.Dispenser +// testConnectorPluginService fulfills the ConnectorPluginService interface. +type testConnectorPluginService map[string]connectorPlugin.Dispenser -func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string, _ string) (connectorPlugin.Dispenser, error) { - plug, ok := tpf[name] +func (s testConnectorPluginService) NewDispenser(_ log.CtxLogger, name string, _ string) (connectorPlugin.Dispenser, error) { + plug, ok := s[name] if !ok { return nil, plugin.ErrPluginNotFound } @@ -802,20 +806,25 @@ func (tpf testPluginFetcher) NewDispenser(_ log.CtxLogger, name string, _ string // testPipelineService fulfills the PipelineService interface. type testPipelineService map[string]*pipeline.Instance -func (t testPipelineService) Get(_ context.Context, pipelineID string) (*pipeline.Instance, error) { - pi, ok := t[pipelineID] +func (s testPipelineService) Get(_ context.Context, pipelineID string) (*pipeline.Instance, error) { + p, ok := s[pipelineID] if !ok { return nil, processor.ErrInstanceNotFound } - return pi, nil + return p, nil } -func (t testPipelineService) List(ctx context.Context) map[string]*pipeline.Instance { - // TODO implement me - panic("implement me") +func (s testPipelineService) List(_ context.Context) map[string]*pipeline.Instance { + instances := make(map[string]*pipeline.Instance) + return instances } -func (t testPipelineService) UpdateStatus(ctx context.Context, pipelineID string, status pipeline.Status, errMsg string) error { - // TODO implement me - panic("implement me") +func (s testPipelineService) UpdateStatus(_ context.Context, pipelineID string, status pipeline.Status, errMsg string) error { + p, ok := s[pipelineID] + if !ok { + return processor.ErrInstanceNotFound + } + p.SetStatus(status) + p.Error = errMsg + return nil } From 7d9210c92c0ff7fe0eb557bb1effb473bb37d8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 10:16:35 +0200 Subject: [PATCH 26/41] add back the needed test file --- .gitignore | 3 +++ pkg/provisioning/test/source-file.txt | 1 + 2 files changed, 4 insertions(+) create mode 100644 pkg/provisioning/test/source-file.txt diff --git a/.gitignore b/.gitignore index bbae96b65..eb00ee4a3 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,6 @@ pkg/plugin/processor/standalone/test/wasm_processors/*/processor.wasm # Test data **/test/*.txt + +# this one is needed for integration tests +!pkg/provisioning/test/source-file.txt diff --git a/pkg/provisioning/test/source-file.txt b/pkg/provisioning/test/source-file.txt new file mode 100644 index 000000000..cc17f9ca3 --- /dev/null +++ b/pkg/provisioning/test/source-file.txt @@ -0,0 +1 @@ +just some testing data :D \ No newline at end of file From 07b8018ec04a3faf4fc75b1a17ee7400ea1fa0d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 10:32:31 +0200 Subject: [PATCH 27/41] update comment --- pkg/lifecycle/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index ddd6048d5..f4aaca7ef 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -103,7 +103,7 @@ type ProcessorService interface { MakeRunnableProcessor(ctx context.Context, i *processor.Instance) (*processor.RunnableProcessor, error) } -// ConnectorPluginService can fetch a plugin. +// ConnectorPluginService can create a connector plugin dispenser. type ConnectorPluginService interface { NewDispenser(logger log.CtxLogger, name string, connectorID string) (connectorPlugin.Dispenser, error) } From 435709f54c68133dec4869429b02c520c4d2f4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 10:46:00 +0200 Subject: [PATCH 28/41] add comment --- pkg/orchestrator/orchestrator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index 24513076b..a5477d9ff 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -126,6 +126,7 @@ type ProcessorPluginService interface { } type LifecycleService interface { + // Start initiates a start of the given pipeline. Start(ctx context.Context, pipelineID string) error // Stop initiates a stop of the given pipeline. The method does not wait for // the pipeline (and its nodes) to actually stop. From 99b215aa6bc77623f7302414f11277ecbb0796e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 10:48:22 +0200 Subject: [PATCH 29/41] locks while retrieving the pipeline --- pkg/lifecycle/service.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index f4aaca7ef..a31623323 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -302,12 +302,15 @@ func (s *Service) waitInternal() error { } // WaitPipeline blocks until the pipeline with the given ID is stopped. -// This is only used in tests to ensure that a pipeline is stopped before func (s *Service) WaitPipeline(id string) error { - if s.runningPipelines[id].t == nil { + s.m.Lock() + p := s.runningPipelines[id] + s.m.Unlock() + + if p.t == nil { return nil } - return s.runningPipelines[id].t.Wait() + return p.t.Wait() } // buildRunnablePipeline will build and connect all nodes configured in the pipeline. From 0521ff8745c16a3cf59f41ba30d4e9f6a752a0cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 11:06:12 +0200 Subject: [PATCH 30/41] update comment --- pkg/lifecycle/service.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index a31623323..ef3d6b26d 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package lifecycle wires up everything under the hood of a Conduit instance -// including metrics, telemetry, logging, and server construction. -// It should only ever interact with the Orchestrator, never individual -// services. All of that responsibility should be left to the Orchestrator. +// Package lifecycle contains the logic to manage the lifecycle of pipelines. +// It is responsible for starting, stopping and managing pipelines. package lifecycle import ( From 4763ac5e509f362d3ea7254f5a383a6eea4d6818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 11:12:49 +0200 Subject: [PATCH 31/41] unsure about the need for this --- pkg/lifecycle/service.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index ef3d6b26d..729c2a80a 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -611,7 +611,6 @@ func (s *Service) buildDestinationNodes( } func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { - // TODO: Handle the tomb outside and after maybe checking the status if rp.t != nil && rp.t.Alive() { return pipeline.ErrPipelineRunning } From ea299092b77e372d70da9aa6f394c800bd30df86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 12:37:16 +0200 Subject: [PATCH 32/41] remove backoff from pipeline service --- pkg/conduit/runtime.go | 2 +- pkg/lifecycle/service_test.go | 12 +++---- pkg/orchestrator/orchestrator_test.go | 2 +- pkg/pipeline/service.go | 5 +-- pkg/pipeline/service_test.go | 45 +++++++++------------------ pkg/provisioning/service_test.go | 2 +- 6 files changed, 25 insertions(+), 43 deletions(-) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index e93700b80..ca018946d 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -212,7 +212,7 @@ func createServices(r *Runtime) error { Jitter: true, } - plService := pipeline.NewService(r.logger, r.DB, backoffCfg) + plService := pipeline.NewService(r.logger, r.DB) connService := connector.NewService(r.logger, r.DB, r.connectorPersister) procService := processor.NewService(r.logger, r.DB, procPluginService) lifecycleService := lifecycle.NewService(r.logger, backoffCfg, connService, procService, connPluginService, plService) diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 34cc8c249..71bc66d14 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -241,7 +241,7 @@ func TestServiceLifecycle_PipelineSuccess(t *testing.T) { defer persister.Wait() b := &backoff.Backoff{} - ps := pipeline.NewService(logger, db, b) + ps := pipeline.NewService(logger, db) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) @@ -303,7 +303,7 @@ func TestServiceLifecycle_PipelineError(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := pipeline.NewService(logger, db, b) + ps := pipeline.NewService(logger, db) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) @@ -390,7 +390,7 @@ func TestServiceLifecycle_StopAll_Recovering(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := pipeline.NewService(logger, db, b) + ps := pipeline.NewService(logger, db) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) @@ -502,7 +502,7 @@ func TestServiceLifecycle_PipelineStop(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := pipeline.NewService(logger, db, b) + ps := pipeline.NewService(logger, db) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) @@ -565,7 +565,7 @@ func TestServiceLifecycle_Run_Rerun(t *testing.T) { persister := connector.NewPersister(logger, db, time.Second, 3) b := &backoff.Backoff{} - ps := pipeline.NewService(logger, db, b) + ps := pipeline.NewService(logger, db) // create a host pipeline pl, err := ps.Create(ctx, uuid.NewString(), pipeline.Config{Name: "test pipeline"}, pipeline.ProvisionTypeAPI) @@ -602,7 +602,7 @@ func TestServiceLifecycle_Run_Rerun(t *testing.T) { is.NoErr(err) // create a new pipeline service and initialize it - ps = pipeline.NewService(logger, db, b) + ps = pipeline.NewService(logger, db) err = ps.Init(ctx) is.NoErr(err) diff --git a/pkg/orchestrator/orchestrator_test.go b/pkg/orchestrator/orchestrator_test.go index 5c8e9310e..bcb125b27 100644 --- a/pkg/orchestrator/orchestrator_test.go +++ b/pkg/orchestrator/orchestrator_test.go @@ -97,7 +97,7 @@ func TestPipelineSimple(t *testing.T) { connectorService := connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)) processorService := processor.NewService(logger, db, procPluginService) - pipelineService := pipeline.NewService(logger, db, b) + pipelineService := pipeline.NewService(logger, db) lifecycleService := lifecycle.NewService(logger, b, connectorService, processorService, connPluginService, pipelineService) diff --git a/pkg/pipeline/service.go b/pkg/pipeline/service.go index 292cf164e..a58f57ab9 100644 --- a/pkg/pipeline/service.go +++ b/pkg/pipeline/service.go @@ -24,7 +24,6 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/conduitio/conduit/pkg/foundation/metrics/measure" - "github.com/jpillora/backoff" ) var idRegex = regexp.MustCompile(`^[A-Za-z0-9-_:.]*$`) @@ -43,17 +42,15 @@ type Service struct { instances map[string]*Instance instanceNames map[string]bool - backoffCfg *backoff.Backoff } // NewService initializes and returns a pipeline Service. -func NewService(logger log.CtxLogger, db database.DB, backoffCfg *backoff.Backoff) *Service { +func NewService(logger log.CtxLogger, db database.DB) *Service { return &Service{ logger: logger.WithComponent("pipeline.Service"), store: NewStore(db), instances: make(map[string]*Instance), instanceNames: make(map[string]bool), - backoffCfg: backoffCfg, } } diff --git a/pkg/pipeline/service_test.go b/pkg/pipeline/service_test.go index 48533df6e..a5a80502a 100644 --- a/pkg/pipeline/service_test.go +++ b/pkg/pipeline/service_test.go @@ -25,7 +25,6 @@ import ( "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" "github.com/google/uuid" - "github.com/jpillora/backoff" "github.com/matryer/is" "go.uber.org/mock/gomock" ) @@ -35,16 +34,15 @@ func TestService_Init_Simple(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) _, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) want := service.List(ctx) // create a new pipeline service and initialize it - service = NewService(logger, db, b) + service = NewService(logger, db) err = service.Init(ctx) is.NoErr(err) @@ -63,7 +61,6 @@ func TestService_Check(t *testing.T) { ctx := context.Background() logger := log.Nop() db := mock.NewDB(gomock.NewController(t)) - b := &backoff.Backoff{} testCases := []struct { name string @@ -83,7 +80,7 @@ func TestService_Check(t *testing.T) { t.Run(tc.name, func(t *testing.T) { is := is.New(t) db.EXPECT().Ping(gomock.Any()).Return(tc.wantErr) - service := NewService(logger, db, b) + service := NewService(logger, db) gotErr := service.Check(ctx) is.Equal(tc.wantErr, gotErr) @@ -95,9 +92,8 @@ func TestService_CreateSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) testCases := []struct { id string @@ -155,9 +151,8 @@ func TestService_Create_ValidateSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) testCases := []struct { name string @@ -198,9 +193,8 @@ func TestService_Create_ValidateError(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) testCases := []struct { name string @@ -268,9 +262,8 @@ func TestService_Create_PipelineNameExists(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) conf := Config{Name: "test-pipeline"} got, err := service.Create(ctx, uuid.NewString(), conf, ProvisionTypeAPI) @@ -286,9 +279,8 @@ func TestService_CreateEmptyName(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) got, err := service.Create(ctx, uuid.NewString(), Config{Name: ""}, ProvisionTypeAPI) is.True(err != nil) is.Equal(got, nil) @@ -299,9 +291,8 @@ func TestService_GetSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) want, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) @@ -315,9 +306,8 @@ func TestService_GetInstanceNotFound(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) // get pipeline instance that does not exist got, err := service.Get(ctx, uuid.NewString()) @@ -331,9 +321,8 @@ func TestService_DeleteSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) instance, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) @@ -350,9 +339,8 @@ func TestService_List(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) want := make(map[string]*Instance) for i := 0; i < 10; i++ { @@ -370,9 +358,8 @@ func TestService_UpdateSuccess(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) instance, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) @@ -391,9 +378,8 @@ func TestService_Update_PipelineNameExists(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) _, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) instance2, err2 := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline2"}, ProvisionTypeAPI) @@ -414,9 +400,8 @@ func TestService_UpdateInvalidConfig(t *testing.T) { ctx := context.Background() logger := log.Nop() db := &inmemory.DB{} - b := &backoff.Backoff{} - service := NewService(logger, db, b) + service := NewService(logger, db) instance, err := service.Create(ctx, uuid.NewString(), Config{Name: "test-pipeline"}, ProvisionTypeAPI) is.NoErr(err) diff --git a/pkg/provisioning/service_test.go b/pkg/provisioning/service_test.go index 544b2628e..3701d1fa7 100644 --- a/pkg/provisioning/service_test.go +++ b/pkg/provisioning/service_test.go @@ -521,7 +521,7 @@ func TestService_IntegrationTestServices(t *testing.T) { Max: time.Second, // 8 tries } - plService := pipeline.NewService(logger, db, b) + plService := pipeline.NewService(logger, db) connService := connector.NewService(logger, db, connector.NewPersister(logger, db, time.Second, 3)) procService := processor.NewService(logger, db, procPluginService) lifecycleService := lifecycle.NewService(logger, b, connService, procService, connPluginService, plService) From 97d24a4c282fe1fc7e657b91341d4ca9b4788be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 12:55:47 +0200 Subject: [PATCH 33/41] update method name on lifecycle service - changes run to init --- pkg/conduit/runtime.go | 2 +- pkg/lifecycle/service.go | 5 ++--- pkg/lifecycle/service_test.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/conduit/runtime.go b/pkg/conduit/runtime.go index ca018946d..c6df8fd15 100644 --- a/pkg/conduit/runtime.go +++ b/pkg/conduit/runtime.go @@ -797,7 +797,7 @@ func (r *Runtime) initServices(ctx context.Context, t *tomb.Tomb) error { } } - err = r.lifecycleService.Run(ctx) + err = r.lifecycleService.Init(ctx) if err != nil { cerrors.ForEach(err, func(err error) { r.logger.Err(ctx, err).Msg("pipeline failed to be started") diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 729c2a80a..48d81e4b3 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -120,14 +120,13 @@ func (s *Service) OnFailure(handler FailureHandler) { s.handlers = append(s.handlers, handler) } -// Run runs pipelines that had the running state in store. -func (s *Service) Run( +// Init start all pipelines that have the StatusSystemStopped. +func (s *Service) Init( ctx context.Context, ) error { var errs []error s.logger.Debug(ctx).Msg("initializing pipelines statuses") - // run pipelines that are in the StatusSystemStopped state instances := s.pipelines.List(ctx) for _, instance := range instances { if instance.GetStatus() == pipeline.StatusSystemStopped { diff --git a/pkg/lifecycle/service_test.go b/pkg/lifecycle/service_test.go index 71bc66d14..d3bb0a7f0 100644 --- a/pkg/lifecycle/service_test.go +++ b/pkg/lifecycle/service_test.go @@ -617,7 +617,7 @@ func TestServiceLifecycle_Run_Rerun(t *testing.T) { destination.Plugin: destDispenser, dlq.Plugin: dlqDispenser, }, ps) - err = ls.Run(ctx) + err = ls.Init(ctx) is.NoErr(err) // give pipeline a chance to start if needed From 7d4da27cbbb91bd8b0f116ea858bb45432194abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 13:05:48 +0200 Subject: [PATCH 34/41] lock running pipelines and optimize stopAll --- pkg/lifecycle/service.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 48d81e4b3..0474b623d 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -183,7 +183,10 @@ func (s *Service) Start( // instead the context for all nodes will be canceled which causes them to stop // running as soon as possible. func (s *Service) Stop(ctx context.Context, pipelineID string, force bool) error { + s.m.Lock() rp, ok := s.runningPipelines[pipelineID] + s.m.Unlock() + if !ok { return cerrors.Errorf("pipeline %s is not running: %w", pipelineID, pipeline.ErrPipelineNotRunning) } @@ -238,20 +241,26 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error return nil } -// StopAll will ask all the pipelines to stop gracefully +// StopAll will ask all the running pipelines to stop gracefully // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { - instances := s.pipelines.List(ctx) - for _, pl := range instances { - rp, ok := s.runningPipelines[pl.ID] - if !ok || (pl.GetStatus() != pipeline.StatusRunning && pl.GetStatus() != pipeline.StatusRecovering) { + s.m.Lock() + instances := s.runningPipelines + s.m.Unlock() + + for _, rp := range instances { + p := rp.pipeline + if p.GetStatus() != pipeline.StatusRunning && p.GetStatus() != pipeline.StatusRecovering { + s.m.Lock() + delete(s.runningPipelines, p.ID) + s.m.Unlock() continue } err := s.stopGraceful(ctx, rp, reason) if err != nil { s.logger.Warn(ctx). Err(err). - Str(log.PipelineIDField, pl.ID). + Str(log.PipelineIDField, p.ID). Msg("could not stop pipeline") } } @@ -286,7 +295,12 @@ func (s *Service) Wait(timeout time.Duration) error { // the pipelines failed to stop gracefully. func (s *Service) waitInternal() error { var errs []error - for _, rp := range s.runningPipelines { + + s.m.Lock() + pipelines := s.runningPipelines + s.m.Unlock() + + for _, rp := range pipelines { if rp.t == nil { continue } From fc3f3b60585845ed25440a6d6a4ead6ffcba7eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 13:12:40 +0200 Subject: [PATCH 35/41] remove redundant call to update status --- pkg/lifecycle/service.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 0474b623d..cdad6f262 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -733,9 +733,7 @@ func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { s.m.Unlock() s.notify(rp.pipeline.ID, err) - // It's important to update the metrics before we handle the error from s.Store.Set() (if any), - // since the source of the truth is the actual pipeline (stored in memory). - return s.pipelines.UpdateStatus(ctx, rp.pipeline.ID, rp.pipeline.GetStatus(), "") + return err }) return nil } From 6e60956c644c9be00c3322f8c648e03ea3ad0e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 13:14:40 +0200 Subject: [PATCH 36/41] fix typo --- pkg/lifecycle/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index cdad6f262..fbafc42be 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -120,7 +120,7 @@ func (s *Service) OnFailure(handler FailureHandler) { s.handlers = append(s.handlers, handler) } -// Init start all pipelines that have the StatusSystemStopped. +// Init starts all pipelines that have the StatusSystemStopped. func (s *Service) Init( ctx context.Context, ) error { From 55ba0247ea1f82e9dff1230488aac4f0054958bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 14:57:03 +0200 Subject: [PATCH 37/41] pr feedback --- pkg/lifecycle/service.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index fbafc42be..c8b2d1ef7 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -245,15 +245,11 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { s.m.Lock() - instances := s.runningPipelines - s.m.Unlock() + defer s.m.Unlock() - for _, rp := range instances { + for _, rp := range s.runningPipelines { p := rp.pipeline if p.GetStatus() != pipeline.StatusRunning && p.GetStatus() != pipeline.StatusRecovering { - s.m.Lock() - delete(s.runningPipelines, p.ID) - s.m.Unlock() continue } err := s.stopGraceful(ctx, rp, reason) From 3596fb82428f71d8d990aed272d2407fd2344157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 15:52:07 +0200 Subject: [PATCH 38/41] uses csync Map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uses https://github.com/ConduitIO/conduit-commons/pull/116 Co-Authored-By: Lovro Mažgon --- go.mod | 14 ++++++------ go.sum | 13 ++++++++++++ pkg/lifecycle/service.go | 46 +++++++++++++++------------------------- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index fb9f8fabd..da0950273 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/NYTimes/gziphandler v1.1.1 github.com/bufbuild/buf v1.41.0 - github.com/conduitio/conduit-commons v0.3.0 + github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff github.com/conduitio/conduit-connector-file v0.7.0 github.com/conduitio/conduit-connector-generator v0.7.0 github.com/conduitio/conduit-connector-kafka v0.9.0 @@ -141,8 +141,8 @@ require ( github.com/daixiang0/gci v0.13.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect - github.com/dgraph-io/badger/v4 v4.2.0 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgraph-io/badger/v4 v4.3.0 // indirect + github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/cli v27.2.1+incompatible // indirect @@ -200,7 +200,7 @@ require ( github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect - github.com/hamba/avro/v2 v2.24.0 // indirect + github.com/hamba/avro/v2 v2.25.2 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -213,8 +213,8 @@ require ( github.com/jackc/pglogrepl v0.0.0-20240307033717-828fbfe908e9 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jdx/go-netrc v1.0.0 // indirect github.com/jgautheron/goconst v1.7.1 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect @@ -375,7 +375,7 @@ require ( modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.31.1 // indirect + modernc.org/sqlite v1.33.1 // indirect modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect diff --git a/go.sum b/go.sum index 32e696896..48dccfc6f 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/conduitio/conduit-commons v0.3.0 h1:nxQ++O4dK1p717upkyzbCQu0FLIFyP3OrgHZ9Zxvzvg= github.com/conduitio/conduit-commons v0.3.0/go.mod h1:roxZ88dv+fpbEjjTzkdGwwbmcpunSuiD8he43y0lAoo= +github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff h1:qsOCwSeqFn6bnOqijWnlmUG6osXpCibJRKpI5xq5zR4= +github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff/go.mod h1:zeKc8PpXRzB94MhhpwdRI0Rd+Jn/uNs2tqxLhPA05y4= github.com/conduitio/conduit-connector-file v0.7.0 h1:lUfDdpRZleJ/DDXX3NCzHN6VUYKORU/b443mJH6PJU4= github.com/conduitio/conduit-connector-file v0.7.0/go.mod h1:OXmcc1eAXmqmn9XoS/C3TdgZn0W1GMyqfNzUZRFmHNU= github.com/conduitio/conduit-connector-generator v0.7.0 h1:Bqsh/ak7gw6k5E8m0PxXOib0zhNlKbrJcIoLLQ0+S08= @@ -278,10 +280,13 @@ github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42 github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/badger/v4 v4.3.0/go.mod h1:Sc0T595g8zqAQRDf44n+z3wG4BOqLwceaFntt8KPxUM= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91/go.mod h1:swkazRqnUf1N62d0Nutz7KIj2UKqsm/H8tD0nBJAXqM= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= @@ -499,6 +504,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjw github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hamba/avro/v2 v2.24.0 h1:axTlaYDkcSY0dVekRSy8cdrsj5MG86WqosUQacKCids= github.com/hamba/avro/v2 v2.24.0/go.mod h1:7vDfy/2+kYCE8WUHoj2et59GTv0ap7ptktMXu0QHePI= +github.com/hamba/avro/v2 v2.25.2 h1:28dqbOCB7wA/3+J1ZN4GQ40tzsFtbtItkTPWgl97el0= +github.com/hamba/avro/v2 v2.25.2/go.mod h1:I8glyswHnpED3Nlx2ZdUe+4LJnCOOyiCzLMno9i/Uu0= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.6.1 h1:P7MR2UP6gNKGPp+y7EZw2kOiq4IR9WiqLvp0XOsVdwI= @@ -536,8 +543,12 @@ github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7Ulw github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jdx/go-netrc v1.0.0 h1:QbLMLyCZGj0NA8glAhxUpf1zDg6cxnWgMBbjq40W0gQ= github.com/jdx/go-netrc v1.0.0/go.mod h1:Gh9eFQJnoTNIRHXl2j5bJXA1u84hQWJWgGh569zF3v8= @@ -1319,6 +1330,8 @@ modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= modernc.org/sqlite v1.31.1 h1:XVU0VyzxrYHlBhIs1DiEgSl0ZtdnPtbLVy8hSkzxGrs= modernc.org/sqlite v1.31.1/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index c8b2d1ef7..132379119 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "time" + "github.com/conduitio/conduit-commons/csync" "github.com/conduitio/conduit/pkg/connector" "github.com/conduitio/conduit/pkg/foundation/cerrors" "github.com/conduitio/conduit/pkg/foundation/log" @@ -59,8 +60,7 @@ type Service struct { connectorPlugins ConnectorPluginService handlers []FailureHandler - runningPipelines map[string]*runnablePipeline - m sync.Mutex + runningPipelines *csync.Map[string, *runnablePipeline] } // NewService initializes and returns a lifecycle.Service. @@ -79,7 +79,7 @@ func NewService( processors: processors, connectorPlugins: connectorPlugins, pipelines: pipelines, - runningPipelines: make(map[string]*runnablePipeline), + runningPipelines: csync.NewMap[string, *runnablePipeline](), } } @@ -171,9 +171,7 @@ func (s *Service) Start( } s.logger.Info(ctx).Str(log.PipelineIDField, pl.ID).Msg("pipeline started") - s.m.Lock() - s.runningPipelines[pl.ID] = rp - s.m.Unlock() + s.runningPipelines.Set(pl.ID, rp) return nil } @@ -183,9 +181,7 @@ func (s *Service) Start( // instead the context for all nodes will be canceled which causes them to stop // running as soon as possible. func (s *Service) Stop(ctx context.Context, pipelineID string, force bool) error { - s.m.Lock() - rp, ok := s.runningPipelines[pipelineID] - s.m.Unlock() + rp, ok := s.runningPipelines.Get(pipelineID) if !ok { return cerrors.Errorf("pipeline %s is not running: %w", pipelineID, pipeline.ErrPipelineNotRunning) @@ -244,13 +240,10 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error // StopAll will ask all the running pipelines to stop gracefully // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { - s.m.Lock() - defer s.m.Unlock() - - for _, rp := range s.runningPipelines { + s.runningPipelines.Range(func(_ string, rp *runnablePipeline) bool { p := rp.pipeline if p.GetStatus() != pipeline.StatusRunning && p.GetStatus() != pipeline.StatusRecovering { - continue + return true } err := s.stopGraceful(ctx, rp, reason) if err != nil { @@ -259,7 +252,8 @@ func (s *Service) StopAll(ctx context.Context, reason error) { Str(log.PipelineIDField, p.ID). Msg("could not stop pipeline") } - } + return true + }) // TODO stop pipelines forcefully after timeout if they are still running } @@ -292,29 +286,25 @@ func (s *Service) Wait(timeout time.Duration) error { func (s *Service) waitInternal() error { var errs []error - s.m.Lock() - pipelines := s.runningPipelines - s.m.Unlock() + pipelines := s.runningPipelines.Copy() - for _, rp := range pipelines { + pipelines.Range(func(_ string, rp *runnablePipeline) bool { if rp.t == nil { - continue + return true } err := rp.t.Wait() if err != nil { errs = append(errs, err) } - } + return true + }) return cerrors.Join(errs...) } // WaitPipeline blocks until the pipeline with the given ID is stopped. func (s *Service) WaitPipeline(id string) error { - s.m.Lock() - p := s.runningPipelines[id] - s.m.Unlock() - - if p.t == nil { + p, ok := s.runningPipelines.Get(id) + if !ok || p.t == nil { return nil } return p.t.Wait() @@ -724,9 +714,7 @@ func (s *Service) runPipeline(ctx context.Context, rp *runnablePipeline) error { Msg("pipeline stopped") // confirmed that all nodes stopped, we can now remove the pipeline from the running pipelines - s.m.Lock() - delete(s.runningPipelines, rp.pipeline.ID) - s.m.Unlock() + s.runningPipelines.Delete(rp.pipeline.ID) s.notify(rp.pipeline.ID, err) return err From 1bf2136285bfa7be4453379554ebf856d9c09589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Barroso?= Date: Fri, 13 Sep 2024 15:58:36 +0200 Subject: [PATCH 39/41] go mod tidy --- go.mod | 1 - go.sum | 24 +++--------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index da0950273..ce70fe7f9 100644 --- a/go.mod +++ b/go.mod @@ -180,7 +180,6 @@ require ( github.com/gofrs/flock v0.12.1 // indirect github.com/gofrs/uuid/v5 v5.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index 48dccfc6f..0f0590d09 100644 --- a/go.sum +++ b/go.sum @@ -198,7 +198,6 @@ github.com/ccojocar/zxcvbn-go v1.0.2/go.mod h1:g1qkXtUSvHP8lhHp5GrSmTz6uWALGRMQd github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4= @@ -219,8 +218,6 @@ github.com/ckaznocha/intrange v0.2.0/go.mod h1:r5I7nUlAAG56xmkOpw4XVr16BXhwYTUdc github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/conduitio/conduit-commons v0.3.0 h1:nxQ++O4dK1p717upkyzbCQu0FLIFyP3OrgHZ9Zxvzvg= -github.com/conduitio/conduit-commons v0.3.0/go.mod h1:roxZ88dv+fpbEjjTzkdGwwbmcpunSuiD8he43y0lAoo= github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff h1:qsOCwSeqFn6bnOqijWnlmUG6osXpCibJRKpI5xq5zR4= github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff/go.mod h1:zeKc8PpXRzB94MhhpwdRI0Rd+Jn/uNs2tqxLhPA05y4= github.com/conduitio/conduit-connector-file v0.7.0 h1:lUfDdpRZleJ/DDXX3NCzHN6VUYKORU/b443mJH6PJU4= @@ -278,15 +275,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= -github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= -github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/badger/v4 v4.3.0 h1:lcsCE1/1qrRhqP+zYx6xDZb8n7U+QlwNicpc676Ub40= github.com/dgraph-io/badger/v4 v4.3.0/go.mod h1:Sc0T595g8zqAQRDf44n+z3wG4BOqLwceaFntt8KPxUM= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 h1:Pux6+xANi0I7RRo5E1gflI4EZ2yx3BGZ75JkAIvGEOA= github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91/go.mod h1:swkazRqnUf1N62d0Nutz7KIj2UKqsm/H8tD0nBJAXqM= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= @@ -307,7 +301,6 @@ github.com/dop251/goja v0.0.0-20240806095544-3491d4a58fbe h1:jwFJkgsdelB87ohlXaA github.com/dop251/goja v0.0.0-20240806095544-3491d4a58fbe/go.mod h1:DF+w/nLMIkvRpyhd/0K+Okbh3fVZBtXLwRtS/ccAa5w= github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c h1:hLoodLRD4KLWIH8eyAQCLcH8EqIrjac7fCkp/fHnvuQ= github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c/go.mod h1:bhGPmCgCCTSRfiMYWjpS46IDo9EUZXlsuUaPXSWGbv0= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -399,8 +392,6 @@ github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= -github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -502,8 +493,6 @@ github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoIS github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= -github.com/hamba/avro/v2 v2.24.0 h1:axTlaYDkcSY0dVekRSy8cdrsj5MG86WqosUQacKCids= -github.com/hamba/avro/v2 v2.24.0/go.mod h1:7vDfy/2+kYCE8WUHoj2et59GTv0ap7ptktMXu0QHePI= github.com/hamba/avro/v2 v2.25.2 h1:28dqbOCB7wA/3+J1ZN4GQ40tzsFtbtItkTPWgl97el0= github.com/hamba/avro/v2 v2.25.2/go.mod h1:I8glyswHnpED3Nlx2ZdUe+4LJnCOOyiCzLMno9i/Uu0= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -541,12 +530,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= @@ -1118,7 +1103,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1328,8 +1312,6 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.31.1 h1:XVU0VyzxrYHlBhIs1DiEgSl0ZtdnPtbLVy8hSkzxGrs= -modernc.org/sqlite v1.31.1/go.mod h1:UqoylwmTb9F+IqXERT8bW9zzOWN8qwAIcLdzeBZs4hA= modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= From fd57ecd49631d2c318af4eb871c80296ffae3835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Fri, 13 Sep 2024 16:30:54 +0200 Subject: [PATCH 40/41] update conduit-commons --- go.mod | 2 +- go.sum | 2 ++ pkg/lifecycle/service.go | 15 +++++++-------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index ce70fe7f9..e01f30e06 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/NYTimes/gziphandler v1.1.1 github.com/bufbuild/buf v1.41.0 - github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff + github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674 github.com/conduitio/conduit-connector-file v0.7.0 github.com/conduitio/conduit-connector-generator v0.7.0 github.com/conduitio/conduit-connector-kafka v0.9.0 diff --git a/go.sum b/go.sum index 0f0590d09..8235d86ac 100644 --- a/go.sum +++ b/go.sum @@ -220,6 +220,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff h1:qsOCwSeqFn6bnOqijWnlmUG6osXpCibJRKpI5xq5zR4= github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff/go.mod h1:zeKc8PpXRzB94MhhpwdRI0Rd+Jn/uNs2tqxLhPA05y4= +github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674 h1:coVk6aVsbP2u2vheM55GaCtu3+JYdIvOimQ0abUrmLA= +github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674/go.mod h1:R/GNsw7iAUy5g2mFUVupAcrfWlGCUACwMvkDDrF39RI= github.com/conduitio/conduit-connector-file v0.7.0 h1:lUfDdpRZleJ/DDXX3NCzHN6VUYKORU/b443mJH6PJU4= github.com/conduitio/conduit-connector-file v0.7.0/go.mod h1:OXmcc1eAXmqmn9XoS/C3TdgZn0W1GMyqfNzUZRFmHNU= github.com/conduitio/conduit-connector-generator v0.7.0 h1:Bqsh/ak7gw6k5E8m0PxXOib0zhNlKbrJcIoLLQ0+S08= diff --git a/pkg/lifecycle/service.go b/pkg/lifecycle/service.go index 132379119..2849da9fe 100644 --- a/pkg/lifecycle/service.go +++ b/pkg/lifecycle/service.go @@ -240,10 +240,10 @@ func (s *Service) stopForceful(ctx context.Context, rp *runnablePipeline) error // StopAll will ask all the running pipelines to stop gracefully // (i.e. that existing messages get processed but not new messages get produced). func (s *Service) StopAll(ctx context.Context, reason error) { - s.runningPipelines.Range(func(_ string, rp *runnablePipeline) bool { + for _, rp := range s.runningPipelines.All() { p := rp.pipeline if p.GetStatus() != pipeline.StatusRunning && p.GetStatus() != pipeline.StatusRecovering { - return true + continue } err := s.stopGraceful(ctx, rp, reason) if err != nil { @@ -252,8 +252,7 @@ func (s *Service) StopAll(ctx context.Context, reason error) { Str(log.PipelineIDField, p.ID). Msg("could not stop pipeline") } - return true - }) + } // TODO stop pipelines forcefully after timeout if they are still running } @@ -286,18 +285,18 @@ func (s *Service) Wait(timeout time.Duration) error { func (s *Service) waitInternal() error { var errs []error + // copy pipelines to keep the map unlocked while we iterate it pipelines := s.runningPipelines.Copy() - pipelines.Range(func(_ string, rp *runnablePipeline) bool { + for _, rp := range pipelines.All() { if rp.t == nil { - return true + continue } err := rp.t.Wait() if err != nil { errs = append(errs, err) } - return true - }) + } return cerrors.Join(errs...) } From 08ed5e86c1984f8b53638affd7b32b5d31af8722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lovro=20Ma=C5=BEgon?= Date: Fri, 13 Sep 2024 16:32:46 +0200 Subject: [PATCH 41/41] go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 8235d86ac..46d131e8e 100644 --- a/go.sum +++ b/go.sum @@ -218,8 +218,6 @@ github.com/ckaznocha/intrange v0.2.0/go.mod h1:r5I7nUlAAG56xmkOpw4XVr16BXhwYTUdc github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff h1:qsOCwSeqFn6bnOqijWnlmUG6osXpCibJRKpI5xq5zR4= -github.com/conduitio/conduit-commons v0.3.1-0.20240913105540-7ca383b14fff/go.mod h1:zeKc8PpXRzB94MhhpwdRI0Rd+Jn/uNs2tqxLhPA05y4= github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674 h1:coVk6aVsbP2u2vheM55GaCtu3+JYdIvOimQ0abUrmLA= github.com/conduitio/conduit-commons v0.3.1-0.20240913141354-f6dd6c835674/go.mod h1:R/GNsw7iAUy5g2mFUVupAcrfWlGCUACwMvkDDrF39RI= github.com/conduitio/conduit-connector-file v0.7.0 h1:lUfDdpRZleJ/DDXX3NCzHN6VUYKORU/b443mJH6PJU4=