diff --git a/conduit/data/block_export_data.go b/conduit/data/block_export_data.go index 209176a3..a8cb9805 100644 --- a/conduit/data/block_export_data.go +++ b/conduit/data/block_export_data.go @@ -2,6 +2,8 @@ package data import ( sdk "github.com/algorand/go-algorand-sdk/v2/types" + + "github.com/algorand/conduit/conduit/telemetry" ) // RoundProvider is the interface which all data types sent to Exporters should implement @@ -16,6 +18,7 @@ type InitProvider interface { GetGenesis() *sdk.Genesis SetGenesis(*sdk.Genesis) NextDBRound() sdk.Round + GetTelemetryClient() telemetry.Client } // BlockData is provided to the Exporter on each round. diff --git a/conduit/data/config.go b/conduit/data/config.go index 83f09baf..09ad5c43 100644 --- a/conduit/data/config.go +++ b/conduit/data/config.go @@ -41,6 +41,15 @@ type Metrics struct { Prefix string `yaml:"prefix"` } +// Telemetry configs for sending Telemetry to OpenSearch +type Telemetry struct { + Enabled bool `yaml:"enabled"` + URI string `yaml:"uri"` + Index string `yaml:"index"` + UserName string `yaml:"username"` + Password string `yaml:"password"` +} + // Config stores configuration specific to the conduit pipeline type Config struct { // ConduitArgs are the program inputs. Should not be serialized for config. @@ -61,6 +70,8 @@ type Config struct { RetryCount uint64 `yaml:"retry-count"` // RetryDelay is a duration amount interpreted from a string RetryDelay time.Duration `yaml:"retry-delay"` + + Telemetry Telemetry `yaml:"telemetry"` } // Valid validates pipeline config diff --git a/conduit/data/config_test.go b/conduit/data/config_test.go index bbbff3f9..74c0f77e 100644 --- a/conduit/data/config_test.go +++ b/conduit/data/config_test.go @@ -85,7 +85,6 @@ exporters: name: "noop" config: connectionstring: ""`, "field exporters not found"}, - {"config not configs", `--- log-level: info importer: diff --git a/conduit/init_provider.go b/conduit/init_provider.go index deaf23d6..6acba2c3 100644 --- a/conduit/init_provider.go +++ b/conduit/init_provider.go @@ -2,19 +2,23 @@ package conduit import ( sdk "github.com/algorand/go-algorand-sdk/v2/types" + + "github.com/algorand/conduit/conduit/telemetry" ) // PipelineInitProvider algod based init provider type PipelineInitProvider struct { - currentRound *sdk.Round - genesis *sdk.Genesis + currentRound *sdk.Round + genesis *sdk.Genesis + telemetryClient telemetry.Client } // MakePipelineInitProvider constructs an init provider. -func MakePipelineInitProvider(currentRound *sdk.Round, genesis *sdk.Genesis) *PipelineInitProvider { +func MakePipelineInitProvider(currentRound *sdk.Round, genesis *sdk.Genesis, client telemetry.Client) *PipelineInitProvider { return &PipelineInitProvider{ - currentRound: currentRound, - genesis: genesis, + currentRound: currentRound, + genesis: genesis, + telemetryClient: client, } } @@ -32,3 +36,8 @@ func (a *PipelineInitProvider) GetGenesis() *sdk.Genesis { func (a *PipelineInitProvider) NextDBRound() sdk.Round { return *a.currentRound } + +// GetTelemetryClient gets the telemetry state in the init provider +func (a *PipelineInitProvider) GetTelemetryClient() telemetry.Client { + return a.telemetryClient +} diff --git a/conduit/pipeline/metadata.go b/conduit/pipeline/metadata.go index 478c4b41..9c71e7d5 100644 --- a/conduit/pipeline/metadata.go +++ b/conduit/pipeline/metadata.go @@ -18,6 +18,7 @@ type state struct { GenesisHash string `json:"genesis-hash"` Network string `json:"network"` NextRound uint64 `json:"next-round"` + TelemetryID string `json:"telemetry-id,omitempty"` } // encodeToFile writes the state object to the dataDir diff --git a/conduit/pipeline/metadata_test.go b/conduit/pipeline/metadata_test.go index 0f7ccdcb..538227cd 100644 --- a/conduit/pipeline/metadata_test.go +++ b/conduit/pipeline/metadata_test.go @@ -54,10 +54,12 @@ func TestBlockMetaDataFile(t *testing.T) { assert.Equal(t, pipelineMetadata.GenesisHash, metaData.GenesisHash) assert.Equal(t, pipelineMetadata.NextRound, metaData.NextRound) assert.Equal(t, pipelineMetadata.Network, metaData.Network) + assert.Equal(t, pipelineMetadata.TelemetryID, metaData.TelemetryID) // Test that file encodes correctly pipelineMetadata.GenesisHash = "HASH" pipelineMetadata.NextRound = 7 + pipelineMetadata.TelemetryID = "SOME_ID" err = pipelineMetadata.encodeToFile(datadir) assert.NoError(t, err) metaData, err = readBlockMetadata(datadir) @@ -65,4 +67,5 @@ func TestBlockMetaDataFile(t *testing.T) { assert.Equal(t, "HASH", metaData.GenesisHash) assert.Equal(t, uint64(7), metaData.NextRound) assert.Equal(t, pipelineMetadata.Network, metaData.Network) + assert.Equal(t, pipelineMetadata.TelemetryID, metaData.TelemetryID) } diff --git a/conduit/pipeline/pipeline.go b/conduit/pipeline/pipeline.go index 892156a5..e8c07784 100644 --- a/conduit/pipeline/pipeline.go +++ b/conduit/pipeline/pipeline.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/conduit/conduit/plugins/exporters" "github.com/algorand/conduit/conduit/plugins/importers" "github.com/algorand/conduit/conduit/plugins/processors" + "github.com/algorand/conduit/conduit/telemetry" ) // Pipeline is a struct that orchestrates the entire @@ -198,6 +199,25 @@ func (p *pipelineImpl) pluginRoundOverride() (uint64, error) { return pluginOverride, nil } +// initializeTelemetry initializes telemetry and reads or sets the GUID in the metadata. +func (p *pipelineImpl) initializeTelemetry() (telemetry.Client, error) { + telemetryConfig := telemetry.MakeTelemetryConfig(p.cfg.Telemetry.URI, p.cfg.Telemetry.Index, p.cfg.Telemetry.UserName, p.cfg.Telemetry.Password) + telemetryClient, err := telemetry.MakeOpenSearchClient(telemetryConfig) + if err != nil { + return nil, fmt.Errorf("failed to initialize telemetry: %w", err) + } + p.logger.Infof("Telemetry initialized with URI: %s", telemetryConfig.URI) + + // If GUID is not in metadata, save it. Otherwise, use the GUID from metadata. + if p.pipelineMetadata.TelemetryID == "" { + p.pipelineMetadata.TelemetryID = telemetryClient.TelemetryConfig.GUID + } else { + telemetryClient.TelemetryConfig.GUID = p.pipelineMetadata.TelemetryID + } + + return telemetryClient, nil +} + // Init prepares the pipeline for processing block data func (p *pipelineImpl) Init() error { p.logger.Infof("Starting Pipeline Initialization") @@ -257,8 +277,27 @@ func (p *pipelineImpl) Init() error { // InitProvider round := sdk.Round(p.pipelineMetadata.NextRound) - // Initial genesis object is nil--gets updated after importer.Init - var initProvider data.InitProvider = conduit.MakePipelineInitProvider(&round, nil) + + // Initialize Telemetry + var telemetryClient telemetry.Client + if p.cfg.Telemetry.Enabled { + // If telemetry cannot be initialized, log a warning and continue + // pipeline initialization. + var telemetryErr error + telemetryClient, telemetryErr = p.initializeTelemetry() + if telemetryErr != nil { + p.logger.Warnf("Telemetry initialization failed, continuing without telemetry: %s", telemetryErr) + } else { + // Try sending a startup event. If it fails, log a warning and continue + event := telemetryClient.MakeTelemetryStartupEvent() + if telemetryErr = telemetryClient.SendEvent(event); telemetryErr != nil { + p.logger.Warnf("failed to send telemetry event: %s", telemetryErr) + } + } + } + + // Initial genesis object is nil and gets updated after importer.Init + var initProvider data.InitProvider = conduit.MakePipelineInitProvider(&round, nil, telemetryClient) p.initProvider = &initProvider // Initialize Importer diff --git a/conduit/pipeline/pipeline_test.go b/conduit/pipeline/pipeline_test.go index dc716537..32c610a4 100644 --- a/conduit/pipeline/pipeline_test.go +++ b/conduit/pipeline/pipeline_test.go @@ -30,6 +30,7 @@ import ( "github.com/algorand/conduit/conduit/plugins/exporters" "github.com/algorand/conduit/conduit/plugins/importers" "github.com/algorand/conduit/conduit/plugins/processors" + "github.com/algorand/conduit/conduit/telemetry" ) // a unique block data to validate with tests @@ -529,6 +530,44 @@ func TestPipelineMetricsConfigs(t *testing.T) { assert.Equal(t, pImpl.cfg.Metrics.Prefix, prefixOverride) } +func TestPipelineTelemetryConfigs(t *testing.T) { + pImpl, _, _, _, _ := mockPipeline(t, "") + + // telemetry OFF, check that client is nil + pImpl.cfg.Telemetry = data.Telemetry{ + Enabled: false, + } + pImpl.Init() + baseClient := (*pImpl.initProvider).GetTelemetryClient() + assert.Nil(t, baseClient) + + // telemetry ON + pImpl.cfg.Telemetry = data.Telemetry{ + Enabled: true, + URI: "test-uri", + Index: "test-index", + UserName: "test-username", + Password: "test-password", + } + pImpl.Init() + baseClient = (*pImpl.initProvider).GetTelemetryClient() + client := baseClient.(*telemetry.OpenSearchClient) + + assert.NotNil(t, client) + assert.NotNil(t, client.Client) + assert.Equal(t, true, client.TelemetryConfig.Enable) + assert.Equal(t, "test-uri", client.TelemetryConfig.URI) + assert.Equal(t, "test-index", client.TelemetryConfig.Index) + assert.Equal(t, "test-username", client.TelemetryConfig.UserName) + assert.Equal(t, "test-password", client.TelemetryConfig.Password) + + event := client.MakeTelemetryStartupEvent() + assert.Equal(t, "starting conduit", event.Message) + assert.NotEmpty(t, event.Time) + assert.NotEmpty(t, event.GUID) + assert.NotEmpty(t, event.Version) +} + func TestRoundOverrideValidConflict(t *testing.T) { t.Run("processor_no_conflict", func(t *testing.T) { pImpl, _, mImporter, mProcessor, _ := mockPipeline(t, "") diff --git a/conduit/plugins/exporters/filewriter/file_exporter_test.go b/conduit/plugins/exporters/filewriter/file_exporter_test.go index f7f0bdb4..555df398 100644 --- a/conduit/plugins/exporters/filewriter/file_exporter_test.go +++ b/conduit/plugins/exporters/filewriter/file_exporter_test.go @@ -80,7 +80,7 @@ func TestExporterInitDefaults(t *testing.T) { defer fileExp.Close() pcfg := plugins.MakePluginConfig(fmt.Sprintf("block-dir: %s", tc.blockdir)) pcfg.DataDir = tempdir - err := fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil), pcfg, logger) + err := fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil, nil), pcfg, logger) require.NoError(t, err) }) } @@ -92,12 +92,12 @@ func TestExporterInit(t *testing.T) { defer fileExp.Close() // creates a new output file - err := fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil), plugins.MakePluginConfig(config), logger) + err := fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil, nil), plugins.MakePluginConfig(config), logger) assert.NoError(t, err) fileExp.Close() // can open existing file - err = fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil), plugins.MakePluginConfig(config), logger) + err = fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil, nil), plugins.MakePluginConfig(config), logger) assert.NoError(t, err) fileExp.Close() } @@ -121,7 +121,7 @@ func sendData(t *testing.T, fileExp exporters.Exporter, config string, numRounds // initialize rnd := sdk.Round(0) - err = fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&rnd, nil), plugins.MakePluginConfig(config), logger) + err = fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&rnd, nil, nil), plugins.MakePluginConfig(config), logger) require.NoError(t, err) // incorrect round @@ -182,7 +182,7 @@ func TestExporterClose(t *testing.T) { config, _ := getConfig(t) fileExp := fileCons.New() rnd := sdk.Round(0) - fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&rnd, nil), plugins.MakePluginConfig(config), logger) + fileExp.Init(context.Background(), conduit.MakePipelineInitProvider(&rnd, nil, nil), plugins.MakePluginConfig(config), logger) require.NoError(t, fileExp.Close()) } diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go b/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go index c0544bc0..62a03773 100644 --- a/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go @@ -41,26 +41,26 @@ func TestExporterMetadata(t *testing.T) { func TestConnectDisconnectSuccess(t *testing.T) { pgsqlExp := pgsqlConstructor.New() cfg := plugins.MakePluginConfig("test: true\nconnection-string: ''") - assert.NoError(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}), cfg, logger)) + assert.NoError(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}, nil), cfg, logger)) assert.NoError(t, pgsqlExp.Close()) } func TestConnectUnmarshalFailure(t *testing.T) { pgsqlExp := pgsqlConstructor.New() cfg := plugins.MakePluginConfig("'") - assert.ErrorContains(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil), cfg, logger), "connect failure in unmarshalConfig") + assert.ErrorContains(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil, nil), cfg, logger), "connect failure in unmarshalConfig") } func TestConnectDbFailure(t *testing.T) { pgsqlExp := pgsqlConstructor.New() cfg := plugins.MakePluginConfig("") - assert.ErrorContains(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil), cfg, logger), "connection string is empty for postgres") + assert.ErrorContains(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, nil, nil), cfg, logger), "connection string is empty for postgres") } func TestReceiveInvalidBlock(t *testing.T) { pgsqlExp := pgsqlConstructor.New() cfg := plugins.MakePluginConfig("test: true") - assert.NoError(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}), cfg, logger)) + assert.NoError(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}, nil), cfg, logger)) invalidBlock := data.BlockData{ BlockHeader: sdk.BlockHeader{ Round: 1, @@ -76,7 +76,7 @@ func TestReceiveInvalidBlock(t *testing.T) { func TestReceiveAddBlockSuccess(t *testing.T) { pgsqlExp := pgsqlConstructor.New() cfg := plugins.MakePluginConfig("test: true") - assert.NoError(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}), cfg, logger)) + assert.NoError(t, pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}, nil), cfg, logger)) block := data.BlockData{ BlockHeader: sdk.BlockHeader{}, @@ -92,7 +92,7 @@ func TestPostgresqlExporterInit(t *testing.T) { cfg := plugins.MakePluginConfig("test: true") // genesis hash mismatch - initProvider := conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}) + initProvider := conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}, nil) initProvider.SetGenesis(&sdk.Genesis{ Network: "test", }) @@ -101,7 +101,7 @@ func TestPostgresqlExporterInit(t *testing.T) { // incorrect round round = 1 - err = pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}), cfg, logger) + err = pgsqlExp.Init(context.Background(), conduit.MakePipelineInitProvider(&round, &sdk.Genesis{}, nil), cfg, logger) assert.Contains(t, err.Error(), "initializing block round 1 but next round to account is 0") } diff --git a/conduit/plugins/importers/algod/algod_importer_test.go b/conduit/plugins/importers/algod/algod_importer_test.go index d01df38c..b5fad1c3 100644 --- a/conduit/plugins/importers/algod/algod_importer_test.go +++ b/conduit/plugins/importers/algod/algod_importer_test.go @@ -48,7 +48,7 @@ func TestCloseSuccess(t *testing.T) { mode: %s netaddr: %s `, archivalModeStr, ts.URL) - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(cfgStr), logger) assert.NoError(t, err) gen, err := testImporter.GetGenesis() assert.NoError(t, err) @@ -330,7 +330,7 @@ func TestInitCatchup(t *testing.T) { } cfgStr, err := yaml.Marshal(cfg) require.NoError(t, err) - err = testImporter.Init(context.Background(), conduit.MakePipelineInitProvider(&ttest.targetRound, nil), plugins.MakePluginConfig(string(cfgStr)), testLogger) + err = testImporter.Init(context.Background(), conduit.MakePipelineInitProvider(&ttest.targetRound, nil, nil), plugins.MakePluginConfig(string(cfgStr)), testLogger) if ttest.errInit != "" { require.ErrorContains(t, err, ttest.errInit, ttest.errInit) } else { @@ -370,7 +370,7 @@ func TestInitParseUrlFailure(t *testing.T) { mode: %s netaddr: %s `, "follower", url) - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(cfgStr), logger) assert.ErrorContains(t, err, "parse") gen, err := testImporter.GetGenesis() require.ErrorContains(t, err, "algod importer is missing its genesis") @@ -390,7 +390,7 @@ func TestInitModeFailure(t *testing.T) { mode: %s netaddr: %s `, name, ts.URL) - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(cfgStr), logger) assert.EqualError(t, err, fmt.Sprintf("algod importer was set to a mode (%s) that wasn't supported", name)) gen, err := testImporter.GetGenesis() require.ErrorContains(t, err, "algod importer is missing its genesis") @@ -409,7 +409,7 @@ func TestInitGenesisFailure(t *testing.T) { mode: %s netaddr: %s `, archivalModeStr, ts.URL) - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(cfgStr), logger) assert.Error(t, err) assert.ErrorContains(t, err, "unable to fetch genesis file") gen, err := testImporter.GetGenesis() @@ -425,7 +425,7 @@ func TestInitUnmarshalFailure(t *testing.T) { logger := logrus.New() testImporter := New() - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig("`"), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig("`"), logger) assert.Error(t, err) assert.ErrorContains(t, err, "connect failure in unmarshalConfig") gen, err := testImporter.GetGenesis() @@ -446,7 +446,7 @@ func TestWaitForBlockBlockFailure(t *testing.T) { mode: %s netaddr: %s `, archivalModeStr, ts.URL) - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(cfgStr), logger) assert.NoError(t, err) gen, err := testImporter.GetGenesis() require.NoError(t, err) @@ -504,7 +504,7 @@ func TestGetBlockSuccess(t *testing.T) { defer cancel() testImporter := &algodImporter{} - err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(string(cfgStr)), logger) + err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(string(cfgStr)), logger) assert.NoError(t, err) gen, err := testImporter.GetGenesis() require.NoError(t, err) @@ -566,7 +566,7 @@ func TestGetBlockContextCancelled(t *testing.T) { mode: %s netaddr: %s `, ttest.name, ttest.algodServer.URL) - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(cfgStr), logger) assert.NoError(t, err) gen, err := testImporter.GetGenesis() require.NoError(t, err) @@ -613,7 +613,7 @@ func TestGetBlockFailure(t *testing.T) { mode: %s netaddr: %s `, ttest.name, ttest.algodServer.URL) - err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger) + err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(cfgStr), logger) assert.NoError(t, err) gen, err := testImporter.GetGenesis() require.NoError(t, err) @@ -707,7 +707,7 @@ func TestGetBlockErrors(t *testing.T) { testImporter := &algodImporter{} var gen *sdk.Genesis - err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), pcfg, testLogger) + err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), pcfg, testLogger) require.NoError(t, err) gen, err = testImporter.GetGenesis() require.NoError(t, err) diff --git a/conduit/plugins/importers/filereader/filereader_test.go b/conduit/plugins/importers/filereader/filereader_test.go index 31082719..42cafc99 100644 --- a/conduit/plugins/importers/filereader/filereader_test.go +++ b/conduit/plugins/importers/filereader/filereader_test.go @@ -87,7 +87,7 @@ func initializeImporter(t *testing.T, numRounds int) (importer importers.Importe } data, err := yaml.Marshal(cfg) require.NoError(t, err) - err = importer.Init(context.Background(), conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(string(data)), logger) + err = importer.Init(context.Background(), conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(string(data)), logger) assert.NoError(t, err) genesis, err = importer.GetGenesis() require.NoError(t, err) @@ -103,7 +103,7 @@ func TestInitSuccess(t *testing.T) { func TestInitUnmarshalFailure(t *testing.T) { testImporter = New() - err := testImporter.Init(context.Background(), conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig("`"), logger) + err := testImporter.Init(context.Background(), conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig("`"), logger) assert.Error(t, err) assert.ErrorContains(t, err, "invalid configuration") testImporter.Close() @@ -134,7 +134,7 @@ func TestRetryAndDuration(t *testing.T) { } data, err := yaml.Marshal(cfg) require.NoError(t, err) - err = importer.Init(context.Background(), conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(string(data)), logger) + err = importer.Init(context.Background(), conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(string(data)), logger) assert.NoError(t, err) start := time.Now() @@ -157,7 +157,7 @@ func TestRetryWithCancel(t *testing.T) { data, err := yaml.Marshal(cfg) ctx, cancel := context.WithCancel(context.Background()) require.NoError(t, err) - err = importer.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(string(data)), logger) + err = importer.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil, nil), plugins.MakePluginConfig(string(data)), logger) assert.NoError(t, err) // Cancel after delay diff --git a/conduit/telemetry/README.md b/conduit/telemetry/README.md new file mode 100644 index 00000000..cbef0db9 --- /dev/null +++ b/conduit/telemetry/README.md @@ -0,0 +1,13 @@ +# Conduit Telemetry + +The goal is to gain information about how our users use Conduit in the wild. This currently does not include things like logging (logrus) or metrics (Prometheus and Datadog for internal performance tests). + +## Implementation + +Telemetry related utilities are in this directory. + +### Configurations +The `conduit.yml` config file includes a telemetry boolean, and optional fields to enter the OpenSearch URI, credentials (username/password), and Index name. When true, `telemetry.Config` is initialized with the optional fields. The GUID is generated when the pipeline is started, and saved to the pipeline's `metadata.json`. If the GUID already exists, it will be read from the file. + +### Client +The configuration and client is represented by the `telemetry.Client` interface and will use the official [Go opensearch client](https://opensearch.org/docs/latest/clients/go/) by default. Users can also define their own clients to send telemetry events. This will also have client functions to write to OpenSearch using the official Go client. Currently, the default telemetry client marshalls the marshalls the `telemetry.Event` struct into JSON before sending the event. diff --git a/conduit/telemetry/telemetry.go b/conduit/telemetry/telemetry.go new file mode 100644 index 00000000..507dab93 --- /dev/null +++ b/conduit/telemetry/telemetry.go @@ -0,0 +1,84 @@ +package telemetry + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/google/uuid" + "github.com/opensearch-project/opensearch-go/v2" + "github.com/opensearch-project/opensearch-go/v2/opensearchapi" + + "github.com/algorand/conduit/version" +) + +// MakeTelemetryConfig initializes a new TelemetryConfig. +func MakeTelemetryConfig(telemetryURI, index, username, password string) Config { + return Config{ + Enable: true, + URI: telemetryURI, + GUID: uuid.New().String(), // Use Google UUID instead of go-algorand utils + Index: index, + UserName: username, + Password: password, + } +} + +// initializeOpenSearchClient creates a new OpenSearch client. +func initializeOpenSearchClient(cfg Config) (*opensearch.Client, error) { + client, err := opensearch.NewClient(opensearch.Config{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Addresses: []string{cfg.URI}, + // These credentials are here intentionally. Not a bug. + Username: cfg.UserName, + Password: cfg.Password, + }) + if err != nil { + return nil, fmt.Errorf("unable to create new OpenSearch client with URI %s: %w", cfg.URI, err) + } + return client, nil +} + +// MakeOpenSearchClient initializes a new TelemetryState. +func MakeOpenSearchClient(cfg Config) (*OpenSearchClient, error) { + client, err := initializeOpenSearchClient(cfg) + if err != nil { + return nil, err + } + + telemetryState := &OpenSearchClient{ + Client: client, + TelemetryConfig: cfg, + } + return telemetryState, nil +} + +// MakeTelemetryStartupEvent sends a startup event when the pipeline is initialized. +func (t *OpenSearchClient) MakeTelemetryStartupEvent() Event { + return Event{ + Message: "starting conduit", + GUID: t.TelemetryConfig.GUID, + Time: time.Now(), + Version: version.LongVersion(), + } +} + +// SendEvent sends a TelemetryEvent to OpenSearch. +func (t *OpenSearchClient) SendEvent(event Event) error { + data, _ := json.Marshal(event) + req := opensearchapi.IndexRequest{ + Index: t.TelemetryConfig.Index, + Body: bytes.NewReader(data), + } + _, err := req.Do(context.Background(), t.Client) + if err != nil { + return fmt.Errorf("failed to insert event: %w", err) + } + return nil +} diff --git a/conduit/telemetry/telemetryCommon.go b/conduit/telemetry/telemetryCommon.go new file mode 100644 index 00000000..7e730997 --- /dev/null +++ b/conduit/telemetry/telemetryCommon.go @@ -0,0 +1,43 @@ +package telemetry + +import ( + "time" + + "github.com/opensearch-project/opensearch-go/v2" +) + +// Config represents the configuration of Telemetry logging +type Config struct { + Enable bool + SendToLog bool + URI string + Name string + GUID string + Index string + UserName string + Password string +} + +// Event represents a single event to be emitted to OpenSearch +type Event struct { + // Time at which the event was created + Time time.Time `json:"timestamp"` + // Event message + Message string `json:"message"` + // Unique ID assigned to the pipeline upon initialization + GUID string `json:"guid"` + // Version of conduit + Version string `json:"version"` +} + +// Client represents the Telemetry client and config +type Client interface { + MakeTelemetryStartupEvent() Event + SendEvent(event Event) error +} + +// OpenSearchClient holds the OpenSearch client and TelemetryConfig +type OpenSearchClient struct { + Client *opensearch.Client + TelemetryConfig Config +} diff --git a/conduit/telemetry/telemetry_test.go b/conduit/telemetry/telemetry_test.go new file mode 100644 index 00000000..8666785a --- /dev/null +++ b/conduit/telemetry/telemetry_test.go @@ -0,0 +1,32 @@ +package telemetry + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTelemetryConfig(t *testing.T) { + + config := MakeTelemetryConfig("test-uri", "test-index", "test-user", "test-password") + require.Equal(t, true, config.Enable) + require.Equal(t, "test-uri", config.URI) + require.Equal(t, "test-index", config.Index) + require.Equal(t, "test-user", config.UserName) + require.Equal(t, "test-password", config.Password) + require.NotEqual(t, "", config.GUID) +} + +func TestMakeTelemetryStartupEvent(t *testing.T) { + config := Config{ + GUID: "test-guid", + } + state, err := MakeOpenSearchClient(config) + require.NoError(t, err) + event := state.MakeTelemetryStartupEvent() + require.Equal(t, "starting conduit", event.Message) + require.Equal(t, "test-guid", event.GUID) + // Can't get version or time nicely in testing, so just check that it was populated + require.NotEmpty(t, event.Time) + require.NotEmpty(t, event.Version) +} diff --git a/go.mod b/go.mod index 8bc363b6..b28b23fe 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,13 @@ require ( github.com/algorand/go-algorand-sdk/v2 v2.2.0 github.com/algorand/go-codec/codec v1.1.10 github.com/algorand/indexer v0.0.0-20230601214318-cb2f42818f82 + github.com/google/uuid v1.3.0 github.com/jackc/pgx/v4 v4.13.0 + github.com/opensearch-project/opensearch-go/v2 v2.3.0 github.com/prometheus/client_golang v1.11.1 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.3.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/yuin/goldmark v1.5.4 gopkg.in/yaml.v3 v3.0.1 @@ -34,7 +36,6 @@ require ( github.com/go-openapi/swag v0.19.5 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-querystring v1.0.0 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/invopop/yaml v0.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -72,10 +73,10 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.17.0 // indirect golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.1.0 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 05b2ae8f..eb7ce832 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,19 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= +github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -275,8 +288,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -404,6 +418,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -505,6 +521,8 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= +github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/orlangure/gnomock v0.12.0 h1:qlQ2iO0Edm73YfIWXqEm+iU4rE6iUx219G9A8u1/wzU= github.com/orlangure/gnomock v0.12.0/go.mod h1:hTsIl+VEiqfXVkeXxsT6RM/Ed6zlsp31VDc75+aKrgw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -600,8 +618,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -618,6 +636,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -710,6 +729,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -754,8 +774,10 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -867,10 +889,16 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -880,8 +908,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -952,12 +981,12 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1151,4 +1180,4 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= \ No newline at end of file diff --git a/pkg/cli/internal/initialize/conduit.test.init.default.yml b/pkg/cli/internal/initialize/conduit.test.init.default.yml index 798647b6..40282b96 100644 --- a/pkg/cli/internal/initialize/conduit.test.init.default.yml +++ b/pkg/cli/internal/initialize/conduit.test.init.default.yml @@ -36,4 +36,13 @@ exporter: # DropCertificate is used to remove the vote certificate from the block data before writing files. drop-certificate: true - +# Enable telemetry for conduit +telemetry: + enabled: false + + # By default the following fields will be configured to send data to Algorand. + # To store your own telemetry events, they can be overridden. + # uri: "" + # index: "" + # username: "" + # password: "" diff --git a/pkg/cli/internal/initialize/conduit.yml.example b/pkg/cli/internal/initialize/conduit.yml.example index 9bf9e9e9..56709af6 100644 --- a/pkg/cli/internal/initialize/conduit.yml.example +++ b/pkg/cli/internal/initialize/conduit.yml.example @@ -32,4 +32,15 @@ processors: %s # An exporter is defined to do something with the data. exporter: -%s \ No newline at end of file +%s + +# Enable telemetry for conduit +telemetry: + enabled: false + + # By default the following fields will be configured to send data to Algorand. + # To store your own telemetry events, they can be overridden. + # uri: "" + # index: "" + # username: "" + # password: ""