misc: Docs Update + Other non-functional changes

.golangci.yml

@@ -7,6 +7,7 @@ linters:
   enable:
     - deadcode
     - errcheck
+    - exportloopref
     - gci
     - gofmt
     - gosimple

.markdownlint.yml

@@ -0,0 +1,7 @@
+default: true
+
+MD010:
+  code_blocks: false
+
+MD013:
+  line_length: 200

conduit/pipeline/pipeline_test.go

@@ -612,6 +612,7 @@ func TestRoundRequestError(t *testing.T) {
 func TestRoundOverride(t *testing.T) {
 	// cli override NextRound, 0 is a test for no override.
 	for i := 0; i < 10; i++ {
+		i := i
 		t.Run(fmt.Sprintf("cli round override %d", i), func(t *testing.T) {
 			t.Parallel()
 			pImpl, _, _, _, _ := mockPipeline(t, "")

conduit/plugins/importers/algod/algod_importer_test.go

@@ -48,8 +48,9 @@ func TestCloseSuccess(t *testing.T) {
 mode: %s
 netaddr: %s
 `, archivalModeStr, ts.URL)
-	_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
+	gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
 	assert.NoError(t, err)
+	assert.NotNil(t, gen)
 	err = testImporter.Close()
 	assert.NoError(t, err)
 }
@@ -118,6 +119,7 @@ func Test_checkRounds(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
+		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 			testLogger, hook := test.NewNullLogger()
@@ -284,11 +286,13 @@ func TestInitCatchup(t *testing.T) {
 			}
 			cfgStr, err := yaml.Marshal(cfg)
 			require.NoError(t, err)
-			_, err = testImporter.Init(context.Background(), conduit.MakePipelineInitProvider(&ttest.targetRound, nil), plugins.MakePluginConfig(string(cfgStr)), testLogger)
+			var gen *sdk.Genesis
+			gen, err = testImporter.Init(context.Background(), conduit.MakePipelineInitProvider(&ttest.targetRound, nil), plugins.MakePluginConfig(string(cfgStr)), testLogger)
 			if ttest.err != "" {
 				require.ErrorContains(t, err, ttest.err, ttest.err)
 			} else {
 				require.NoError(t, err)
+				assert.NotNil(t, gen)
 			}
 			_ = testImporter.Close()
 			// Make sure each of the expected log messages are present
@@ -315,8 +319,9 @@ func TestInitParseUrlFailure(t *testing.T) {
 mode: %s
 netaddr: %s
 `, "follower", url)
-	_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
+	gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
 	assert.ErrorContains(t, err, "parse")
+	assert.Nil(t, gen)
 }
 
 func TestInitModeFailure(t *testing.T) {
@@ -332,8 +337,9 @@ func TestInitModeFailure(t *testing.T) {
 mode: %s
 netaddr: %s
 `, name, ts.URL)
-	_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
+	gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
 	assert.EqualError(t, err, fmt.Sprintf("algod importer was set to a mode (%s) that wasn't supported", name))
+	assert.Nil(t, gen)
 }
 
 func TestInitGenesisFailure(t *testing.T) {
@@ -348,9 +354,10 @@ func TestInitGenesisFailure(t *testing.T) {
 mode: %s
 netaddr: %s
 `, archivalModeStr, ts.URL)
-	_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
+	gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
 	assert.Error(t, err)
 	assert.ErrorContains(t, err, "unable to fetch genesis file")
+	assert.Nil(t, gen)
 	testImporter.Close()
 }
 
@@ -361,9 +368,10 @@ func TestInitUnmarshalFailure(t *testing.T) {
 	logger := logrus.New()
 
 	testImporter := New()
-	_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig("`"), logger)
+	gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig("`"), logger)
 	assert.Error(t, err)
 	assert.ErrorContains(t, err, "connect failure in unmarshalConfig")
+	assert.Nil(t, gen)
 	testImporter.Close()
 }
 
@@ -389,8 +397,9 @@ func TestWaitForBlockBlockFailure(t *testing.T) {
 mode: %s
 netaddr: %s
 `, archivalModeStr, ts.URL)
-	_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
+	gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
 	assert.NoError(t, err)
+	assert.NotNil(t, gen)
 	assert.NotEqual(t, testImporter, nil)
 
 	blk, err := testImporter.GetBlock(uint64(10))
@@ -444,8 +453,10 @@ func TestGetBlockSuccess(t *testing.T) {
 			defer cancel()
 			testImporter := &algodImporter{}
 
-			_, err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(string(cfgStr)), logger)
+			var gen *sdk.Genesis
+			gen, err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(string(cfgStr)), logger)
 			assert.NoError(t, err)
+			assert.NotNil(t, gen)
 			assert.NotEqual(t, testImporter, nil)
 
 			downloadedBlk, err := testImporter.GetBlock(uint64(0))
@@ -503,8 +514,9 @@ func TestGetBlockContextCancelled(t *testing.T) {
 mode: %s
 netaddr: %s
 `, ttest.name, ttest.algodServer.URL)
-			_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
+			gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
 			assert.NoError(t, err)
+			assert.NotNil(t, gen)
 			assert.NotEqual(t, testImporter, nil)
 
 			cancel()
@@ -547,8 +559,9 @@ func TestGetBlockFailure(t *testing.T) {
 mode: %s
 netaddr: %s
 `, ttest.name, ttest.algodServer.URL)
-			_, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
+			gen, err := testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), plugins.MakePluginConfig(cfgStr), logger)
 			assert.NoError(t, err)
+			assert.NotNil(t, gen)
 			assert.NotEqual(t, testImporter, nil)
 
 			_, err = testImporter.GetBlock(uint64(10))
@@ -637,8 +650,10 @@ func TestGetBlockErrors(t *testing.T) {
 			pRound := sdk.Round(1)
 
 			testImporter := &algodImporter{}
-			_, err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), pcfg, testLogger)
+			var gen *sdk.Genesis
+			gen, err = testImporter.Init(ctx, conduit.MakePipelineInitProvider(&pRound, nil), pcfg, testLogger)
 			require.NoError(t, err)
+			require.NotNil(t, gen)
 
 			// Run the test
 			_, err = testImporter.GetBlock(tc.rnd)

docs/Configuration.md

@@ -8,6 +8,7 @@ Use `./conduit -h` for command options.
 There are several top level configurations for configuring behavior of the conduit process. Most detailed configuration is made on a per-plugin basis. These are split between `Importer`, `Processor` and `Exporter` plugins.
 
 Here is an example configuration which shows the general format:
+
 ```yaml
 # optional: hide the startup banner.
 hide-banner: true|false

docs/Development.md

@@ -1,23 +1,26 @@
 # Creating A Plugin
 
 There are three different interfaces to implement, depending on what sort of functionality you are adding:
+
 * Importer: for sourcing data into the system.
 * Processor: for manipulating data as it goes through the system.
 * Exporter: for sending processed data somewhere.
 
 All plugins should be implemented in the respective `importers`, `processors`, or `exporters` package.
 
 For interface details, refer to the godoc:
+
 * [Importer](https://pkg.go.dev/github.com/algorand/conduit/conduit/plugins/importers)
 * [Processor](https://pkg.go.dev/github.com/algorand/conduit/conduit/plugins/processors)
 * [Exporter](https://pkg.go.dev/github.com/algorand/conduit/conduit/plugins/exporters)
 
-# Registering a plugin
+## Registering a plugin
 
-## Register the Constructor
+### Register the Constructor
 
-The constructor is registered to the system by name in the init this is how the configuration is able to dynamically create pipelines:
-```
+The constructor is registered to the system by name in the init. This is how the configuration is able to dynamically create pipelines. For example:
+
+```go
 func init() {
 	exporters.RegisterExporter(noopExporterMetadata.ExpName, exporters.ExporterConstructorFunc(func() exporters.Exporter {
 		return &noopExporter{}
@@ -27,38 +30,41 @@ func init() {
 
 There are similar interfaces for each plugin type.
 
-## Load the Plugin
+### Load the Plugin
 
-Each plugin package contains an `all.go` file. Add your plugin to the import statement, this causes the init function to be called and ensures the plugin is registered.
+Each plugin package contains an `all.go` file.
+Adding an import statement for the plugin ensures that the plugin is registered by Conduit.
+This occurs because the `init()` function (see previous) will be called at import time.
 
-# Implement the interface
+## Implement the interface
 
 Generally speaking, you can follow the code in one of the existing plugins.
 
-# Lifecycle
+## Lifecycle
 
-## Init
+### Init
 
-Each plugin will have it's `Init` function called once as the pipline is constructed.
+Each plugin will have its `Init` function called once as the pipline is constructed.
 
 The context provided to this function should be saved, and used to terminate any long-running operations if necessary.
 
-## Per-round function
+### Per-round function
 
 Each plugin type has a function which is called once per round:
-* Importer: `GetBlock` called when a particular round is required. Generally this will be increasing over time.
-* Processor: `Process` called to process a round.
+
+* Importer: `GetBlock` is called when a particular round is required.
+* Processor: `Process` is called to process a round.
 * Exporter: `Receive` for consuming a round.
 
-## Close
+### Close
 
 Called during a graceful shutdown. We make every effort to call this function, but it is not guaranteed.
 
-## Hooks
+### Hooks
 
 There are special lifecycle hooks that can be registered on any plugin by implementing additional interfaces.
 
-### RoundRequestor
+#### RoundRequestor
 
 ```go
 // RoundRequestor is an optional interface. Plugins should implement it if
@@ -82,7 +88,7 @@ type RoundRequestor interface {
 }
 ```
 
-### Completed
+#### Completed
 
 When all processing has completed for a round, the `OnComplete` function is called on any plugin that implements it.
 
@@ -91,14 +97,15 @@ When all processing has completed for a round, the `OnComplete` function is call
 // finished. It can be used for things like finalizing state.
 type Completed interface {
 	// OnComplete will be called by the Conduit framework when the pipeline
-	// finishes processing a round.
+	// finishes processing a round.	
 	OnComplete(input data.BlockData) error
 }
 ```
 
-### PluginMetrics
+#### PluginMetrics
 
-After the pipeline has been initialized, and before it has been started, plugins may provide prometheus metric handlers. The subsystem is a configurable value that should be passed into the Prometheus metric constructors.
+After the pipeline has been initialized, and before it has been started, plugins may provide prometheus metric handlers.
+The subsystem is a configurable value that should be passed into the Prometheus metric constructors.
 The ProvideMetrics function will only be called once.
 
 ```go

docs/GettingStarted.md

@@ -1,6 +1,5 @@
 # Getting Started
 
-
 ## Installation
 
 ### Download
@@ -13,7 +12,9 @@ The latest `conduit` binary can be downloaded from the [GitHub releases page](ht
 2. Run `make conduit`.
 3. The binary is created at `cmd/conduit/conduit`.
 
-## Getting Started
+## Configuration and Plugins
+
+### Configuration File
 
 Conduit requires a configuration file to set up and run a data pipeline. To generate an initial skeleton for a conduit
 config file, you can run `./conduit init -d data`. This will set up a sample data directory with a config located at
@@ -24,17 +25,21 @@ You can find a valid config file in [Configuration.md](Configuration.md) or via
 
 Once you have a valid config file in a directory, `config_directory`, launch conduit with `./conduit -d config_directory`.
 
-# Configuration and Plugins
+### Plugins
+
 Conduit comes with an initial set of plugins available for use in pipelines. For more information on the possible
 plugins and how to include these plugins in your pipeline's configuration file see [Configuration.md](Configuration.md).
 
-# Tutorials
+## Tutorials
+
+### Migrate from the Legacy Indexer Architecture to a Conduit-backed Indexer
 
-## Migrate from the Legacy Indexer Architecture to a Conduit-backed Indexer
 [How to migrate from a legacy Indexer architecture to a Conduit-backed Indexer deployment. .](./tutorials/IndexerMigration.md)
 
-## Set up Conduit for the Indexer API
+### Set up Conduit for the Indexer API
+
 [How to configure algod, PostgreSQL and Conduit as an Indexer API backend.](./tutorials/IndexerWriter.md)
 
-## Writing Block Data to the Filesystem
+### Writing Block Data to the Filesystem
+
 [Use the file exporter to write data to files.](./tutorials/WritingBlocksToFile.md)