init: Option to write config file to stdout.

cmd/conduit/internal/initialize/init.go

@@ -2,7 +2,9 @@ package initialize
 
 import (
 	_ "embed"
+	"errors"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
@@ -21,6 +23,8 @@ var InitCommand = makeInitCmd()
 
 const defaultDataDirectory = "data"
 
+var errStdoutAndPath = errors.New("do not provide a path and toStdout")
+
 //go:embed conduit.yml.example
 var sampleConfig string
 
@@ -41,25 +45,34 @@ func formatArrayObject(obj string) string {
 
 }
 
-func runConduitInit(path string, importerFlag string, processorsFlag []string, exporterFlag string) error {
-	var location string
-	if path == "" {
-		path = defaultDataDirectory
-		location = "in the current working directory"
-	} else {
-		location = fmt.Sprintf("at '%s'", path)
+func runConduitInit(path string, configWriter io.Writer, importerFlag string, processorsFlag []string, exporterFlag string) error {
+	if configWriter != nil && path != "" {
+		return errStdoutAndPath
 	}
 
-	if err := os.MkdirAll(path, 0755); err != nil {
-		return err
-	}
+	var location string
 
-	configFilePath := filepath.Join(path, conduit.DefaultConfigName)
-	f, err := os.Create(configFilePath)
-	if err != nil {
-		return fmt.Errorf("runConduitInit(): failed to create %s", configFilePath)
+	// to get here, the path must be initialized
+	if configWriter == nil {
+		if path == "" && configWriter == nil {
+			path = defaultDataDirectory
+			location = "in the current working directory"
+		} else {
+			location = fmt.Sprintf("at '%s'", path)
+		}
+
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return err
+		}
+
+		configFilePath := filepath.Join(path, conduit.DefaultConfigName)
+		f, err := os.Create(configFilePath)
+		if err != nil {
+			return fmt.Errorf("runConduitInit(): failed to create %s", configFilePath)
+		}
+		defer f.Close()
+		configWriter = f
 	}
-	defer f.Close()
 
 	var importer string
 	if importerFlag == "" {
@@ -106,19 +119,21 @@ func runConduitInit(path string, importerFlag string, processorsFlag []string, e
 
 	config := fmt.Sprintf(sampleConfig, importer, processors, exporter)
 
-	_, _ = f.WriteString(config)
-
+	_, err := configWriter.Write([]byte(config))
 	if err != nil {
 		return fmt.Errorf("runConduitInit(): failed to write sample config: %w", err)
 	}
 
-	fmt.Printf("A data directory has been created %s.\n", location)
-	fmt.Printf("\nBefore it can be used, the config file needs to be updated with\n")
-	fmt.Printf("values for the selected import, export and processor modules. For example,\n")
-	fmt.Printf("if the default algod importer was used, set the address/token and the block-dir\n")
-	fmt.Printf("path where Conduit should write the block files.\n")
-	fmt.Printf("\nOnce the config file is updated, start Conduit with:\n")
-	fmt.Printf("  ./conduit -d %s\n", path)
+	// If a data dir is created, print some additional help.
+	if path != "" {
+		fmt.Printf("A data directory has been created %s.\n", location)
+		fmt.Printf("\nBefore it can be used, the config file needs to be updated with\n")
+		fmt.Printf("values for the selected import, export and processor modules. For example,\n")
+		fmt.Printf("if the default algod importer was used, set the address/token and the block-dir\n")
+		fmt.Printf("path where Conduit should write the block files.\n")
+		fmt.Printf("\nOnce the config file is updated, start Conduit with:\n")
+		fmt.Printf("  ./conduit -d %s\n", path)
+	}
 	return nil
 }
 
@@ -131,24 +146,30 @@ func makeInitCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "init",
 		Short: "initializes a Conduit data directory",
-		Long: `Initializes a Conduit data directory and conduit.yml file. By default
+		Long: `Initializes a conduit.yml file and writes it to stdout. By default
 the config file uses an algod importer in follower mode and a block
 file writer exporter. The plugin templates can be changed using the
 different options.
 
 Once initialized the conduit.yml file needs to be modified. Refer to the file
 comments for details.
 
+If the 'data' option is used, the file will be written to that
+directory and additional help is written to stdout.
+
 Once configured, launch conduit with './conduit -d /path/to/data'.`,
 		Example: "conduit init  -d /path/to/data -i importer -p processor1,processor2 -e exporter",
 		Args:    cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return runConduitInit(data, importer, processors, exporter)
+			if data == "" {
+				return runConduitInit("", os.Stdout, importer, processors, exporter)
+			}
+			return runConduitInit(data, nil, importer, processors, exporter)
 		},
 		SilenceUsage: true,
 	}
 
-	cmd.Flags().StringVarP(&data, "data", "d", "", "Full path to new data directory. If not set, a directory named 'data' will be created in the current directory.")
+	cmd.Flags().StringVarP(&data, "data", "d", "", "Full path to new data directory to initialize. If not set, the conduit configuration YAML is written to stdout.")
 	cmd.Flags().StringVarP(&importer, "importer", "i", "", "data importer name.")
 	cmd.Flags().StringSliceVarP(&processors, "processors", "p", []string{}, "comma-separated list of processors.")
 	cmd.Flags().StringVarP(&exporter, "exporter", "e", "", "data exporter name.")

cmd/conduit/internal/initialize/init_test.go

@@ -1,10 +1,12 @@
 package initialize
 
 import (
+	"bytes"
 	_ "embed"
 	"fmt"
 	"gopkg.in/yaml.v3"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -24,10 +26,7 @@ var defaultYml string
 
 // TestInitDataDirectory tests the initialization of the data directory
 func TestInitDataDirectory(t *testing.T) {
-	verifyFile := func(file string, importer string, exporter string, processors []string) {
-		require.FileExists(t, file)
-		data, err := os.ReadFile(file)
-		require.NoError(t, err)
+	verifyYaml := func(data []byte, importer string, exporter string, processors []string) {
 		var cfg pipeline.Config
 		require.NoError(t, yaml.Unmarshal(data, &cfg))
 		assert.Equal(t, importer, cfg.Importer.Name)
@@ -37,22 +36,39 @@ func TestInitDataDirectory(t *testing.T) {
 			assert.Equal(t, processors[i], cfg.Processors[i].Name)
 		}
 	}
+	verifyFile := func(file string, importer string, exporter string, processors []string) {
+		require.FileExists(t, file)
+		data, err := os.ReadFile(file)
+		require.NoError(t, err)
+		verifyYaml(data, importer, exporter, processors)
+	}
 
 	// Defaults
 	dataDirectory := t.TempDir()
-	err := runConduitInit(dataDirectory, "", []string{}, "")
+	err := runConduitInit(dataDirectory, nil, "", []string{}, "")
 	require.NoError(t, err)
 	verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), algodimporter.PluginName, filewriter.PluginName, nil)
 
 	// Explicit defaults
 	dataDirectory = t.TempDir()
-	err = runConduitInit(dataDirectory, algodimporter.PluginName, []string{noopProcessor.PluginName}, filewriter.PluginName)
+	err = runConduitInit(dataDirectory, nil, algodimporter.PluginName, []string{noopProcessor.PluginName}, filewriter.PluginName)
 	require.NoError(t, err)
 	verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), algodimporter.PluginName, filewriter.PluginName, []string{noopProcessor.PluginName})
 
 	// Different
 	dataDirectory = t.TempDir()
-	err = runConduitInit(dataDirectory, fileimporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}, noopExporter.PluginName)
+	err = runConduitInit(dataDirectory, nil, fileimporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}, noopExporter.PluginName)
 	require.NoError(t, err)
 	verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), fileimporter.PluginName, noopExporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName})
+
+	// Stdout
+	var buf bytes.Buffer
+	err = runConduitInit("", &buf, fileimporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}, noopExporter.PluginName)
+	require.NoError(t, err)
+	verifyYaml(buf.Bytes(), fileimporter.PluginName, noopExporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName})
+}
+
+func TestBadInput(t *testing.T) {
+	err := runConduitInit("some-path", &strings.Builder{}, "", []string{}, "")
+	require.ErrorIs(t, err, errStdoutAndPath)
 }

docs/GettingStarted.md

@@ -21,7 +21,7 @@ that work is complete users should be able to use `go install` to install binari
 ## Getting Started
 
 Conduit requires a configuration file to set up and run a data pipeline. To generate an initial skeleton for a conduit
-config file, you can run `./conduit init`. This will set up a sample data directory with a config located at
+config file, you can run `./conduit init -d data`. This will set up a sample data directory with a config located at
 `data/conduit.yml`.
 
 You will need to manually edit the data in the config file, filling in a valid configuration for conduit to run.  

docs/tutorials/WritingBlocksToFile.md

@@ -71,7 +71,7 @@ config:
 
 Let's start assembling a configuration file which describes our conduit pipeline. For that we'll run 
 ```
-conduit init
+conduit init -d data
 ```
 This will create a configuration directory if we don't provide one to it, and write a skeleton config file
 there which we will use as the starting point for our pipeline. Here is the config file which the `init` subcommand has