Merge branch 'master' into core_v21

Makefile

@@ -19,7 +19,7 @@ XDRS = $(DOWNLOADABLE_XDRS) xdr/Stellar-lighthorizon.x \
 
 
 XDRGEN_COMMIT=e2cac557162d99b12ae73b846cf3d5bfe16636de
-XDR_COMMIT=1a04392432dacc0092caaeae22a600ea1af3c6a5
+XDR_COMMIT=70180d5e8d9caee9e8645ed8a38c36a8cf403cd9
 
 .PHONY: xdr xdr-clean xdr-update
 

exp/services/ledgerexporter/Makefile

@@ -2,13 +2,13 @@ SUDO := $(shell docker version >/dev/null 2>&1 || echo "sudo")
 
 # https://github.com/opencontainers/image-spec/blob/master/annotations.md
 BUILD_DATE := $(shell date -u +%FT%TZ)
-VERSION ?= $(shell git rev-parse --short HEAD)
+VERSION ?= 1.0.0-$(shell git rev-parse --short HEAD)
 DOCKER_IMAGE := stellar/ledger-exporter
 
 docker-build:
 	cd ../../../ && \
 	$(SUDO) docker build --platform linux/amd64 --pull --label org.opencontainers.image.created="$(BUILD_DATE)" \
-	--build-arg VERSION=$(VERSION) \
+    --build-arg GOFLAGS="-ldflags=-X=github.com/stellar/go/exp/services/ledgerexporter/internal.version=$(VERSION)" \
 $(if $(STELLAR_CORE_VERSION), --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION)) \
 	-f exp/services/ledgerexporter/docker/Dockerfile \
 	-t $(DOCKER_IMAGE):$(VERSION) \

exp/services/ledgerexporter/docker/Dockerfile

@@ -9,6 +9,7 @@ RUN go mod download
 
 COPY . ./
 
+ARG GOFLAGS
 RUN go install github.com/stellar/go/exp/services/ledgerexporter
 
 FROM ubuntu:22.04

exp/services/ledgerexporter/internal/app.go

@@ -37,7 +37,8 @@ const (
 )
 
 var (
-	logger = log.New().WithField("service", "ledger-exporter")
+	logger  = log.New().WithField("service", "ledger-exporter")
+	version = "develop"
 )
 
 func NewDataAlreadyExportedError(Start uint32, End uint32) *DataAlreadyExportedError {
@@ -95,13 +96,15 @@ func (a *App) init(ctx context.Context) error {
 	var err error
 	var archive historyarchive.ArchiveInterface
 
+	logger.Infof("Starting Ledger Exporter with version %s", version)
+
 	registry := prometheus.NewRegistry()
 	registry.MustRegister(
 		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{Namespace: "ledger_exporter"}),
 		collectors.NewGoCollector(),
 	)
 
-	if a.config, err = NewConfig(ctx, a.flags); err != nil {
+	if a.config, err = NewConfig(a.flags); err != nil {
 		return errors.Wrap(err, "Could not load configuration")
 	}
 	if archive, err = datastore.CreateHistoryArchiveFromNetworkName(ctx, a.config.Network); err != nil {
@@ -126,7 +129,7 @@ func (a *App) init(ctx context.Context) error {
 	}
 
 	queue := NewUploadQueue(uploadQueueCapacity, registry)
-	if a.exportManager, err = NewExportManager(a.config.LedgerBatchConfig, a.ledgerBackend, queue, registry); err != nil {
+	if a.exportManager, err = NewExportManager(a.config, a.ledgerBackend, queue, registry); err != nil {
 		return err
 	}
 	a.uploader = NewUploader(a.dataStore, queue, registry)
@@ -256,7 +259,7 @@ func (a *App) Run() {
 // newLedgerBackend Creates and initializes captive core ledger backend
 // Currently, only supports captive-core as ledger backend
 func newLedgerBackend(config *Config, prometheusRegistry *prometheus.Registry) (ledgerbackend.LedgerBackend, error) {
-	captiveConfig, err := config.GenerateCaptiveCoreConfig()
+	captiveConfig, err := config.generateCaptiveCoreConfig()
 	if err != nil {
 		return nil, err
 	}

exp/services/ledgerexporter/internal/config.go

@@ -3,7 +3,9 @@ package ledgerexporter
 import (
 	"context"
 	_ "embed"
+	"fmt"
 	"os/exec"
+	"strings"
 
 	"github.com/stellar/go/historyarchive"
 	"github.com/stellar/go/ingest/ledgerbackend"
@@ -45,15 +47,16 @@ type Config struct {
 	StartLedger uint32
 	EndLedger   uint32
 	Resume      bool
+
+	CoreVersion string
 }
 
 // This will generate the config based on commandline flags and toml
 //
-// ctx                   - the caller context
 // flags                 - command line flags
 //
 // return                - *Config or an error if any range validation failed.
-func NewConfig(ctx context.Context, flags Flags) (*Config, error) {
+func NewConfig(flags Flags) (*Config, error) {
 	config := &Config{}
 
 	config.StartLedger = uint32(flags.StartLedger)
@@ -100,17 +103,21 @@ func (config *Config) ValidateAndSetLedgerRange(ctx context.Context, archive his
 	return nil
 }
 
-func (config *Config) GenerateCaptiveCoreConfig() (ledgerbackend.CaptiveCoreConfig, error) {
+func (config *Config) generateCaptiveCoreConfig() (ledgerbackend.CaptiveCoreConfig, error) {
 	coreConfig := &config.StellarCoreConfig
 
 	// Look for stellar-core binary in $PATH, if not supplied
-	if coreConfig.StellarCoreBinaryPath == "" {
+	if config.StellarCoreConfig.StellarCoreBinaryPath == "" {
 		var err error
-		if coreConfig.StellarCoreBinaryPath, err = exec.LookPath("stellar-core"); err != nil {
+		if config.StellarCoreConfig.StellarCoreBinaryPath, err = exec.LookPath("stellar-core"); err != nil {
 			return ledgerbackend.CaptiveCoreConfig{}, errors.Wrap(err, "Failed to find stellar-core binary")
 		}
 	}
 
+	if err := config.setCoreVersionInfo(); err != nil {
+		return ledgerbackend.CaptiveCoreConfig{}, fmt.Errorf("failed to set stellar-core version info: %w", err)
+	}
+
 	var captiveCoreConfig []byte
 	// Default network config
 	switch config.Network {
@@ -151,6 +158,30 @@ func (config *Config) GenerateCaptiveCoreConfig() (ledgerbackend.CaptiveCoreConf
 	}, nil
 }
 
+// By default, it points to exec.Command, overridden for testing purpose
+var execCommand = exec.Command
+
+// Executes the "stellar-core version" command and parses its output to extract
+// the core version
+// The output of the "version" command is expected to be a multi-line string where the
+// first line is the core version in format "vX.Y.Z-*".
+func (c *Config) setCoreVersionInfo() (err error) {
+	versionCmd := execCommand(c.StellarCoreConfig.StellarCoreBinaryPath, "version")
+	versionOutput, err := versionCmd.Output()
+	if err != nil {
+		return fmt.Errorf("failed to execute stellar-core version command: %w", err)
+	}
+
+	// Split the output into lines
+	rows := strings.Split(string(versionOutput), "\n")
+	if len(rows) == 0 || len(rows[0]) == 0 {
+		return fmt.Errorf("stellar-core version not found")
+	}
+	c.CoreVersion = rows[0]
+	logger.Infof("stellar-core version: %s", c.CoreVersion)
+	return nil
+}
+
 func (config *Config) processToml(tomlPath string) error {
 	// Load config TOML file
 	cfg, err := toml.LoadFile(tomlPath)
@@ -182,10 +213,7 @@ func (config *Config) adjustLedgerRange() {
 
 	// Align the end ledger (for bounded cases) to the nearest "LedgersPerFile" boundary.
 	if config.EndLedger != 0 {
-		// Add an extra batch only if "LedgersPerFile" is greater than 1 and the end ledger doesn't fall on the boundary.
-		if config.LedgerBatchConfig.LedgersPerFile > 1 && config.EndLedger%config.LedgerBatchConfig.LedgersPerFile != 0 {
-			config.EndLedger = (config.EndLedger/config.LedgerBatchConfig.LedgersPerFile + 1) * config.LedgerBatchConfig.LedgersPerFile
-		}
+		config.EndLedger = config.LedgerBatchConfig.GetSequenceNumberEndBoundary(config.EndLedger)
 	}
 
 	logger.Infof("Computed effective export boundary ledger range: start=%d, end=%d", config.StartLedger, config.EndLedger)

exp/services/ledgerexporter/internal/config_test.go

@@ -3,10 +3,14 @@ package ledgerexporter
 import (
 	"context"
 	"fmt"
+	"os"
+	"os/exec"
 	"testing"
 
-	"github.com/stellar/go/historyarchive"
 	"github.com/stretchr/testify/require"
+
+	"github.com/stellar/go/historyarchive"
+	"github.com/stellar/go/support/errors"
 )
 
 func TestNewConfigResumeEnabled(t *testing.T) {
@@ -15,8 +19,7 @@ func TestNewConfigResumeEnabled(t *testing.T) {
 	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.On("GetRootHAS").Return(historyarchive.HistoryArchiveState{CurrentLedger: 5}, nil).Once()
 
-	config, err := NewConfig(ctx,
-		Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/test.toml", Resume: true})
+	config, err := NewConfig(Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/test.toml", Resume: true})
 	config.ValidateAndSetLedgerRange(ctx, mockArchive)
 	require.NoError(t, err)
 	require.Equal(t, config.DataStoreConfig.Type, "ABC")
@@ -29,23 +32,19 @@ func TestNewConfigResumeEnabled(t *testing.T) {
 }
 
 func TestNewConfigResumeDisabled(t *testing.T) {
-	ctx := context.Background()
 
 	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.On("GetRootHAS").Return(historyarchive.HistoryArchiveState{CurrentLedger: 5}, nil).Once()
 
 	// resume disabled by default
-	config, err := NewConfig(ctx,
-		Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/test.toml"})
+	config, err := NewConfig(Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/test.toml"})
 	require.NoError(t, err)
 	require.False(t, config.Resume)
 }
 
 func TestInvalidTomlConfig(t *testing.T) {
-	ctx := context.Background()
 
-	_, err := NewConfig(ctx,
-		Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/no_network.toml", Resume: true})
+	_, err := NewConfig(Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/no_network.toml", Resume: true})
 	require.ErrorContains(t, err, "Invalid TOML config")
 }
 
@@ -110,8 +109,7 @@ func TestValidateStartAndEndLedger(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			config, err := NewConfig(ctx,
-				Flags{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml"})
+			config, err := NewConfig(Flags{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml"})
 			require.NoError(t, err)
 			err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
 			if tt.errMsg != "" {
@@ -154,31 +152,31 @@ func TestAdjustedLedgerRangeBoundedMode(t *testing.T) {
 			start:         0,
 			end:           1,
 			expectedStart: 2,
-			expectedEnd:   10,
+			expectedEnd:   9,
 		},
 		{
 			name:          "Round down start ledger and round up end ledger, 15 ledgers per file ",
 			configFile:    "test/15perfile.toml",
 			start:         4,
 			end:           10,
 			expectedStart: 2,
-			expectedEnd:   15,
+			expectedEnd:   14,
 		},
 		{
 			name:          "Round down start ledger and round up end ledger, 64 ledgers per file ",
 			configFile:    "test/64perfile.toml",
 			start:         400,
 			end:           500,
 			expectedStart: 384,
-			expectedEnd:   512,
+			expectedEnd:   511,
 		},
 		{
 			name:          "No change, 64 ledger per file",
 			configFile:    "test/64perfile.toml",
 			start:         64,
 			end:           128,
 			expectedStart: 64,
-			expectedEnd:   128,
+			expectedEnd:   191,
 		},
 	}
 
@@ -188,8 +186,7 @@ func TestAdjustedLedgerRangeBoundedMode(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			config, err := NewConfig(ctx,
-				Flags{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile})
+			config, err := NewConfig(Flags{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile})
 			require.NoError(t, err)
 			err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
 			require.NoError(t, err)
@@ -257,8 +254,7 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			config, err := NewConfig(ctx,
-				Flags{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile})
+			config, err := NewConfig(Flags{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile})
 			require.NoError(t, err)
 			err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
 			require.NoError(t, err)
@@ -267,3 +263,69 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
 		})
 	}
 }
+
+var cmdOut = ""
+
+func fakeExecCommand(command string, args ...string) *exec.Cmd {
+	cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...)
+	cmd := exec.Command(os.Args[0], cs...)
+	cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+cmdOut)
+	return cmd
+}
+
+func TestExecCmdHelperProcess(t *testing.T) {
+	if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" {
+		return
+	}
+	fmt.Fprintf(os.Stdout, os.Getenv("CMD_OUT"))
+	os.Exit(0)
+}
+
+func TestSetCoreVersionInfo(t *testing.T) {
+	tests := []struct {
+		name            string
+		commandOutput   string
+		expectedError   error
+		expectedCoreVer string
+	}{
+		{
+			name: "version found",
+			commandOutput: "v20.2.0-2-g6e73c0a88\n" +
+				"rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" +
+				"soroban-env-host: \n" +
+				"    curr:\n" +
+				"       package version: 20.2.0\n" +
+				"       git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" +
+				"       ledger protocol version: 20\n" +
+				"       pre-release version: 0\n" +
+				"       rs-stellar-xdr:\n" +
+				"           package version: 20.1.0\n" +
+				"           git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" +
+				"           base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n",
+			expectedError:   nil,
+			expectedCoreVer: "v20.2.0-2-g6e73c0a88",
+		},
+		{
+			name:            "core version not found",
+			commandOutput:   "",
+			expectedError:   errors.New("stellar-core version not found"),
+			expectedCoreVer: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config := Config{}
+
+			cmdOut = tt.commandOutput
+			execCommand = fakeExecCommand
+			err := config.setCoreVersionInfo()
+
+			if tt.expectedError != nil {
+				require.EqualError(t, err, tt.expectedError.Error())
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tt.expectedCoreVer, config.CoreVersion)
+			}
+		})
+	}
+}