diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index d07133c04a..1778b7a929 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -87,7 +87,8 @@ jobs: # Workaround for https://github.com/actions/virtual-environments/issues/5245, # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul # message about why the installation fails) - sudo apt-get remove -y libc++1-10 libc++abi1-10 || true + sudo apt list --installed | grep libc++ + sudo apt-get remove -y libc++1-* libc++abi1-* || true sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add - sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list' @@ -113,7 +114,7 @@ jobs: key: horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./services/horizon/internal/integration/**') }}-${{ env.PROTOCOL_20_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_19_CORE_DOCKER_IMG }} - if: ${{ steps.horizon_binary_tests_hash.outputs.cache-hit != 'true' }} - run: go test -race -timeout 35m -v ./services/horizon/internal/integration/... + run: go test -race -timeout 45m -v ./services/horizon/internal/integration/... verify-range: name: Test (and push) verify-range image diff --git a/go.mod b/go.mod index a86b185392..bddeda2c03 100644 --- a/go.mod +++ b/go.mod @@ -115,15 +115,14 @@ require ( golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect - golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/text v0.3.8 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 - golang.org/x/tools v0.1.10 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/tools v0.1.12 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84 // indirect google.golang.org/grpc v1.38.0 // indirect diff --git a/go.sum b/go.sum index 3205cd3894..f126f5ff17 100644 --- a/go.sum +++ b/go.sum @@ -458,8 +458,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -501,8 +501,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -581,8 +582,9 @@ golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -594,8 +596,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -653,12 +656,11 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index 68145b517e..600e11d329 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -3,7 +3,7 @@ package ledgerbackend import ( "bytes" "fmt" - "io/ioutil" + "os" "os/exec" "regexp" "strconv" @@ -331,13 +331,19 @@ type CaptiveCoreTomlParams struct { // NewCaptiveCoreTomlFromFile constructs a new CaptiveCoreToml instance by merging configuration // from the toml file located at `configPath` and the configuration provided by `params`. func NewCaptiveCoreTomlFromFile(configPath string, params CaptiveCoreTomlParams) (*CaptiveCoreToml, error) { - var captiveCoreToml CaptiveCoreToml - data, err := ioutil.ReadFile(configPath) + data, err := os.ReadFile(configPath) if err != nil { return nil, errors.Wrap(err, "could not load toml path") } + return NewCaptiveCoreTomlFromData(data, params) +} + +// NewCaptiveCoreTomlFromData constructs a new CaptiveCoreToml instance by merging configuration +// from the toml data and the configuration provided by `params`. +func NewCaptiveCoreTomlFromData(data []byte, params CaptiveCoreTomlParams) (*CaptiveCoreToml, error) { + var captiveCoreToml CaptiveCoreToml - if err = captiveCoreToml.unmarshal(data, params.Strict); err != nil { + if err := captiveCoreToml.unmarshal(data, params.Strict); err != nil { return nil, errors.Wrap(err, "could not unmarshal captive core toml") } // disallow setting BUCKET_DIR_PATH through a file since it can cause multiple @@ -346,14 +352,13 @@ func NewCaptiveCoreTomlFromFile(configPath string, params CaptiveCoreTomlParams) return nil, errors.New("could not unmarshal captive core toml: setting BUCKET_DIR_PATH is disallowed for Captive Core, use CAPTIVE_CORE_STORAGE_PATH instead") } - if err = captiveCoreToml.validate(params); err != nil { + if err := captiveCoreToml.validate(params); err != nil { return nil, errors.Wrap(err, "invalid captive core toml") } if len(captiveCoreToml.HistoryEntries) > 0 { log.Warnf( - "Configuring captive core with history archive from %s instead of %v", - configPath, + "Configuring captive core with history archive from %s", params.HistoryArchiveURLs, ) } diff --git a/network/main.go b/network/main.go index 8c2524bc63..059a264b81 100644 --- a/network/main.go +++ b/network/main.go @@ -21,6 +21,18 @@ const ( FutureNetworkPassphrase = "Test SDF Future Network ; October 2022" ) +var ( + // PublicNetworkhistoryArchiveURLs is a list of history archive URLs for stellar 'pubnet' + PublicNetworkhistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-live/core_live_001/", + "https://history.stellar.org/prd/core-live/core_live_002/", + "https://history.stellar.org/prd/core-live/core_live_003/"} + + // TestNetworkhistoryArchiveURLs is a list of history archive URLs for stellar 'testnet' + TestNetworkhistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-testnet/core_testnet_001/", + "https://history.stellar.org/prd/core-testnet/core_testnet_002/", + "https://history.stellar.org/prd/core-testnet/core_testnet_003"} +) + // ID returns the network ID derived from the provided passphrase. This value // also happens to be the raw (i.e. not strkey encoded) secret key for the root // account of the network. diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index a6a471f153..9ade9c77c2 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -3,13 +3,11 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). + ## Unreleased -### Changes -- Update XDR definitions for soroban usage ([4576](https://github.com/stellar/go/pull/4576)) -- Include InvokeHostFunction Details on Operation API resources ([4608](https://github.com/stellar/go/pull/4608)) - The command line flag --remote-captive-core-url has been removed as remote captive core functionality is now deprecated ([4940](https://github.com/stellar/go/pull/4940)). - +- Added new command-line flag --network to specify the Stellar network (pubnet or testnet), aiming at simplifying the configuration process by automatically configuring the following parameters based on the chosen network: --history-archive-urls, --network-passphrase, --captive-core-config-path. ([4949](https://github.com/stellar/go/pull/4949)). ## 2.26.0 ### Changes diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index c0fdaf7545..db9eccea30 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -121,7 +121,7 @@ var ingestVerifyRangeCmd = &cobra.Command{ } if ingestVerifyState && !mngr.IsCheckpoint(ingestVerifyTo) { - return fmt.Errorf("`--to` must be a checkpoint ledger when `--verify-state` is set.") + return fmt.Errorf("`--to` must be a checkpoint ledger when `--verify-state` is set") } ingestConfig := ingest.Config{ @@ -276,7 +276,7 @@ var ingestTriggerStateRebuildCmd = &cobra.Command{ return fmt.Errorf("cannot open Horizon DB: %v", err) } - historyQ := &history.Q{horizonSession} + historyQ := &history.Q{SessionInterface: horizonSession} if err := historyQ.UpdateIngestVersion(ctx, 0); err != nil { return fmt.Errorf("cannot trigger state rebuild: %v", err) } @@ -300,7 +300,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{ return fmt.Errorf("cannot open Horizon DB: %v", err) } - historyQ := &history.Q{horizonSession} + historyQ := &history.Q{SessionInterface: horizonSession} lastIngestedLedger, err := historyQ.GetLastLedgerIngestNonBlocking(ctx) if err != nil { @@ -372,7 +372,7 @@ var ingestBuildStateCmd = &cobra.Command{ return fmt.Errorf("cannot open Horizon DB: %v", err) } - historyQ := &history.Q{horizonSession} + historyQ := &history.Q{SessionInterface: horizonSession} lastIngestedLedger, err := historyQ.GetLastLedgerIngestNonBlocking(context.Background()) if err != nil { diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index a387a8e90f..af63675d0c 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -36,7 +36,7 @@ func (e ErrUsage) Error() string { return e.cmd.UsageString() } -// Indicates we want to exit with a specific error code without printing an error. +// ErrExitCode Indicates we want to exit with a specific error code without printing an error. type ErrExitCode int func (e ErrExitCode) Error() string { diff --git a/services/horizon/docker/captive-core-pubnet.cfg b/services/horizon/docker/captive-core-pubnet.cfg deleted file mode 100644 index 177de5d9d9..0000000000 --- a/services/horizon/docker/captive-core-pubnet.cfg +++ /dev/null @@ -1,192 +0,0 @@ -PEER_PORT=11725 - -FAILURE_SAFETY=1 - -[[HOME_DOMAINS]] -HOME_DOMAIN="stellar.org" -QUALITY="HIGH" - -[[HOME_DOMAINS]] -HOME_DOMAIN="satoshipay.io" -QUALITY="HIGH" - -[[HOME_DOMAINS]] -HOME_DOMAIN="lobstr.co" -QUALITY="HIGH" - -[[HOME_DOMAINS]] -HOME_DOMAIN="www.coinqvest.com" -QUALITY="HIGH" - -[[HOME_DOMAINS]] -HOME_DOMAIN="publicnode.org" -QUALITY="HIGH" - -[[HOME_DOMAINS]] -HOME_DOMAIN="stellar.blockdaemon.com" -QUALITY="HIGH" - -[[HOME_DOMAINS]] -HOME_DOMAIN = "www.franklintempleton.com" -QUALITY = "HIGH" - -[[VALIDATORS]] -NAME="sdf_1" -HOME_DOMAIN="stellar.org" -PUBLIC_KEY="GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH" -ADDRESS="core-live-a.stellar.org:11625" -HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_001/{0} -o {1}" - -[[VALIDATORS]] -NAME="sdf_2" -HOME_DOMAIN="stellar.org" -PUBLIC_KEY="GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK" -ADDRESS="core-live-b.stellar.org:11625" -HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_002/{0} -o {1}" - -[[VALIDATORS]] -NAME="sdf_3" -HOME_DOMAIN="stellar.org" -PUBLIC_KEY="GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ" -ADDRESS="core-live-c.stellar.org:11625" -HISTORY="curl -sf https://history.stellar.org/prd/core-live/core_live_003/{0} -o {1}" - -[[VALIDATORS]] -NAME="satoshipay_singapore" -HOME_DOMAIN="satoshipay.io" -PUBLIC_KEY="GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT" -ADDRESS="stellar-sg-sin.satoshipay.io:11625" -HISTORY="curl -sf https://stellar-history-sg-sin.satoshipay.io/{0} -o {1}" - -[[VALIDATORS]] -NAME="satoshipay_iowa" -HOME_DOMAIN="satoshipay.io" -PUBLIC_KEY="GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY" -ADDRESS="stellar-us-iowa.satoshipay.io:11625" -HISTORY="curl -sf https://stellar-history-us-iowa.satoshipay.io/{0} -o {1}" - -[[VALIDATORS]] -NAME="satoshipay_frankfurt" -HOME_DOMAIN="satoshipay.io" -PUBLIC_KEY="GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE" -ADDRESS="stellar-de-fra.satoshipay.io:11625" -HISTORY="curl -sf https://stellar-history-de-fra.satoshipay.io/{0} -o {1}" - -[[VALIDATORS]] -NAME="lobstr_1_europe" -HOME_DOMAIN="lobstr.co" -PUBLIC_KEY="GCFONE23AB7Y6C5YZOMKUKGETPIAJA4QOYLS5VNS4JHBGKRZCPYHDLW7" -ADDRESS="v1.stellar.lobstr.co:11625" -HISTORY="curl -sf https://stellar-archive-1-lobstr.s3.amazonaws.com/{0} -o {1}" - -[[VALIDATORS]] -NAME="lobstr_2_europe" -HOME_DOMAIN="lobstr.co" -PUBLIC_KEY="GDXQB3OMMQ6MGG43PWFBZWBFKBBDUZIVSUDAZZTRAWQZKES2CDSE5HKJ" -ADDRESS="v2.stellar.lobstr.co:11625" -HISTORY="curl -sf https://stellar-archive-2-lobstr.s3.amazonaws.com/{0} -o {1}" - -[[VALIDATORS]] -NAME="lobstr_3_north_america" -HOME_DOMAIN="lobstr.co" -PUBLIC_KEY="GD5QWEVV4GZZTQP46BRXV5CUMMMLP4JTGFD7FWYJJWRL54CELY6JGQ63" -ADDRESS="v3.stellar.lobstr.co:11625" -HISTORY="curl -sf https://stellar-archive-3-lobstr.s3.amazonaws.com/{0} -o {1}" - -[[VALIDATORS]] -NAME="lobstr_4_asia" -HOME_DOMAIN="lobstr.co" -PUBLIC_KEY="GA7TEPCBDQKI7JQLQ34ZURRMK44DVYCIGVXQQWNSWAEQR6KB4FMCBT7J" -ADDRESS="v4.stellar.lobstr.co:11625" -HISTORY="curl -sf https://stellar-archive-4-lobstr.s3.amazonaws.com/{0} -o {1}" - -[[VALIDATORS]] -NAME="lobstr_5_australia" -HOME_DOMAIN="lobstr.co" -PUBLIC_KEY="GA5STBMV6QDXFDGD62MEHLLHZTPDI77U3PFOD2SELU5RJDHQWBR5NNK7" -ADDRESS="v5.stellar.lobstr.co:11625" -HISTORY="curl -sf https://stellar-archive-5-lobstr.s3.amazonaws.com/{0} -o {1}" - -[[VALIDATORS]] -NAME="coinqvest_hong_kong" -HOME_DOMAIN="www.coinqvest.com" -PUBLIC_KEY="GAZ437J46SCFPZEDLVGDMKZPLFO77XJ4QVAURSJVRZK2T5S7XUFHXI2Z" -ADDRESS="hongkong.stellar.coinqvest.com:11625" -HISTORY="curl -sf https://hongkong.stellar.coinqvest.com/history/{0} -o {1}" - -[[VALIDATORS]] -NAME="coinqvest_germany" -HOME_DOMAIN="www.coinqvest.com" -PUBLIC_KEY="GD6SZQV3WEJUH352NTVLKEV2JM2RH266VPEM7EH5QLLI7ZZAALMLNUVN" -ADDRESS="germany.stellar.coinqvest.com:11625" -HISTORY="curl -sf https://germany.stellar.coinqvest.com/history/{0} -o {1}" - -[[VALIDATORS]] -NAME="coinqvest_finland" -HOME_DOMAIN="www.coinqvest.com" -PUBLIC_KEY="GADLA6BJK6VK33EM2IDQM37L5KGVCY5MSHSHVJA4SCNGNUIEOTCR6J5T" -ADDRESS="finland.stellar.coinqvest.com:11625" -HISTORY="curl -sf https://finland.stellar.coinqvest.com/history/{0} -o {1}" - -[[VALIDATORS]] -NAME="bootes" -HOME_DOMAIN="publicnode.org" -PUBLIC_KEY="GCVJ4Z6TI6Z2SOGENSPXDQ2U4RKH3CNQKYUHNSSPYFPNWTLGS6EBH7I2" -ADDRESS="bootes.publicnode.org" -HISTORY="curl -sf https://bootes-history.publicnode.org/{0} -o {1}" - -[[VALIDATORS]] -NAME="hercules" -HOME_DOMAIN="publicnode.org" -PUBLIC_KEY="GBLJNN3AVZZPG2FYAYTYQKECNWTQYYUUY2KVFN2OUKZKBULXIXBZ4FCT" -ADDRESS="hercules.publicnode.org" -HISTORY="curl -sf https://hercules-history.publicnode.org/{0} -o {1}" - -[[VALIDATORS]] -NAME="lyra" -HOME_DOMAIN="publicnode.org" -PUBLIC_KEY="GCIXVKNFPKWVMKJKVK2V4NK7D4TC6W3BUMXSIJ365QUAXWBRPPJXIR2Z" -ADDRESS="lyra.publicnode.org" -HISTORY="curl -sf https://lyra-history.publicnode.org/{0} -o {1}" - -[[VALIDATORS]] -NAME="Blockdaemon_Validator_1" -HOME_DOMAIN="stellar.blockdaemon.com" -PUBLIC_KEY="GAAV2GCVFLNN522ORUYFV33E76VPC22E72S75AQ6MBR5V45Z5DWVPWEU" -ADDRESS="stellar-full-validator1.bdnodes.net" -HISTORY="curl -sf https://stellar-full-history1.bdnodes.net/{0} -o {1}" - -[[VALIDATORS]] -NAME="Blockdaemon_Validator_2" -HOME_DOMAIN="stellar.blockdaemon.com" -PUBLIC_KEY="GAVXB7SBJRYHSG6KSQHY74N7JAFRL4PFVZCNWW2ARI6ZEKNBJSMSKW7C" -ADDRESS="stellar-full-validator2.bdnodes.net" -HISTORY="curl -sf https://stellar-full-history2.bdnodes.net/{0} -o {1}" - -[[VALIDATORS]] -NAME="Blockdaemon_Validator_3" -HOME_DOMAIN="stellar.blockdaemon.com" -PUBLIC_KEY="GAYXZ4PZ7P6QOX7EBHPIZXNWY4KCOBYWJCA4WKWRKC7XIUS3UJPT6EZ4" -ADDRESS="stellar-full-validator3.bdnodes.net" -HISTORY="curl -sf https://stellar-full-history3.bdnodes.net/{0} -o {1}" - -[[VALIDATORS]] -NAME = "FT_SCV_1" -HOME_DOMAIN = "www.franklintempleton.com" -PUBLIC_KEY = "GARYGQ5F2IJEBCZJCBNPWNWVDOFK7IBOHLJKKSG2TMHDQKEEC6P4PE4V" -ADDRESS = "stellar1.franklintempleton.com:11625" -HISTORY = "curl -sf https://stellar-history-usw.franklintempleton.com/azuswshf401/{0} -o {1}" - -[[VALIDATORS]] -NAME = "FT_SCV_2" -HOME_DOMAIN = "www.franklintempleton.com" -PUBLIC_KEY = "GCMSM2VFZGRPTZKPH5OABHGH4F3AVS6XTNJXDGCZ3MKCOSUBH3FL6DOB" -ADDRESS = "stellar2.franklintempleton.com:11625" -HISTORY = "curl -sf https://stellar-history-usc.franklintempleton.com/azuscshf401/{0} -o {1}" - -[[VALIDATORS]] -NAME = "FT_SCV_3" -HOME_DOMAIN = "www.franklintempleton.com" -PUBLIC_KEY = "GA7DV63PBUUWNUFAF4GAZVXU2OZMYRATDLKTC7VTCG7AU4XUPN5VRX4A" -ADDRESS = "stellar3.franklintempleton.com:11625" -HISTORY = "curl -sf https://stellar-history-ins.franklintempleton.com/azinsshf401/{0} -o {1}" \ No newline at end of file diff --git a/services/horizon/docker/captive-core-testnet.cfg b/services/horizon/docker/captive-core-testnet.cfg deleted file mode 100644 index af327834d7..0000000000 --- a/services/horizon/docker/captive-core-testnet.cfg +++ /dev/null @@ -1,29 +0,0 @@ -PEER_PORT=11725 - -UNSAFE_QUORUM=true -FAILURE_SAFETY=1 - -[[HOME_DOMAINS]] -HOME_DOMAIN="testnet.stellar.org" -QUALITY="HIGH" - -[[VALIDATORS]] -NAME="sdf_testnet_1" -HOME_DOMAIN="testnet.stellar.org" -PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" -ADDRESS="core-testnet1.stellar.org" -HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_001/{0} -o {1}" - -[[VALIDATORS]] -NAME="sdf_testnet_2" -HOME_DOMAIN="testnet.stellar.org" -PUBLIC_KEY="GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP" -ADDRESS="core-testnet2.stellar.org" -HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_002/{0} -o {1}" - -[[VALIDATORS]] -NAME="sdf_testnet_3" -HOME_DOMAIN="testnet.stellar.org" -PUBLIC_KEY="GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z" -ADDRESS="core-testnet3.stellar.org" -HISTORY="curl -sf http://history.stellar.org/prd/core-testnet/core_testnet_003/{0} -o {1}" \ No newline at end of file diff --git a/services/horizon/docker/docker-compose.pubnet.yml b/services/horizon/docker/docker-compose.pubnet.yml index f6e380749d..169045d7d1 100644 --- a/services/horizon/docker/docker-compose.pubnet.yml +++ b/services/horizon/docker/docker-compose.pubnet.yml @@ -3,8 +3,4 @@ services: horizon: platform: linux/amd64 environment: - - HISTORY_ARCHIVE_URLS=https://history.stellar.org/prd/core-live/core_live_001 - - NETWORK_PASSPHRASE=Public Global Stellar Network ; September 2015 - - CAPTIVE_CORE_CONFIG_APPEND_PATH=/captive-core-pubnet.cfg - volumes: - - ./captive-core-pubnet.cfg:/captive-core-pubnet.cfg + - NETWORK=pubnet \ No newline at end of file diff --git a/services/horizon/docker/docker-compose.standalone.yml b/services/horizon/docker/docker-compose.standalone.yml index 43bd9ac9d7..636c0eaf3b 100644 --- a/services/horizon/docker/docker-compose.standalone.yml +++ b/services/horizon/docker/docker-compose.standalone.yml @@ -37,6 +37,7 @@ services: - HISTORY_ARCHIVE_URLS=http://host.docker.internal:1570 - NETWORK_PASSPHRASE=Standalone Network ; February 2017 - CAPTIVE_CORE_CONFIG_APPEND_PATH=/captive-core-standalone.cfg + - STELLAR_CORE_URL=http://host.docker.internal:11626 volumes: - ./captive-core-standalone.cfg:/captive-core-standalone.cfg diff --git a/services/horizon/docker/docker-compose.yml b/services/horizon/docker/docker-compose.yml index 40bced6677..af0b6721b6 100644 --- a/services/horizon/docker/docker-compose.yml +++ b/services/horizon/docker/docker-compose.yml @@ -27,13 +27,8 @@ services: - "11725:11725" environment: - DATABASE_URL=postgres://postgres@host.docker.internal:5432/horizon?sslmode=disable - - CAPTIVE_CORE_CONFIG_APPEND_PATH=/captive-core-testnet.cfg - - HISTORY_ARCHIVE_URLS=https://history.stellar.org/prd/core-testnet/core_testnet_001 - - NETWORK_PASSPHRASE=Test SDF Network ; September 2015 - - INGEST=true + - NETWORK=testnet - PER_HOUR_RATE_LIMIT=0 - volumes: - - ./captive-core-testnet.cfg:/captive-core-testnet.cfg command: ["--apply-migrations"] extra_hosts: - "host.docker.internal:host-gateway" diff --git a/services/horizon/internal/app.go b/services/horizon/internal/app.go index 6c6132f739..3cf4d6e3d8 100644 --- a/services/horizon/internal/app.go +++ b/services/horizon/internal/app.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "os/signal" + "strings" "sync" "syscall" "time" @@ -118,22 +119,6 @@ func (a *App) Serve() error { signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - if a.config.UsingDefaultPubnetConfig { - const warnMsg = "Horizon started using the default pubnet configuration. " + - "This is not safe! Please provide a custom --captive-core-config-path." - log.Warn(warnMsg) - go func() { - for { - select { - case <-time.After(time.Hour): - log.Warn(warnMsg) - case <-a.done: - return - } - } - }() - } - go func() { select { case <-signalChan: @@ -212,6 +197,13 @@ func (a *App) Paths() paths.Finder { return a.paths } +func isLocalAddress(url string, port uint) bool { + localHostURL := fmt.Sprintf("http://localhost:%d", port) + localIPURL := fmt.Sprintf("http://127.0.0.1:%d", port) + + return strings.HasPrefix(url, localHostURL) || strings.HasPrefix(url, localIPURL) +} + // UpdateCoreLedgerState triggers a refresh of Stellar-Core ledger state. // This is done separately from Horizon ledger state update to prevent issues // in case Stellar-Core query timeout. @@ -221,7 +213,7 @@ func (a *App) UpdateCoreLedgerState(ctx context.Context) { // #4446 If the ingestion state machine is in the build state, the query can time out // because the captive-core buffer may be full. In this case, skip the check. if a.config.CaptiveCoreToml != nil && - a.config.StellarCoreURL == fmt.Sprintf("http://localhost:%d", a.config.CaptiveCoreToml.HTTPPort) && + isLocalAddress(a.config.StellarCoreURL, a.config.CaptiveCoreToml.HTTPPort) && a.ingester != nil && a.ingester.GetCurrentState() == ingest.Build { return } diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index 23329423a5..5a9776e5e2 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -21,7 +21,6 @@ type Config struct { EnableCaptiveCoreIngestion bool EnableIngestionFiltering bool - UsingDefaultPubnetConfig bool CaptiveCoreBinaryPath string RemoteCaptiveCoreURL string CaptiveCoreConfigPath string @@ -114,4 +113,6 @@ type Config struct { BehindAWSLoadBalancer bool // RoundingSlippageFilter excludes trades from /trade_aggregations with rounding slippage >x bps RoundingSlippageFilter int + // Stellar network: 'testnet' or 'pubnet' + Network string } diff --git a/services/horizon/configs/captive-core-pubnet.cfg b/services/horizon/internal/configs/captive-core-pubnet.cfg similarity index 100% rename from services/horizon/configs/captive-core-pubnet.cfg rename to services/horizon/internal/configs/captive-core-pubnet.cfg diff --git a/services/horizon/configs/captive-core-testnet.cfg b/services/horizon/internal/configs/captive-core-testnet.cfg similarity index 100% rename from services/horizon/configs/captive-core-testnet.cfg rename to services/horizon/internal/configs/captive-core-testnet.cfg diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 64982e3068..ec8e4dc1ae 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -1,12 +1,12 @@ package horizon import ( + _ "embed" "fmt" "go/types" stdLog "log" "os" "os/exec" - "path/filepath" "strings" "github.com/sirupsen/logrus" @@ -18,6 +18,7 @@ import ( apkg "github.com/stellar/go/support/app" support "github.com/stellar/go/support/config" "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" "github.com/stellar/throttled" ) @@ -38,10 +39,22 @@ const ( captiveCoreConfigAppendPathName = "captive-core-config-append-path" // CaptiveCoreConfigPathName is the command line flag for configuring the path to the captive core configuration file CaptiveCoreConfigPathName = "captive-core-config-path" - // captive-core-use-db is the command line flag for enabling captive core runtime to use an external db url connection rather than RAM for ledger states + // CaptiveCoreConfigUseDB is the command line flag for enabling captive core runtime to use an external db url + // connection rather than RAM for ledger states CaptiveCoreConfigUseDB = "captive-core-use-db" + // NetworkPassphraseFlagName is the command line flag for specifying the network passphrase + NetworkPassphraseFlagName = "network-passphrase" + // HistoryArchiveURLsFlagName is the command line flag for specifying the history archive URLs + HistoryArchiveURLsFlagName = "history-archive-urls" + // NetworkFlagName is the command line flag for specifying the "network" + NetworkFlagName = "network" + EnableIngestionFilteringFlag = "exp-enable-ingestion-filtering" captiveCoreMigrationHint = "If you are migrating from Horizon 1.x.y, start with the Migration Guide here: https://developers.stellar.org/docs/run-api-server/migrating/" + // StellarPubnet is a constant representing the Stellar public network + StellarPubnet = "pubnet" + // StellarTestnet is a constant representing the Stellar test network + StellarTestnet = "testnet" ) // validateBothOrNeither ensures that both options are provided, if either is provided. @@ -194,12 +207,28 @@ func Flags() (*Config, support.ConfigOptions) { ConfigKey: &config.EnableCaptiveCoreIngestion, }, &support.ConfigOption{ - Name: "exp-enable-ingestion-filtering", + Name: EnableIngestionFilteringFlag, OptType: types.Bool, - FlagDefault: false, + FlagDefault: true, Required: false, - Usage: "causes Horizon to enable the experimental Ingestion Filtering and the ingestion admin HTTP endpoint at /ingestion/filter", ConfigKey: &config.EnableIngestionFiltering, + CustomSetValue: func(opt *support.ConfigOption) error { + + // Always enable ingestion filtering by default. + config.EnableIngestionFiltering = true + + if val := viper.GetString(opt.Name); val != "" { + stdLog.Printf( + "DEPRECATED - No ingestion filter rules are defined by default, which equates to no filtering " + + "of historical data. If you have never added filter rules to this deployment, then nothing further needed. " + + "If you have defined ingestion filter rules prior but disabled filtering overall by setting this flag " + + "disabled with --exp-enable-ingestion-filtering=false, then you should now delete the filter rules using " + + "the Horizon Admin API to achieve the same no-filtering result. Remove usage of this flag in all cases.", + ) + } + return nil + }, + Hidden: true, }, &support.ConfigOption{ Name: "captive-core-http-port", @@ -226,7 +255,7 @@ func Flags() (*Config, support.ConfigOptions) { return nil }, Required: false, - Usage: "Storage location for Captive Core bucket data", + Usage: "Storage location for Captive Core bucket data. If not set, the current working directory is used as the default location.", ConfigKey: &config.CaptiveCoreStoragePath, }, &support.ConfigOption{ @@ -253,15 +282,19 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "stellar-core to connect with (for http commands). If unset and the local Captive core is enabled, it will use http://localhost:", }, &support.ConfigOption{ - Name: "history-archive-urls", - ConfigKey: &config.HistoryArchiveURLs, - OptType: types.String, - Required: false, - FlagDefault: "", + Name: HistoryArchiveURLsFlagName, + ConfigKey: &config.HistoryArchiveURLs, + OptType: types.String, + Required: false, CustomSetValue: func(co *support.ConfigOption) error { stringOfUrls := viper.GetString(co.Name) urlStrings := strings.Split(stringOfUrls, ",") - *(co.ConfigKey.(*[]string)) = urlStrings + //urlStrings contains a single empty value when stringOfUrls is empty + if len(urlStrings) == 1 && urlStrings[0] == "" { + *(co.ConfigKey.(*[]string)) = []string{} + } else { + *(co.ConfigKey.(*[]string)) = urlStrings + } return nil }, Usage: "comma-separated list of stellar history archives to connect with", @@ -412,10 +445,10 @@ func Flags() (*Config, support.ConfigOptions) { " A value of zero (the default) disables the limit.", }, &support.ConfigOption{ - Name: "network-passphrase", + Name: NetworkPassphraseFlagName, ConfigKey: &config.NetworkPassphrase, OptType: types.String, - Required: true, + Required: false, Usage: "Override the network passphrase", }, &support.ConfigOption{ @@ -557,6 +590,25 @@ func Flags() (*Config, support.ConfigOptions) { Required: false, Usage: "excludes trades from /trade_aggregations unless their rounding slippage is 0 { + return fmt.Errorf("invalid config: %s not allowed with %s network", HistoryArchiveURLsFlagName, config.Network) + } + + var defaultNetworkConfig networkConfig + switch config.Network { + case StellarPubnet: + defaultNetworkConfig = pubnetConf + case StellarTestnet: + defaultNetworkConfig = testnetConf + default: + return fmt.Errorf("no default configuration found for network %s", config.Network) + } + config.NetworkPassphrase = defaultNetworkConfig.networkPassphrase + config.HistoryArchiveURLs = defaultNetworkConfig.historyArchiveURLs + + if config.CaptiveCoreConfigPath == "" { + return loadDefaultCaptiveCoreToml(config, defaultNetworkConfig.defaultConfig) + } + + return loadCaptiveCoreTomlFromFile(config) +} + +// createCaptiveCoreConfigFromParameters generates the Captive Core configuration. +// validates the configuration settings, sets necessary values, and loads the Captive Core TOML file. +func createCaptiveCoreConfigFromParameters(config *Config) error { + + if config.NetworkPassphrase == "" { + return fmt.Errorf("%s must be set", NetworkPassphraseFlagName) + } + + if len(config.HistoryArchiveURLs) == 0 { + return fmt.Errorf("%s must be set", HistoryArchiveURLsFlagName) + } + + if config.CaptiveCoreConfigPath != "" { + return loadCaptiveCoreTomlFromFile(config) + } else { + var err error + config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreToml(config.CaptiveCoreTomlParams) + if err != nil { + return errors.Wrap(err, "invalid captive core toml file") + } + } + + return nil +} + +// setCaptiveCoreConfiguration prepares configuration for the Captive Core +func setCaptiveCoreConfiguration(config *Config) error { + stdLog.Println("Preparing captive core...") + + // If the user didn't specify a Stellar Core binary, we can check the + // $PATH and possibly fill it in for them. + if config.CaptiveCoreBinaryPath == "" { + var err error + if config.CaptiveCoreBinaryPath, err = getCaptiveCoreBinaryPath(); err != nil { + return fmt.Errorf("captive core requires %s", StellarCoreBinaryPathName) + } + } + + if config.Network != "" { + err := createCaptiveCoreConfigFromNetwork(config) + if err != nil { + return errors.Wrap(err, "error generating default captive core config.") + } + } else { + err := createCaptiveCoreConfigFromParameters(config) + if err != nil { + return errors.Wrap(err, "error generating captive core config.") + } + } + + // If we don't supply an explicit core URL and running captive core process with the http port enabled, + // point to it. + if config.StellarCoreURL == "" && config.CaptiveCoreToml.HTTPPort != 0 { + config.StellarCoreURL = fmt.Sprintf("http://localhost:%d", config.CaptiveCoreToml.HTTPPort) + } + + return nil +} + // ApplyFlags applies the command line flags on the given Config instance func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOptions) error { // Verify required options and load the config struct @@ -608,6 +816,8 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption config.Ingest = true } + config.EnableIngestionFiltering = true + if config.Ingest { // Migrations should be checked as early as possible. Apply and check // only on ingesting instances which are required to have write-access @@ -623,94 +833,10 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption return err } - // config.HistoryArchiveURLs contains a single empty value when empty so using - // viper.GetString is easier. - if len(config.HistoryArchiveURLs) == 1 && config.HistoryArchiveURLs[0] == "" { - return fmt.Errorf("--history-archive-urls must be set when --ingest is set") - } - if config.EnableCaptiveCoreIngestion { - stdLog.Println("Preparing captive core...") - - binaryPath := viper.GetString(StellarCoreBinaryPathName) - - // If the user didn't specify a Stellar Core binary, we can check the - // $PATH and possibly fill it in for them. - if binaryPath == "" { - if result, err := exec.LookPath("stellar-core"); err == nil { - binaryPath = result - viper.Set(StellarCoreBinaryPathName, binaryPath) - config.CaptiveCoreBinaryPath = binaryPath - } else { - return fmt.Errorf("invalid config: captive core requires --%s. %s", - StellarCoreBinaryPathName, captiveCoreMigrationHint) - } - } else { - config.CaptiveCoreBinaryPath = binaryPath - } - - config.CaptiveCoreTomlParams.CoreBinaryPath = config.CaptiveCoreBinaryPath - if config.CaptiveCoreConfigPath == "" { - if options.RequireCaptiveCoreConfig { - var err error - errorMessage := fmt.Errorf( - "invalid config: captive core requires that --%s is set. %s", - CaptiveCoreConfigPathName, captiveCoreMigrationHint, - ) - - var configFileName string - // Default config files will be located along the binary in the release archive. - switch config.NetworkPassphrase { - case network.TestNetworkPassphrase: - configFileName = "captive-core-testnet.cfg" - config.HistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-testnet/core_testnet_001/"} - case network.PublicNetworkPassphrase: - configFileName = "captive-core-pubnet.cfg" - config.HistoryArchiveURLs = []string{"https://history.stellar.org/prd/core-live/core_live_001/"} - config.UsingDefaultPubnetConfig = true - default: - return errorMessage - } - - executablePath, err := os.Executable() - if err != nil { - return errorMessage - } - - config.CaptiveCoreConfigPath = filepath.Join(filepath.Dir(executablePath), configFileName) - if _, err = os.Stat(config.CaptiveCoreConfigPath); os.IsNotExist(err) { - return errorMessage - } - - config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase - config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromFile(config.CaptiveCoreConfigPath, config.CaptiveCoreTomlParams) - if err != nil { - return fmt.Errorf("Invalid captive core toml file %v", err) - } - } else { - var err error - config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs - config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase - config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreToml(config.CaptiveCoreTomlParams) - if err != nil { - return fmt.Errorf("Invalid captive core toml file %v", err) - } - } - } else { - var err error - config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs - config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase - config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromFile(config.CaptiveCoreConfigPath, config.CaptiveCoreTomlParams) - if err != nil { - return fmt.Errorf("Invalid captive core toml file %v", err) - } - } - - // If we don't supply an explicit core URL and we are running a local - // captive core process with the http port enabled, point to it. - if config.StellarCoreURL == "" && config.CaptiveCoreToml.HTTPPort != 0 { - config.StellarCoreURL = fmt.Sprintf("http://localhost:%d", config.CaptiveCoreToml.HTTPPort) - viper.Set(StellarCoreURLFlagName, config.StellarCoreURL) + err := setCaptiveCoreConfiguration(config) + if err != nil { + return errors.Wrap(err, "error generating captive core configuration") } } } else { diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go new file mode 100644 index 0000000000..9505fbe44e --- /dev/null +++ b/services/horizon/internal/flags_test.go @@ -0,0 +1,133 @@ +package horizon + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_createCaptiveCoreDefaultConfig(t *testing.T) { + + var errorMsgDefaultConfig = "invalid config: %s not allowed with %s network" + tests := []struct { + name string + config Config + networkPassphrase string + historyArchiveURLs []string + errStr string + }{ + { + name: "testnet default config", + config: Config{Network: StellarTestnet}, + networkPassphrase: testnetConf.networkPassphrase, + historyArchiveURLs: testnetConf.historyArchiveURLs, + }, + { + name: "pubnet default config", + config: Config{Network: StellarPubnet}, + networkPassphrase: pubnetConf.networkPassphrase, + historyArchiveURLs: pubnetConf.historyArchiveURLs, + }, + { + name: "testnet validation; history archive urls supplied", + config: Config{Network: StellarTestnet, + HistoryArchiveURLs: []string{"network history archive urls supplied"}, + }, + errStr: fmt.Sprintf(errorMsgDefaultConfig, HistoryArchiveURLsFlagName, StellarTestnet), + }, + { + name: "pubnet validation; history archive urls supplied", + config: Config{Network: StellarPubnet, + HistoryArchiveURLs: []string{"network history archive urls supplied"}, + }, + errStr: fmt.Sprintf(errorMsgDefaultConfig, HistoryArchiveURLsFlagName, StellarPubnet), + }, + { + name: "testnet validation; network passphrase supplied", + config: Config{Network: StellarTestnet, + NetworkPassphrase: "network passphrase supplied", + HistoryArchiveURLs: []string{}, + }, + errStr: fmt.Sprintf(errorMsgDefaultConfig, NetworkPassphraseFlagName, StellarTestnet), + }, + { + name: "pubnet validation; network passphrase supplied", + config: Config{Network: StellarPubnet, + NetworkPassphrase: "pubnet network passphrase supplied", + HistoryArchiveURLs: []string{}, + }, + errStr: fmt.Sprintf(errorMsgDefaultConfig, NetworkPassphraseFlagName, StellarPubnet), + }, + { + name: "unknown network specified", + config: Config{Network: "unknown", + NetworkPassphrase: "", + HistoryArchiveURLs: []string{}, + }, + errStr: "no default configuration found for network unknown", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := createCaptiveCoreConfigFromNetwork(&tt.config) + if tt.errStr == "" { + assert.NoError(t, e) + assert.Equal(t, tt.networkPassphrase, tt.config.NetworkPassphrase) + assert.Equal(t, tt.historyArchiveURLs, tt.config.HistoryArchiveURLs) + } else { + assert.Equal(t, tt.errStr, e.Error()) + } + }) + } +} + +func Test_createCaptiveCoreConfig(t *testing.T) { + + var errorMsgConfig = "%s must be set" + tests := []struct { + name string + config Config + networkPassphrase string + historyArchiveURLs []string + errStr string + }{ + { + name: "no network specified", + config: Config{ + NetworkPassphrase: "NetworkPassphrase", + HistoryArchiveURLs: []string{"HistoryArchiveURLs"}, + }, + networkPassphrase: "NetworkPassphrase", + historyArchiveURLs: []string{"HistoryArchiveURLs"}, + }, + { + name: "no network specified; passphrase not supplied", + config: Config{ + HistoryArchiveURLs: []string{"HistoryArchiveURLs"}, + }, + errStr: fmt.Sprintf(errorMsgConfig, NetworkPassphraseFlagName), + }, + { + name: "no network specified; history archive urls not supplied", + config: Config{ + NetworkPassphrase: "NetworkPassphrase", + }, + errStr: fmt.Sprintf(errorMsgConfig, HistoryArchiveURLsFlagName), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := createCaptiveCoreConfigFromParameters(&tt.config) + if tt.errStr == "" { + assert.NoError(t, e) + assert.Equal(t, tt.networkPassphrase, tt.config.NetworkPassphrase) + assert.Equal(t, tt.historyArchiveURLs, tt.config.HistoryArchiveURLs) + } else { + require.Error(t, e) + assert.Equal(t, tt.errStr, e.Error()) + } + }) + } +} diff --git a/services/horizon/internal/integration/ingestion_filtering_test.go b/services/horizon/internal/integration/ingestion_filtering_test.go index 47cfc1ccbc..6d0bcee69e 100644 --- a/services/horizon/internal/integration/ingestion_filtering_test.go +++ b/services/horizon/internal/integration/ingestion_filtering_test.go @@ -13,29 +13,24 @@ import ( "github.com/stretchr/testify/assert" ) -func TestFilteringAccountWhiteList(t *testing.T) { +func TestFilteringWithNoFilters(t *testing.T) { tt := assert.New(t) const adminPort uint16 = 6000 itest := integration.NewTest(t, integration.Config{ HorizonIngestParameters: map[string]string{ - "admin-port": strconv.Itoa(int(adminPort)), - "exp-enable-ingestion-filtering": "true", + "admin-port": strconv.Itoa(int(adminPort)), }, }) fullKeys, accounts := itest.CreateAccounts(2, "10000") - whitelistedAccount := accounts[0] - whitelistedAccountKey := fullKeys[0] nonWhitelistedAccount := accounts[1] nonWhitelistedAccountKey := fullKeys[1] - enabled := true // all assets are allowed by default because the asset filter config is empty. defaultAllowedAsset := txnbuild.CreditAsset{Code: "PTS", Issuer: itest.Master().Address()} - itest.MustEstablishTrustline(whitelistedAccountKey, whitelistedAccount, defaultAllowedAsset) itest.MustEstablishTrustline(nonWhitelistedAccountKey, nonWhitelistedAccount, defaultAllowedAsset) - // assert that by system default, filters with no rules yet, allow all first + // Assert that by default, the system allows all the accounts. txResp := itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), &txnbuild.Payment{ Destination: nonWhitelistedAccount.GetAccountID(), @@ -43,9 +38,30 @@ func TestFilteringAccountWhiteList(t *testing.T) { Asset: defaultAllowedAsset, }, ) - txResp, err := itest.Client().TransactionDetail(txResp.Hash) tt.NoError(err) +} + +func TestFilteringAccountWhiteList(t *testing.T) { + tt := assert.New(t) + const adminPort uint16 = 6000 + itest := integration.NewTest(t, integration.Config{ + HorizonIngestParameters: map[string]string{ + "admin-port": strconv.Itoa(int(adminPort)), + }, + }) + + fullKeys, accounts := itest.CreateAccounts(2, "10000") + whitelistedAccount := accounts[0] + whitelistedAccountKey := fullKeys[0] + nonWhitelistedAccount := accounts[1] + nonWhitelistedAccountKey := fullKeys[1] + enabled := true + + // all assets are allowed by default because the asset filter config is empty. + defaultAllowedAsset := txnbuild.CreditAsset{Code: "PTS", Issuer: itest.Master().Address()} + itest.MustEstablishTrustline(whitelistedAccountKey, whitelistedAccount, defaultAllowedAsset) + itest.MustEstablishTrustline(nonWhitelistedAccountKey, nonWhitelistedAccount, defaultAllowedAsset) // Setup a whitelisted account rule, force refresh of filter configs to be quick filters.SetFilterConfigCheckIntervalSeconds(1) @@ -54,7 +70,7 @@ func TestFilteringAccountWhiteList(t *testing.T) { Whitelist: []string{whitelistedAccount.GetAccountID()}, Enabled: &enabled, } - err = itest.AdminClient().SetIngestionAccountFilter(expectedAccountFilter) + err := itest.AdminClient().SetIngestionAccountFilter(expectedAccountFilter) tt.NoError(err) accountFilter, err := itest.AdminClient().GetIngestionAccountFilter() @@ -67,7 +83,7 @@ func TestFilteringAccountWhiteList(t *testing.T) { time.Sleep(time.Duration(filters.GetFilterConfigCheckIntervalSeconds()) * time.Second) // Make sure that when using a non-whitelisted account, the transaction is not stored - txResp = itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), + txResp := itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), &txnbuild.Payment{ Destination: nonWhitelistedAccount.GetAccountID(), Amount: "10", @@ -94,8 +110,7 @@ func TestFilteringAssetWhiteList(t *testing.T) { const adminPort uint16 = 6000 itest := integration.NewTest(t, integration.Config{ HorizonIngestParameters: map[string]string{ - "admin-port": strconv.Itoa(int(adminPort)), - "exp-enable-ingestion-filtering": "true", + "admin-port": strconv.Itoa(int(adminPort)), }, }) @@ -110,18 +125,6 @@ func TestFilteringAssetWhiteList(t *testing.T) { itest.MustEstablishTrustline(defaultAllowedAccountKey, defaultAllowedAccount, nonWhitelistedAsset) enabled := true - // assert that by system default, filters with no rules yet, allow all first - txResp := itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), - &txnbuild.Payment{ - Destination: defaultAllowedAccount.GetAccountID(), - Amount: "10", - Asset: nonWhitelistedAsset, - }, - ) - - _, err := itest.Client().TransactionDetail(txResp.Hash) - tt.NoError(err) - // Setup a whitelisted asset rule, force refresh of filters to be quick filters.SetFilterConfigCheckIntervalSeconds(1) @@ -144,7 +147,7 @@ func TestFilteringAssetWhiteList(t *testing.T) { time.Sleep(time.Duration(filters.GetFilterConfigCheckIntervalSeconds()) * time.Second) // Make sure that when using a non-whitelisted asset, the transaction is not stored - txResp = itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), + txResp := itest.MustSubmitOperations(itest.MasterAccount(), itest.Master(), &txnbuild.Payment{ Destination: defaultAllowedAccount.GetAccountID(), Amount: "10", diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index f33be9b2ab..44adb35d00 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -2,13 +2,20 @@ package integration import ( + "bytes" + "fmt" + "io" "io/ioutil" + stdLog "log" "os" "os/exec" "path" "strings" + "sync" "testing" + "github.com/spf13/cobra" + "github.com/stellar/go/services/horizon/internal/paths" "github.com/stellar/go/services/horizon/internal/simplepath" @@ -212,6 +219,92 @@ func TestDisablePathFinding(t *testing.T) { }) } +func TestIngestionFilteringAlwaysDefaultingToTrue(t *testing.T) { + t.Run("ingestion filtering flag set to default value", func(t *testing.T) { + test := NewParameterTest(t, map[string]string{}) + err := test.StartHorizon() + assert.NoError(t, err) + test.WaitForHorizon() + assert.Equal(t, test.HorizonIngest().Config().EnableIngestionFiltering, true) + test.Shutdown() + }) + t.Run("ingestion filtering flag set to false", func(t *testing.T) { + test := NewParameterTest(t, map[string]string{"exp-enable-ingestion-filtering": "false"}) + err := test.StartHorizon() + assert.NoError(t, err) + test.WaitForHorizon() + assert.Equal(t, test.HorizonIngest().Config().EnableIngestionFiltering, true) + test.Shutdown() + }) +} + +func TestDeprecatedOutputForIngestionFilteringFlag(t *testing.T) { + originalStderr := os.Stderr + r, w, _ := os.Pipe() + os.Stderr = w + stdLog.SetOutput(os.Stderr) + + test := NewParameterTest(t, map[string]string{"exp-enable-ingestion-filtering": "false"}) + err := test.StartHorizon() + assert.NoError(t, err) + test.WaitForHorizon() + + // Use a wait group to wait for the goroutine to finish before proceeding + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + if err := w.Close(); err != nil { + t.Errorf("Failed to close Stdout") + return + } + }() + + outputBytes, _ := io.ReadAll(r) + wg.Wait() // Wait for the goroutine to finish before proceeding + _ = r.Close() + os.Stderr = originalStderr + + assert.Contains(t, string(outputBytes), "DEPRECATED - No ingestion filter rules are defined by default, which equates to "+ + "no filtering of historical data. If you have never added filter rules to this deployment, then nothing further needed. "+ + "If you have defined ingestion filter rules prior but disabled filtering overall by setting this flag disabled with "+ + "--exp-enable-ingestion-filtering=false, then you should now delete the filter rules using the Horizon Admin API to achieve "+ + "the same no-filtering result. Remove usage of this flag in all cases.") +} + +func TestHelpOutputForNoIngestionFilteringFlag(t *testing.T) { + config, flags := horizon.Flags() + + horizonCmd := &cobra.Command{ + Use: "horizon", + Short: "Client-facing api server for the Stellar network", + SilenceErrors: true, + SilenceUsage: true, + Long: "Client-facing API server for the Stellar network.", + RunE: func(cmd *cobra.Command, args []string) error { + _, err := horizon.NewAppFromFlags(config, flags) + if err != nil { + return err + } + return nil + }, + } + + var writer io.Writer = &bytes.Buffer{} + horizonCmd.SetOutput(writer) + + horizonCmd.SetArgs([]string{"-h"}) + if err := flags.Init(horizonCmd); err != nil { + fmt.Println(err) + } + if err := horizonCmd.Execute(); err != nil { + fmt.Println(err) + } + + output := writer.(*bytes.Buffer).String() + assert.NotContains(t, output, "--exp-enable-ingestion-filtering") +} + // Pattern taken from testify issue: // https://github.com/stretchr/testify/issues/858#issuecomment-600491003 // @@ -223,8 +316,8 @@ type FatalTestCase struct { suite.Suite } -func (t *FatalTestCase) Exits(subprocess func()) { - testName := t.T().Name() +func (suite *FatalTestCase) Exits(subprocess func()) { + testName := suite.T().Name() if os.Getenv("ASSERT_EXISTS_"+testName) == "1" { subprocess() return @@ -234,12 +327,12 @@ func (t *FatalTestCase) Exits(subprocess func()) { cmd.Env = append(os.Environ(), "ASSERT_EXISTS_"+testName+"=1") err := cmd.Run() - t.T().Log("Result:", err) + suite.T().Log("Result:", err) if e, ok := err.(*exec.ExitError); ok && !e.Success() { return } - t.Fail("expecting unsuccessful exit, got", err) + suite.Fail("expecting unsuccessful exit, got", err) } // validateNoBucketDirPath ensures the Stellar Core auto-generated configuration diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index 222f3c9234..1a6f65e453 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -105,21 +105,6 @@ type Test struct { passPhrase string } -func NewTestForRemoteHorizon(t *testing.T, horizonURL string, passPhrase string, masterKey *keypair.Full) *Test { - adminClient, err := sdk.NewAdminClient(0, "", 0) - if err != nil { - t.Fatal(err) - } - - return &Test{ - t: t, - horizonClient: &sdk.Client{HorizonURL: horizonURL}, - horizonAdminClient: adminClient, - masterKey: masterKey, - passPhrase: passPhrase, - } -} - // NewTest starts a new environment for integration test at a given // protocol version and blocks until Horizon starts ingesting. // diff --git a/support/config/config_option.go b/support/config/config_option.go index ca70080606..ca4a94aea6 100644 --- a/support/config/config_option.go +++ b/support/config/config_option.go @@ -23,10 +23,10 @@ type ConfigOptions []*ConfigOption // Init calls Init on each ConfigOption passing on the cobra.Command. func (cos ConfigOptions) Init(cmd *cobra.Command) error { for _, co := range cos { - err := co.Init(cmd) - if err != nil { + if err := co.Init(cmd); err != nil { return err } + co.SetDeprecated(cmd) } return nil } @@ -69,6 +69,7 @@ type ConfigOption struct { CustomSetValue func(*ConfigOption) error // Optional function for custom validation/transformation ConfigKey interface{} // Pointer to the final key in the linked Config struct flag *pflag.Flag // The persistent flag that the config option is attached to + Hidden bool // A flag which indicates whether to hide the flag from --help output } // Init handles initialisation steps, including configuring and binding the env variable name. @@ -82,6 +83,13 @@ func (co *ConfigOption) Init(cmd *cobra.Command) error { return co.setFlag(cmd) } +// SetDeprecated Hides the deprecated flag from --help output +func (co *ConfigOption) SetDeprecated(cmd *cobra.Command) { + if co.Hidden { + co.flag.Hidden = true + } +} + // Bind binds the config option to viper. func (co *ConfigOption) Bind() { viper.BindPFlag(co.Name, co.flag) diff --git a/support/scripts/build_release_artifacts/main.go b/support/scripts/build_release_artifacts/main.go index 4acbba1b2c..0a96d627fa 100644 --- a/support/scripts/build_release_artifacts/main.go +++ b/support/scripts/build_release_artifacts/main.go @@ -321,11 +321,6 @@ func prepareDest(pkg, bin, version, os, arch string) string { run("cp", "COPYING", dest) run("cp", filepath.Join(pkg, "README.md"), dest) run("cp", filepath.Join(pkg, "CHANGELOG.md"), dest) - if bin == "horizon" { - // Add default config files for Captive-Core - run("cp", filepath.Join(pkg, "configs/captive-core-pubnet.cfg"), dest) - run("cp", filepath.Join(pkg, "configs/captive-core-testnet.cfg"), dest) - } return dest }