horizon: Merge master into horizon-protocol-19 branch

.github/workflows/horizon.yml

@@ -15,7 +15,7 @@ jobs:
         go: [1.17, 1.18]
         pg: [9.6.5]
         ingestion-backend: [db, captive-core, captive-core-remote-storage]
-        captive-core: [18.0.3-746.f3baea6.focal]
+        captive-core: [18.5.0-873.rc1.d387c6a71.focal]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
@@ -47,6 +47,16 @@ jobs:
         # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
         fetch-depth: '0'
 
+    # In order to debug the integration tests, run 'touch continue' once you connect to the ssh session
+    #
+    # - name: Setup upterm session
+    #  uses: lhotari/action-upterm@d23c2722bdab893785c9fbeae314cbf080645bd7
+    #  with:
+    #    ## limits ssh access and adds the ssh public key for the user which triggered the workflow
+    #    limit-access-to-actor: true
+    #    ## limits ssh access and adds the ssh public keys of the listed GitHub users
+    #    limit-access-to-users: <yourGithubUser>
+
     - uses: ./.github/actions/setup-go
       with:
         go-version: ${{ matrix.go }}
@@ -57,6 +67,11 @@ jobs:
     - if: ${{ startsWith(matrix.ingestion-backend, 'captive-core') }}
       name: Install and enable Captive Core
       run: |
+        # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+        # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpul
+        # message about why the installation fails)
+        sudo apt-get remove -y libc++1-10 libc++abi1-10 || true
+
         sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
         sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
         sudo apt-get update && sudo apt-get install -y stellar-core=${{ matrix.captive-core }}

clients/horizonclient/claimable_balance_request.go

@@ -13,24 +13,29 @@ import (
 func (cbr ClaimableBalanceRequest) BuildURL() (endpoint string, err error) {
 	endpoint = "claimable_balances"
 
+	//max limit is 200
+	if cbr.Limit > 200 {
+		return endpoint, errors.New("invalid request: limit " + fmt.Sprint(cbr.Limit) + " is greater than limit max of 200")
+	}
+
 	// Only one filter parameter is allowed, and you can't mix an ID query and
 	// filters.
 	nParams := countParams(cbr.Asset, cbr.Claimant, cbr.Sponsor, cbr.ID)
-	if nParams > 1 {
+	if cbr.ID != "" && nParams > 1 {
 		return endpoint, errors.New("invalid request: too many parameters")
 	}
 
 	if cbr.ID != "" {
 		endpoint = fmt.Sprintf("%s/%s", endpoint, cbr.ID)
 	} else {
+		params := map[string]string{
+			"claimant": cbr.Claimant,
+			"sponsor":  cbr.Sponsor,
+			"asset":    cbr.Asset,
+		}
 		queryParams := addQueryParams(
-			map[string]string{
-				"claimant": cbr.Claimant,
-				"sponsor":  cbr.Sponsor,
-				"asset":    cbr.Asset,
-			},
+			params, limit(cbr.Limit), cursor(cbr.Cursor),
 		)
-
 		endpoint = fmt.Sprintf("%s?%s", endpoint, queryParams)
 	}
 

clients/horizonclient/claimable_balance_request_test.go

@@ -0,0 +1,53 @@
+package horizonclient
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestClaimableBalanceBuildUrl(t *testing.T) {
+
+	cbr := ClaimableBalanceRequest{
+		ID: "1235",
+	}
+	url, err := cbr.BuildURL()
+	assert.Equal(t, "claimable_balances/1235", url)
+	assert.NoError(t, err)
+
+	//if the ID is included, you cannot include another parameter
+	cbr = ClaimableBalanceRequest{
+		ID:       "1235",
+		Claimant: "CLAIMANTADDRESS",
+	}
+	_, err = cbr.BuildURL()
+	assert.EqualError(t, err, "invalid request: too many parameters")
+
+	//if you have two parameters, and neither of them are ID, it must use both in the URL
+	cbr = ClaimableBalanceRequest{
+		Claimant: "CLAIMANTADDRESS",
+		Asset:    "TEST:ISSUERADDRESS",
+	}
+	url, err = cbr.BuildURL()
+	assert.NoError(t, err)
+	assert.Equal(t, "claimable_balances?asset=TEST%3AISSUERADDRESS&claimant=CLAIMANTADDRESS", url)
+
+	//check limit
+	cbr = ClaimableBalanceRequest{
+		Claimant: "CLAIMANTADDRESS",
+		Asset:    "TEST:ISSUERADDRESS",
+		Limit:    200,
+	}
+	url, err = cbr.BuildURL()
+	assert.NoError(t, err)
+	assert.Equal(t, "claimable_balances?asset=TEST%3AISSUERADDRESS&claimant=CLAIMANTADDRESS&limit=200", url)
+
+	cbr = ClaimableBalanceRequest{
+		Claimant: "CLAIMANTADDRESS",
+		Asset:    "TEST:ISSUERADDRESS",
+		Limit:    201,
+	}
+	_, err = cbr.BuildURL()
+	assert.EqualError(t, err, "invalid request: limit 201 is greater than limit max of 200")
+
+}

clients/horizonclient/main.go

@@ -434,6 +434,8 @@ type ClaimableBalanceRequest struct {
 	Asset    string
 	Sponsor  string
 	Claimant string
+	Cursor   string
+	Limit    uint
 }
 
 // ServerTimeRecord contains data for the current unix time of a horizon server instance, and the local time when it was recorded.

go.list

@@ -92,6 +92,7 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
 golang.org/x/text v0.3.6
+golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
 google.golang.org/api v0.50.0
 google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84
 google.golang.org/grpc v1.38.0

go.mod

@@ -114,6 +114,7 @@ require (
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
 	golang.org/x/tools v0.1.4 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -546,6 +546,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs=
+golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

ingest/ledgerbackend/captive_core_backend_test.go

@@ -133,8 +133,9 @@ type testLedgerHeader struct {
 }
 
 func TestCaptiveNew(t *testing.T) {
-	storagePath, err := os.MkdirTemp(os.TempDir(), "captive-core-*")
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
 	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
 
 	executablePath := "/etc/stellar-core"
 	networkPassphrase := network.PublicNetworkPassphrase
@@ -827,8 +828,9 @@ func TestCaptiveGetLedger_NextLedger0RangeFromIsSmallerThanLedgerFromBuffer(t *t
 }
 
 func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) {
-	storagePath, err := os.MkdirTemp(os.TempDir(), "captive-core-*")
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
 	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
 
 	ctx := context.Background()
 	executablePath := "/etc/stellar-core"

ingest/ledgerbackend/file_watcher_test.go

@@ -45,8 +45,9 @@ func (m *mockHash) hashFile(fp string) (hash, error) {
 }
 
 func createFWFixtures(t *testing.T) (*mockHash, *stellarCoreRunner, *fileWatcher) {
-	storagePath, err := os.MkdirTemp(os.TempDir(), "captive-core-*")
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
 	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
 
 	ms := &mockHash{
 		hashResult:   hash{},
@@ -75,8 +76,9 @@ func createFWFixtures(t *testing.T) (*mockHash, *stellarCoreRunner, *fileWatcher
 }
 
 func TestNewFileWatcherError(t *testing.T) {
-	storagePath, err := os.MkdirTemp(os.TempDir(), "captive-core-*")
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
 	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
 
 	ms := &mockHash{
 		hashResult:   hash{},

ingest/ledgerbackend/stellar_core_runner_test.go

@@ -13,8 +13,9 @@ import (
 )
 
 func TestCloseBeforeStartOffline(t *testing.T) {
-	storagePath, err := os.MkdirTemp(os.TempDir(), "captive-core-*")
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
 	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
 
 	captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{})
 	assert.NoError(t, err)
@@ -42,8 +43,9 @@ func TestCloseBeforeStartOffline(t *testing.T) {
 }
 
 func TestCloseBeforeStartOnline(t *testing.T) {
-	storagePath, err := os.MkdirTemp(os.TempDir(), "captive-core-*")
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
 	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
 
 	captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{})
 	assert.NoError(t, err)
@@ -72,8 +74,9 @@ func TestCloseBeforeStartOnline(t *testing.T) {
 }
 
 func TestCloseBeforeStartOnlineWithError(t *testing.T) {
-	storagePath, err := os.MkdirTemp(os.TempDir(), "captive-core-*")
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
 	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
 
 	captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{})
 	assert.NoError(t, err)

services/horizon/docker/docker-compose.integration-tests.yml

@@ -11,12 +11,10 @@ services:
     command: ["-p", "5641"]
   core:
     platform: linux/amd64
-    # TODO replace with official SDF image when ready. Note that this:
-    # https://github.com/stellar/stellar-core/commit/31597b760f8e325fc84da0937adc373a78878ca9
-    # breaks the tests. I reverted it before building temp docker image.
-    # Command used to build custom image:
-    # docker build -t bartekno/stellar-core:17.4.0-p18 --build-arg STELLAR_CORE_VERSION=17.3.1-679.c5f6349.focal~protocol18~buildtests --build-arg DISTRO=focal .
-    image: ${CORE_IMAGE:-stellar/stellar-core:18}
+    # Note: Please keep the image pinned to an immutable tag matching the Captive Core version.
+    #       This avoid implicit updates which break compatibility between
+    #       the Core container and captive core.
+    image: ${CORE_IMAGE:-stellar/stellar-core:18.5.0-873.rc1.d387c6a71.focal}
     depends_on:
       - core-postgres
     restart: on-failure

services/horizon/internal/actions/path.go

@@ -164,11 +164,15 @@ func (handler FindPathsHandler) GetResource(w HeaderWriter, r *http.Request) (in
 	if len(query.SourceAssets) > 0 {
 		var lastIngestedLedger uint32
 		records, lastIngestedLedger, err = handler.PathFinder.Find(ctx, query, handler.MaxPathLength)
-		if err == simplepath.ErrEmptyInMemoryOrderBook {
-			err = horizonProblem.StillIngesting
-		}
-		if err != nil {
-			return nil, err
+		switch err {
+		case simplepath.ErrEmptyInMemoryOrderBook:
+			return nil, horizonProblem.StillIngesting
+		case paths.ErrRateLimitExceeded:
+			return nil, horizonProblem.ServerOverCapacity
+		default:
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		if handler.SetLastLedgerHeader {
@@ -338,11 +342,15 @@ func (handler FindFixedPathsHandler) GetResource(w HeaderWriter, r *http.Request
 			destinationAssets,
 			handler.MaxPathLength,
 		)
-		if err == simplepath.ErrEmptyInMemoryOrderBook {
-			err = horizonProblem.StillIngesting
-		}
-		if err != nil {
-			return nil, err
+		switch err {
+		case simplepath.ErrEmptyInMemoryOrderBook:
+			return nil, horizonProblem.StillIngesting
+		case paths.ErrRateLimitExceeded:
+			return nil, horizonProblem.ServerOverCapacity
+		default:
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		if handler.SetLastLedgerHeader {

services/horizon/internal/actions_path_test.go

@@ -76,6 +76,62 @@ func mockPathFindingClient(
 	return test.NewRequestHelper(router)
 }
 
+func TestPathActionsLimitExceeded(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+	test.ResetHorizonDB(t, tt.HorizonDB)
+
+	assertions := &test.Assertions{tt.Assert}
+	finder := paths.MockFinder{}
+	finder.On("Find", mock.Anything, mock.Anything, uint(3)).
+		Return([]paths.Path{}, uint32(0), paths.ErrRateLimitExceeded).Times(2)
+	finder.On("FindFixedPaths", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
+		Return([]paths.Path{}, uint32(0), paths.ErrRateLimitExceeded).Times(1)
+
+	rh := mockPathFindingClient(
+		tt,
+		&finder,
+		2,
+		tt.HorizonSession(),
+	)
+
+	var q = make(url.Values)
+
+	q.Add(
+		"source_assets",
+		"native",
+	)
+	q.Add(
+		"destination_asset_issuer",
+		"GDSBCQO34HWPGUGQSP3QBFEXVTSR2PW46UIGTHVWGWJGQKH3AFNHXHXN",
+	)
+	q.Add("destination_asset_type", "credit_alphanum4")
+	q.Add("destination_asset_code", "EUR")
+	q.Add("destination_amount", "10")
+
+	for _, uri := range []string{"/paths", "/paths/strict-receive"} {
+		w := rh.Get(uri + "?" + q.Encode())
+		assertions.Equal(horizonProblem.ServerOverCapacity.Status, w.Code)
+		assertions.Problem(w.Body, horizonProblem.ServerOverCapacity)
+		assertions.Equal("", w.Header().Get(actions.LastLedgerHeaderName))
+	}
+
+	q = make(url.Values)
+
+	q.Add("destination_assets", "native")
+	q.Add("source_asset_issuer", "GDSBCQO34HWPGUGQSP3QBFEXVTSR2PW46UIGTHVWGWJGQKH3AFNHXHXN")
+	q.Add("source_asset_type", "credit_alphanum4")
+	q.Add("source_asset_code", "EUR")
+	q.Add("source_amount", "10")
+
+	w := rh.Get("/paths/strict-send" + "?" + q.Encode())
+	assertions.Equal(horizonProblem.ServerOverCapacity.Status, w.Code)
+	assertions.Problem(w.Body, horizonProblem.ServerOverCapacity)
+	assertions.Equal("", w.Header().Get(actions.LastLedgerHeaderName))
+
+	finder.AssertExpectations(t)
+}
+
 func TestPathActionsStillIngesting(t *testing.T) {
 	tt := test.Start(t)
 	defer tt.Finish()

services/horizon/internal/app.go

@@ -205,6 +205,11 @@ func (a *App) Config() Config {
 	return a.config
 }
 
+// Paths returns the paths.Finder instance used by horizon
+func (a *App) Paths() paths.Finder {
+	return a.paths
+}
+
 // UpdateCoreLedgerState triggers a refresh of Stellar-Core ledger state.
 // This is done separately from Horizon ledger state update to prevent issues
 // in case Stellar-Core query timeout.

services/horizon/internal/config.go

@@ -49,7 +49,12 @@ type Config struct {
 	MaxPathLength uint
 	// MaxAssetsPerPathRequest is the maximum number of assets considered for `/paths/strict-send` and `/paths/strict-recieve`
 	MaxAssetsPerPathRequest int
-	DisablePoolPathFinding  bool
+	// DisablePoolPathFinding configures horizon to run path finding without including liquidity pools
+	// in the path finding search.
+	DisablePoolPathFinding bool
+	// MaxPathFindingRequests is the maximum number of path finding requests horizon will allow
+	// in a 1-second period. A value of 0 disables the limit.
+	MaxPathFindingRequests uint
 
 	NetworkPassphrase string
 	SentryDSN         string