From 40d505759d70f72b5937f53447da774105c3bff6 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Fri, 10 Jan 2025 14:42:51 -0700 Subject: [PATCH] Move xdrill ledger functions to ingest as subpackage --- exp/xdrill/ledger/ledger.go | 204 ------------------- exp/xdrill/utils/utils.go | 128 ------------ ingest/ledger/ledger.go | 150 ++++++++++++++ {exp/xdrill => ingest}/ledger/ledger_test.go | 99 ++++++--- xdr/node_id.go | 22 ++ 5 files changed, 244 insertions(+), 359 deletions(-) delete mode 100644 exp/xdrill/ledger/ledger.go delete mode 100644 exp/xdrill/utils/utils.go create mode 100644 ingest/ledger/ledger.go rename {exp/xdrill => ingest}/ledger/ledger_test.go (61%) create mode 100644 xdr/node_id.go diff --git a/exp/xdrill/ledger/ledger.go b/exp/xdrill/ledger/ledger.go deleted file mode 100644 index b6cb9f9d48..0000000000 --- a/exp/xdrill/ledger/ledger.go +++ /dev/null @@ -1,204 +0,0 @@ -package ledger - -import ( - "encoding/base64" - "fmt" - "time" - - "github.com/stellar/go/exp/xdrill/utils" - "github.com/stellar/go/xdr" -) - -func Sequence(l xdr.LedgerCloseMeta) uint32 { - return uint32(l.LedgerHeaderHistoryEntry().Header.LedgerSeq) -} - -func ID(l xdr.LedgerCloseMeta) int64 { - return utils.NewID(int32(l.LedgerSequence()), 0, 0).ToInt64() -} - -func Hash(l xdr.LedgerCloseMeta) string { - return utils.HashToHexString(l.LedgerHeaderHistoryEntry().Hash) -} - -func PreviousHash(l xdr.LedgerCloseMeta) string { - return utils.HashToHexString(l.PreviousLedgerHash()) -} - -func CloseTime(l xdr.LedgerCloseMeta) int64 { - return l.LedgerCloseTime() -} - -func ClosedAt(l xdr.LedgerCloseMeta) time.Time { - return time.Unix(l.LedgerCloseTime(), 0).UTC() -} - -func TotalCoins(l xdr.LedgerCloseMeta) int64 { - return int64(l.LedgerHeaderHistoryEntry().Header.TotalCoins) -} - -func FeePool(l xdr.LedgerCloseMeta) int64 { - return int64(l.LedgerHeaderHistoryEntry().Header.FeePool) -} - -func BaseFee(l xdr.LedgerCloseMeta) uint32 { - return uint32(l.LedgerHeaderHistoryEntry().Header.BaseFee) -} - -func BaseReserve(l xdr.LedgerCloseMeta) uint32 { - return uint32(l.LedgerHeaderHistoryEntry().Header.BaseReserve) -} - -func MaxTxSetSize(l xdr.LedgerCloseMeta) uint32 { - return uint32(l.LedgerHeaderHistoryEntry().Header.MaxTxSetSize) -} - -func LedgerVersion(l xdr.LedgerCloseMeta) uint32 { - return uint32(l.LedgerHeaderHistoryEntry().Header.LedgerVersion) -} - -func SorobanFeeWrite1Kb(l xdr.LedgerCloseMeta) *int64 { - lcmV1, ok := l.GetV1() - if !ok { - return nil - } - - extV1, ok := lcmV1.Ext.GetV1() - if !ok { - return nil - } - - result := int64(extV1.SorobanFeeWrite1Kb) - - return &result -} - -func TotalByteSizeOfBucketList(l xdr.LedgerCloseMeta) *uint64 { - lcmV1, ok := l.GetV1() - if !ok { - return nil - } - - result := uint64(lcmV1.TotalByteSizeOfBucketList) - - return &result -} - -func NodeID(l xdr.LedgerCloseMeta) *string { - LedgerCloseValueSignature, ok := l.LedgerHeaderHistoryEntry().Header.ScpValue.Ext.GetLcValueSignature() - if !ok { - return nil - - } - nodeID, ok := utils.GetAddress(LedgerCloseValueSignature.NodeId) - if !ok { - return nil - } - - return &nodeID -} - -func Signature(l xdr.LedgerCloseMeta) *string { - LedgerCloseValueSignature, ok := l.LedgerHeaderHistoryEntry().Header.ScpValue.Ext.GetLcValueSignature() - if !ok { - return nil - } - - result := base64.StdEncoding.EncodeToString(LedgerCloseValueSignature.Signature) - - return &result -} - -// Add docstring to larger, more complicated functions -func TransactionCounts(l xdr.LedgerCloseMeta) (successTxCount, failedTxCount int32, ok bool) { - var results []xdr.TransactionResultMeta - - transactions := getTransactionSet(l) - results = l.TxProcessing() - txCount := len(transactions) - if txCount != len(results) { - return 0, 0, false - } - - for i := 0; i < txCount; i++ { - if results[i].Result.Successful() { - successTxCount++ - } else { - failedTxCount++ - } - } - - return successTxCount, failedTxCount, true -} - -// Add docstring to larger, more complicated functions -func OperationCounts(l xdr.LedgerCloseMeta) (operationCount, txSetOperationCount int32, ok bool) { - var results []xdr.TransactionResultMeta - - transactions := getTransactionSet(l) - results = l.TxProcessing() - - txCount := len(transactions) - if txCount != len(results) { - return 0, 0, false - } - - for i := 0; i < txCount; i++ { - operations := transactions[i].Operations() - numberOfOps := int32(len(operations)) - txSetOperationCount += numberOfOps - - // for successful transactions, the operation count is based on the operations results slice - if results[i].Result.Successful() { - operationResults, ok := results[i].Result.OperationResults() - if !ok { - return 0, 0, false - } - - operationCount += int32(len(operationResults)) - } - - } - - return operationCount, txSetOperationCount, true -} - -func getTransactionSet(l xdr.LedgerCloseMeta) (transactionProcessing []xdr.TransactionEnvelope) { - switch l.V { - case 0: - return l.V0.TxSet.Txs - case 1: - switch l.V1.TxSet.V { - case 0: - return getTransactionPhase(l.V1.TxSet.V1TxSet.Phases) - default: - panic(fmt.Sprintf("unsupported LedgerCloseMeta.V1.TxSet.V: %d", l.V1.TxSet.V)) - } - default: - panic(fmt.Sprintf("unsupported LedgerCloseMeta.V: %d", l.V)) - } -} - -func getTransactionPhase(transactionPhase []xdr.TransactionPhase) (transactionEnvelope []xdr.TransactionEnvelope) { - transactionSlice := []xdr.TransactionEnvelope{} - for _, phase := range transactionPhase { - switch phase.V { - case 0: - components := phase.MustV0Components() - for _, component := range components { - switch component.Type { - case 0: - transactionSlice = append(transactionSlice, component.TxsMaybeDiscountedFee.Txs...) - - default: - panic(fmt.Sprintf("unsupported TxSetComponentType: %d", component.Type)) - } - - } - default: - panic(fmt.Sprintf("unsupported TransactionPhase.V: %d", phase.V)) - } - } - - return transactionSlice -} diff --git a/exp/xdrill/utils/utils.go b/exp/xdrill/utils/utils.go deleted file mode 100644 index 645967679e..0000000000 --- a/exp/xdrill/utils/utils.go +++ /dev/null @@ -1,128 +0,0 @@ -package utils - -import ( - "encoding/hex" - - "github.com/stellar/go/keypair" - "github.com/stellar/go/strkey" - "github.com/stellar/go/txnbuild" - "github.com/stellar/go/xdr" -) - -// HashToHexString is utility function that converts and xdr.Hash type to a hex string -func HashToHexString(inputHash xdr.Hash) string { - sliceHash := inputHash[:] - hexString := hex.EncodeToString(sliceHash) - return hexString -} - -type ID struct { - LedgerSequence int32 - TransactionOrder int32 - OperationOrder int32 -} - -const ( - // LedgerMask is the bitmask to mask out ledger sequences in a - // TotalOrderID - LedgerMask = (1 << 32) - 1 - // TransactionMask is the bitmask to mask out transaction indexes - TransactionMask = (1 << 20) - 1 - // OperationMask is the bitmask to mask out operation indexes - OperationMask = (1 << 12) - 1 - - // LedgerShift is the number of bits to shift an int64 to target the - // ledger component - LedgerShift = 32 - // TransactionShift is the number of bits to shift an int64 to - // target the transaction component - TransactionShift = 12 - // OperationShift is the number of bits to shift an int64 to target - // the operation component - OperationShift = 0 -) - -// New creates a new total order ID -func NewID(ledger int32, tx int32, op int32) *ID { - return &ID{ - LedgerSequence: ledger, - TransactionOrder: tx, - OperationOrder: op, - } -} - -// ToInt64 converts this struct back into an int64 -func (id ID) ToInt64() (result int64) { - - if id.LedgerSequence < 0 { - panic("invalid ledger sequence") - } - - if id.TransactionOrder > TransactionMask { - panic("transaction order overflow") - } - - if id.OperationOrder > OperationMask { - panic("operation order overflow") - } - - result = result | ((int64(id.LedgerSequence) & LedgerMask) << LedgerShift) - result = result | ((int64(id.TransactionOrder) & TransactionMask) << TransactionShift) - result = result | ((int64(id.OperationOrder) & OperationMask) << OperationShift) - return -} - -// TODO: This should be moved into the go monorepo xdr functions -// Or nodeID should just be an xdr.AccountId but the error message would be incorrect -func GetAddress(nodeID xdr.NodeId) (string, bool) { - switch nodeID.Type { - case xdr.PublicKeyTypePublicKeyTypeEd25519: - ed, ok := nodeID.GetEd25519() - if !ok { - return "", false - } - raw := make([]byte, 32) - copy(raw, ed[:]) - encodedAddress, err := strkey.Encode(strkey.VersionByteAccountID, raw) - if err != nil { - return "", false - } - return encodedAddress, true - default: - return "", false - } -} - -func CreateSampleTx(sequence int64, operationCount int) xdr.TransactionEnvelope { - kp, err := keypair.Random() - PanicOnError(err) - - operations := []txnbuild.Operation{} - operationType := &txnbuild.BumpSequence{ - BumpTo: 0, - } - for i := 0; i < operationCount; i++ { - operations = append(operations, operationType) - } - - sourceAccount := txnbuild.NewSimpleAccount(kp.Address(), int64(0)) - tx, err := txnbuild.NewTransaction( - txnbuild.TransactionParams{ - SourceAccount: &sourceAccount, - Operations: operations, - BaseFee: txnbuild.MinBaseFee, - Preconditions: txnbuild.Preconditions{TimeBounds: txnbuild.NewInfiniteTimeout()}, - }, - ) - PanicOnError(err) - - env := tx.ToXDR() - return env -} - -// PanicOnError is a function that panics if the provided error is not nil -func PanicOnError(err error) { - if err != nil { - panic(err) - } -} diff --git a/ingest/ledger/ledger.go b/ingest/ledger/ledger.go new file mode 100644 index 0000000000..fb58dbd9a5 --- /dev/null +++ b/ingest/ledger/ledger.go @@ -0,0 +1,150 @@ +package ledger + +import ( + "encoding/base64" + "time" + + "github.com/stellar/go/toid" + "github.com/stellar/go/xdr" +) + +type Ledger struct { + Ledger xdr.LedgerCloseMeta +} + +func (l Ledger) Sequence() uint32 { + return uint32(l.Ledger.LedgerHeaderHistoryEntry().Header.LedgerSeq) +} + +func (l Ledger) ID() int64 { + return toid.New(int32(l.Ledger.LedgerSequence()), 0, 0).ToInt64() +} + +func (l Ledger) Hash() string { + return l.Ledger.LedgerHeaderHistoryEntry().Hash.HexString() +} + +func (l Ledger) PreviousHash() string { + return l.Ledger.PreviousLedgerHash().HexString() +} + +func (l Ledger) CloseTime() int64 { + return l.Ledger.LedgerCloseTime() +} + +func (l Ledger) ClosedAt() time.Time { + return time.Unix(l.Ledger.LedgerCloseTime(), 0).UTC() +} + +func (l Ledger) TotalCoins() int64 { + return int64(l.Ledger.LedgerHeaderHistoryEntry().Header.TotalCoins) +} + +func (l Ledger) FeePool() int64 { + return int64(l.Ledger.LedgerHeaderHistoryEntry().Header.FeePool) +} + +func (l Ledger) BaseFee() uint32 { + return uint32(l.Ledger.LedgerHeaderHistoryEntry().Header.BaseFee) +} + +func (l Ledger) BaseReserve() uint32 { + return uint32(l.Ledger.LedgerHeaderHistoryEntry().Header.BaseReserve) +} + +func (l Ledger) MaxTxSetSize() uint32 { + return uint32(l.Ledger.LedgerHeaderHistoryEntry().Header.MaxTxSetSize) +} + +func (l Ledger) LedgerVersion() uint32 { + return uint32(l.Ledger.LedgerHeaderHistoryEntry().Header.LedgerVersion) +} + +func (l Ledger) SorobanFeeWrite1Kb() (int64, bool) { + lcmV1, ok := l.Ledger.GetV1() + if !ok { + return 0, false + } + + extV1, ok := lcmV1.Ext.GetV1() + if !ok { + return 0, false + } + + return int64(extV1.SorobanFeeWrite1Kb), true +} + +func (l Ledger) TotalByteSizeOfBucketList() (uint64, bool) { + lcmV1, ok := l.Ledger.GetV1() + if !ok { + return 0, false + } + + return uint64(lcmV1.TotalByteSizeOfBucketList), true +} + +func (l Ledger) NodeID() (string, bool) { + LedgerCloseValueSignature, ok := l.Ledger.LedgerHeaderHistoryEntry().Header.ScpValue.Ext.GetLcValueSignature() + if !ok { + return "", false + + } + return LedgerCloseValueSignature.NodeId.GetAddress() +} + +func (l Ledger) Signature() (string, bool) { + LedgerCloseValueSignature, ok := l.Ledger.LedgerHeaderHistoryEntry().Header.ScpValue.Ext.GetLcValueSignature() + if !ok { + return "", false + } + + return base64.StdEncoding.EncodeToString(LedgerCloseValueSignature.Signature), true +} + +// Add docstring to larger, more complicated functions +func (l Ledger) TransactionCounts() (successTxCount, failedTxCount int32, ok bool) { + var results []xdr.TransactionResultMeta + + transactions := l.Ledger.TransactionEnvelopes() + results = l.Ledger.TxProcessing() + txCount := len(transactions) + if txCount != len(results) { + return 0, 0, false + } + + for i := 0; i < txCount; i++ { + if results[i].Result.Successful() { + successTxCount++ + } else { + failedTxCount++ + } + } + + return successTxCount, failedTxCount, true +} + +// Add docstring to larger, more complicated functions +func (l Ledger) OperationCounts() (operationCount, txSetOperationCount int32, ok bool) { + var results []xdr.TransactionResultMeta + + transactions := l.Ledger.TransactionEnvelopes() + results = l.Ledger.TxProcessing() + + for i, result := range results { + operations := transactions[i].Operations() + numberOfOps := int32(len(operations)) + txSetOperationCount += numberOfOps + + // for successful transactions, the operation count is based on the operations results slice + if result.Result.Successful() { + operationResults, ok := result.Result.OperationResults() + if !ok { + return 0, 0, false + } + + operationCount += int32(len(operationResults)) + } + } + + return operationCount, txSetOperationCount, true +} diff --git a/exp/xdrill/ledger/ledger_test.go b/ingest/ledger/ledger_test.go similarity index 61% rename from exp/xdrill/ledger/ledger_test.go rename to ingest/ledger/ledger_test.go index 7adac0a4cb..3c4b388ec8 100644 --- a/exp/xdrill/ledger/ledger_test.go +++ b/ingest/ledger/ledger_test.go @@ -4,48 +4,59 @@ import ( "testing" "time" - "github.com/stellar/go/exp/xdrill/utils" + "github.com/stellar/go/keypair" + "github.com/stellar/go/txnbuild" "github.com/stellar/go/xdr" "github.com/stretchr/testify/assert" ) func TestLedger(t *testing.T) { - ledger := ledgerTestInput() + ledger := Ledger{ + Ledger: ledgerTestInput(), + } - assert.Equal(t, uint32(30578981), Sequence(ledger)) - assert.Equal(t, int64(131335723340005376), ID(ledger)) - assert.Equal(t, "26932dc4d84b5fabe9ae744cb43ce4c6daccf98c86a991b2a14945b1adac4d59", Hash(ledger)) - assert.Equal(t, "f63c15d0eaf48afbd751a4c4dfade54a3448053c47c5a71d622668ae0cc2a208", PreviousHash(ledger)) - assert.Equal(t, int64(1594584547), CloseTime(ledger)) - assert.Equal(t, time.Time(time.Date(2020, time.July, 12, 20, 9, 7, 0, time.UTC)), ClosedAt(ledger)) - assert.Equal(t, int64(1054439020873472865), TotalCoins(ledger)) - assert.Equal(t, int64(18153766209161), FeePool(ledger)) - assert.Equal(t, uint32(100), BaseFee(ledger)) - assert.Equal(t, uint32(5000000), BaseReserve(ledger)) - assert.Equal(t, uint32(1000), MaxTxSetSize(ledger)) - assert.Equal(t, uint32(13), LedgerVersion(ledger)) + assert.Equal(t, uint32(30578981), ledger.Sequence()) + assert.Equal(t, int64(131335723340005376), ledger.ID()) + assert.Equal(t, "26932dc4d84b5fabe9ae744cb43ce4c6daccf98c86a991b2a14945b1adac4d59", ledger.Hash()) + assert.Equal(t, "f63c15d0eaf48afbd751a4c4dfade54a3448053c47c5a71d622668ae0cc2a208", ledger.PreviousHash()) + assert.Equal(t, int64(1594584547), ledger.CloseTime()) + assert.Equal(t, time.Time(time.Date(2020, time.July, 12, 20, 9, 7, 0, time.UTC)), ledger.ClosedAt()) + assert.Equal(t, int64(1054439020873472865), ledger.TotalCoins()) + assert.Equal(t, int64(18153766209161), ledger.FeePool()) + assert.Equal(t, uint32(100), ledger.BaseFee()) + assert.Equal(t, uint32(5000000), ledger.BaseReserve()) + assert.Equal(t, uint32(1000), ledger.MaxTxSetSize()) + assert.Equal(t, uint32(13), ledger.LedgerVersion()) - freeWrite := SorobanFeeWrite1Kb(ledger) - assert.Equal(t, int64(12), *freeWrite) + var ok bool + var freeWrite int64 + freeWrite, ok = ledger.SorobanFeeWrite1Kb() + assert.Equal(t, true, ok) + assert.Equal(t, int64(12), freeWrite) - bucketSize := TotalByteSizeOfBucketList(ledger) - assert.Equal(t, uint64(56), *bucketSize) + var bucketSize uint64 + bucketSize, ok = ledger.TotalByteSizeOfBucketList() + assert.Equal(t, true, ok) + assert.Equal(t, uint64(56), bucketSize) - nodeID := NodeID(ledger) - assert.Equal(t, "GARAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA76O", *nodeID) + var nodeID string + nodeID, ok = ledger.NodeID() + assert.Equal(t, true, ok) + assert.Equal(t, "GARAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA76O", nodeID) - signature := Signature(ledger) - assert.Equal(t, "9g==", *signature) + var signature string + signature, ok = ledger.Signature() + assert.Equal(t, true, ok) + assert.Equal(t, "9g==", signature) - var ok bool var success int32 var failed int32 - success, failed, ok = TransactionCounts(ledger) + success, failed, ok = ledger.TransactionCounts() assert.Equal(t, true, ok) assert.Equal(t, int32(1), success) assert.Equal(t, int32(1), failed) - success, failed, ok = OperationCounts(ledger) + success, failed, ok = ledger.OperationCounts() assert.Equal(t, true, ok) assert.Equal(t, int32(1), success) assert.Equal(t, int32(13), failed) @@ -100,8 +111,8 @@ func ledgerTestInput() (lcm xdr.LedgerCloseMeta) { Type: 0, TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ Txs: []xdr.TransactionEnvelope{ - utils.CreateSampleTx(0, 3), - utils.CreateSampleTx(1, 10), + createSampleTx(3), + createSampleTx(10), }, }, }, @@ -157,3 +168,37 @@ func ledgerTestInput() (lcm xdr.LedgerCloseMeta) { return lcm } + +func createSampleTx(operationCount int) xdr.TransactionEnvelope { + kp, err := keypair.Random() + panicOnError(err) + + operations := []txnbuild.Operation{} + operationType := &txnbuild.BumpSequence{ + BumpTo: 0, + } + for i := 0; i < operationCount; i++ { + operations = append(operations, operationType) + } + + sourceAccount := txnbuild.NewSimpleAccount(kp.Address(), int64(0)) + tx, err := txnbuild.NewTransaction( + txnbuild.TransactionParams{ + SourceAccount: &sourceAccount, + Operations: operations, + BaseFee: txnbuild.MinBaseFee, + Preconditions: txnbuild.Preconditions{TimeBounds: txnbuild.NewInfiniteTimeout()}, + }, + ) + panicOnError(err) + + env := tx.ToXDR() + return env +} + +// PanicOnError is a function that panics if the provided error is not nil +func panicOnError(err error) { + if err != nil { + panic(err) + } +} diff --git a/xdr/node_id.go b/xdr/node_id.go new file mode 100644 index 0000000000..b3f9c77eb8 --- /dev/null +++ b/xdr/node_id.go @@ -0,0 +1,22 @@ +package xdr + +import "github.com/stellar/go/strkey" + +func (n NodeId) GetAddress() (string, bool) { + switch n.Type { + case PublicKeyTypePublicKeyTypeEd25519: + ed, ok := n.GetEd25519() + if !ok { + return "", false + } + raw := make([]byte, 32) + copy(raw, ed[:]) + encodedAddress, err := strkey.Encode(strkey.VersionByteAccountID, raw) + if err != nil { + return "", false + } + return encodedAddress, true + default: + return "", false + } +}