exp/ingest/io: Add support for ingesting txmeta v2 (#2096)

This commit adds support for ingesting `TransactionMeta` when `V=2` and updates XDR files. As explained by @MonsieurNicolas in #1902. > There is some missing meta in protocol version older than version 10. This impacts transactions with one time signers (that get removed after operations in v9 and earlier; and during signature verification in v10 and later). Those changes are invisible in the meta which may confuse certain ingestion processors that keep track of signers on accounts.
stellar · Jan 15, 2020 · 5201dfb · 5201dfb
1 parent e632f45
commit 5201dfb
Show file tree

Hide file tree

Showing 5 changed files with 1,085 additions and 16 deletions.
diff --git a/exp/ingest/io/ledger_transaction.go b/exp/ingest/io/ledger_transaction.go
@@ -177,8 +177,22 @@ func (t *LedgerTransaction) GetChanges() []Change {
 			)
 			changes = append(changes, opChanges...)
 		}
+	case 2:
+		v2Meta := t.Meta.MustV2()
+		txChangesBefore := getChangesFromLedgerEntryChanges(v2Meta.TxChangesBefore)
+		changes = append(changes, txChangesBefore...)
+
+		for _, operationMeta := range v2Meta.Operations {
+			opChanges := getChangesFromLedgerEntryChanges(
+				operationMeta.Changes,
+			)
+			changes = append(changes, opChanges...)
+		}
+
+		txChangesAfter := getChangesFromLedgerEntryChanges(v2Meta.TxChangesAfter)
+		changes = append(changes, txChangesAfter...)
 	default:
-		panic("Unkown TransactionMeta version")
+		panic("Unsupported TransactionMeta version")
 	}
 
 	return changes

diff --git a/exp/ingest/io/ledger_transaction_test.go b/exp/ingest/io/ledger_transaction_test.go
@@ -89,6 +89,148 @@ func TestFeeAndMetaChangesSeparate(t *testing.T) {
 	assert.Equal(t, metaChanges[0].Post.Data.MustAccount().Balance, xdr.Int64(400))
 }
 
+func TestMetaV2Order(t *testing.T) {
+	tx := LedgerTransaction{
+		Meta: xdr.TransactionMeta{
+			V: 2,
+			V2: &xdr.TransactionMetaV2{
+				TxChangesBefore: xdr.LedgerEntryChanges{
+					xdr.LedgerEntryChange{
+						Type: xdr.LedgerEntryChangeTypeLedgerEntryState,
+						State: &xdr.LedgerEntry{
+							Data: xdr.LedgerEntryData{
+								Type: xdr.LedgerEntryTypeAccount,
+								Account: &xdr.AccountEntry{
+									AccountId: xdr.MustAddress("GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX"),
+									Balance:   100,
+								},
+							},
+						},
+					},
+					xdr.LedgerEntryChange{
+						Type: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
+						Updated: &xdr.LedgerEntry{
+							Data: xdr.LedgerEntryData{
+								Type: xdr.LedgerEntryTypeAccount,
+								Account: &xdr.AccountEntry{
+									AccountId: xdr.MustAddress("GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX"),
+									Balance:   200,
+								},
+							},
+						},
+					},
+					xdr.LedgerEntryChange{
+						Type: xdr.LedgerEntryChangeTypeLedgerEntryState,
+						State: &xdr.LedgerEntry{
+							Data: xdr.LedgerEntryData{
+								Type: xdr.LedgerEntryTypeAccount,
+								Account: &xdr.AccountEntry{
+									AccountId: xdr.MustAddress("GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A"),
+									Balance:   100,
+								},
+							},
+						},
+					},
+					xdr.LedgerEntryChange{
+						Type: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
+						Updated: &xdr.LedgerEntry{
+							Data: xdr.LedgerEntryData{
+								Type: xdr.LedgerEntryTypeAccount,
+								Account: &xdr.AccountEntry{
+									AccountId: xdr.MustAddress("GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A"),
+									Balance:   200,
+								},
+							},
+						},
+					},
+				},
+				Operations: []xdr.OperationMeta{
+					{
+						Changes: xdr.LedgerEntryChanges{
+							xdr.LedgerEntryChange{
+								Type: xdr.LedgerEntryChangeTypeLedgerEntryState,
+								State: &xdr.LedgerEntry{
+									Data: xdr.LedgerEntryData{
+										Type: xdr.LedgerEntryTypeAccount,
+										Account: &xdr.AccountEntry{
+											AccountId: xdr.MustAddress("GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A"),
+											Balance:   300,
+										},
+									},
+								},
+							},
+							xdr.LedgerEntryChange{
+								Type: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
+								Updated: &xdr.LedgerEntry{
+									Data: xdr.LedgerEntryData{
+										Type: xdr.LedgerEntryTypeAccount,
+										Account: &xdr.AccountEntry{
+											AccountId: xdr.MustAddress("GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A"),
+											Balance:   400,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				TxChangesAfter: xdr.LedgerEntryChanges{
+					xdr.LedgerEntryChange{
+						Type: xdr.LedgerEntryChangeTypeLedgerEntryState,
+						State: &xdr.LedgerEntry{
+							Data: xdr.LedgerEntryData{
+								Type: xdr.LedgerEntryTypeAccount,
+								Account: &xdr.AccountEntry{
+									AccountId: xdr.MustAddress("GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX"),
+									Balance:   300,
+								},
+							},
+						},
+					},
+					xdr.LedgerEntryChange{
+						Type: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
+						Updated: &xdr.LedgerEntry{
+							Data: xdr.LedgerEntryData{
+								Type: xdr.LedgerEntryTypeAccount,
+								Account: &xdr.AccountEntry{
+									AccountId: xdr.MustAddress("GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX"),
+									Balance:   400,
+								},
+							},
+						},
+					},
+				},
+			},
+		}}
+
+	metaChanges := tx.GetChanges()
+	assert.Len(t, metaChanges, 4)
+
+	change := metaChanges[0]
+	id := change.Pre.Data.MustAccount().AccountId
+	assert.Equal(t, id.Address(), "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX")
+	assert.Equal(t, change.Pre.Data.MustAccount().Balance, xdr.Int64(100))
+	assert.Equal(t, change.Post.Data.MustAccount().Balance, xdr.Int64(200))
+
+	change = metaChanges[1]
+	id = change.Pre.Data.MustAccount().AccountId
+	assert.Equal(t, id.Address(), "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A")
+	assert.Equal(t, change.Pre.Data.MustAccount().Balance, xdr.Int64(100))
+	assert.Equal(t, change.Post.Data.MustAccount().Balance, xdr.Int64(200))
+
+	change = metaChanges[2]
+	id = change.Pre.Data.MustAccount().AccountId
+	assert.Equal(t, id.Address(), "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A")
+	assert.Equal(t, change.Pre.Data.MustAccount().Balance, xdr.Int64(300))
+	assert.Equal(t, change.Post.Data.MustAccount().Balance, xdr.Int64(400))
+
+	change = metaChanges[3]
+	id = change.Pre.Data.MustAccount().AccountId
+	assert.Equal(t, id.Address(), "GACMZD5VJXTRLKVET72CETCYKELPNCOTTBDC6DHFEUPLG5DHEK534JQX")
+	assert.Equal(t, change.Pre.Data.MustAccount().Balance, xdr.Int64(300))
+	assert.Equal(t, change.Post.Data.MustAccount().Balance, xdr.Int64(400))
+}
+
 func TestChangeAccountChangedExceptSignersLastModifiedLedgerSeq(t *testing.T) {
 	change := Change{
 		Type: xdr.LedgerEntryTypeAccount,

diff --git a/xdr/Stellar-ledger.x b/xdr/Stellar-ledger.x
@@ -306,6 +306,15 @@ struct TransactionMetaV1
     OperationMeta operations<>;   // meta for each operation
 };
 
+struct TransactionMetaV2
+{
+    LedgerEntryChanges txChangesBefore; // tx level changes before operations
+                                        // are applied if any
+    OperationMeta operations<>;         // meta for each operation
+    LedgerEntryChanges txChangesAfter;  // tx level changes after operations are
+                                        // applied if any
+};
+
 // this is the meta produced when applying transactions
 // it does not include pre-apply updates such as fees
 union TransactionMeta switch (int v)
@@ -314,5 +323,50 @@ case 0:
     OperationMeta operations<>;
 case 1:
     TransactionMetaV1 v1;
+case 2:
+    TransactionMetaV2 v2;
+};
+
+// This struct groups together changes on a per transaction basis
+// note however that fees and transaction application are done in separate
+// phases
+struct TransactionResultMeta
+{
+    TransactionResultPair result;
+    LedgerEntryChanges feeProcessing;
+    TransactionMeta txApplyProcessing;
+};
+
+// this represents a single upgrade that was performed as part of a ledger
+// upgrade
+struct UpgradeEntryMeta
+{
+    LedgerUpgrade upgrade;
+    LedgerEntryChanges changes;
+};
+
+struct LedgerCloseMetaV0
+{
+    LedgerHeaderHistoryEntry ledgerHeader;
+    // NB: txSet is sorted in "Hash order"
+    TransactionSet txSet;
+
+    // NB: transactions are sorted in apply order here
+    // fees for all transactions are processed first
+    // followed by applying transactions
+    TransactionResultMeta txProcessing<>;
+
+    // upgrades are applied last
+    UpgradeEntryMeta upgradesProcessing<>;
+
+    // other misc information attached to the ledger close
+    SCPHistoryEntry scpInfo<>;
 };
+
+union LedgerCloseMeta switch (int v)
+{
+case 0:
+     LedgerCloseMetaV0 v0;
+};
+
 }
diff --git a/xdr/Stellar-overlay.x b/xdr/Stellar-overlay.x
@@ -90,7 +90,10 @@ enum MessageType
     GET_SCP_STATE = 12,
 
     // new messages
-    HELLO = 13
+    HELLO = 13,
+
+    SURVEY_REQUEST = 14,
+    SURVEY_RESPONSE = 15
 };
 
 struct DontHave
@@ -99,6 +102,80 @@ struct DontHave
     uint256 reqHash;
 };
 
+enum SurveyMessageCommandType
+{
+    SURVEY_TOPOLOGY = 0
+};
+
+struct SurveyRequestMessage
+{
+    NodeID surveyorPeerID;
+    NodeID surveyedPeerID;
+    uint32 ledgerNum;
+    Curve25519Public encryptionKey;
+    SurveyMessageCommandType commandType;
+};
+
+struct SignedSurveyRequestMessage
+{
+    Signature requestSignature;
+    SurveyRequestMessage request;
+};
+
+typedef opaque EncryptedBody<64000>;
+struct SurveyResponseMessage
+{
+    NodeID surveyorPeerID;
+    NodeID surveyedPeerID;
+    uint32 ledgerNum;
+    SurveyMessageCommandType commandType;
+    EncryptedBody encryptedBody;
+};
+
+struct SignedSurveyResponseMessage
+{
+    Signature responseSignature;
+    SurveyResponseMessage response;
+};
+
+struct PeerStats
+{
+    NodeID id;
+    string versionStr<100>;
+    uint64 messagesRead;
+    uint64 messagesWritten;
+    uint64 bytesRead;
+    uint64 bytesWritten;
+    uint64 secondsConnected;
+
+    uint64 uniqueFloodBytesRecv;
+    uint64 duplicateFloodBytesRecv;
+    uint64 uniqueFetchBytesRecv;
+    uint64 duplicateFetchBytesRecv;
+
+    uint64 uniqueFloodMessageRecv;
+    uint64 duplicateFloodMessageRecv;
+    uint64 uniqueFetchMessageRecv;
+    uint64 duplicateFetchMessageRecv;
+};
+
+typedef PeerStats PeerStatList<25>;
+
+struct TopologyResponseBody
+{
+    PeerStatList inboundPeers;
+    PeerStatList outboundPeers;
+
+    uint32 totalInboundPeerCount;
+    uint32 totalOutboundPeerCount;
+};
+
+union SurveyResponseBody switch (SurveyMessageCommandType type)
+{
+    case SURVEY_TOPOLOGY:
+        TopologyResponseBody topologyResponseBody;
+};
+
 union StellarMessage switch (MessageType type)
 {
 case ERROR_MSG:
@@ -122,6 +199,12 @@ case TX_SET:
 case TRANSACTION:
     TransactionEnvelope transaction;
 
+case SURVEY_REQUEST:
+    SignedSurveyRequestMessage signedSurveyRequestMessage;
+
+case SURVEY_RESPONSE:
+    SignedSurveyResponseMessage signedSurveyResponseMessage;
+
 // SCP
 case GET_SCP_QUORUMSET:
     uint256 qSetHash;