diff --git a/docs/db-schema.md b/docs/db-schema.md index 51629a1b70..380fdecacb 100644 --- a/docs/db-schema.md +++ b/docs/db-schema.md @@ -51,8 +51,8 @@ homedomain | VARCHAR(44) | (BASE64) thresholds | TEXT | (BASE64) flags | INT NOT NULL | lastmodified | INT NOT NULL | lastModifiedLedgerSeq -buyingliabilities | BIGINT CHECK (buyingliabilities >= 0) | -sellingliabilities | BIGINT CHECK (sellingliabilities >= 0) | +extension | TEXT | Extension specific to AccountEntry (XDR) +ledgerext | TEXT | Extension common to all LedgerEntry types (XDR) signers | TEXT | (XDR) ## offers @@ -73,6 +73,8 @@ priced | INT NOT NULL | Price.d price | DOUBLE PRECISION NOT NULL | computed price n/d, used for ordering offers flags | INT NOT NULL | lastmodified | INT NOT NULL | lastModifiedLedgerSeq +extension | TEXT | Extension specific to OfferEntry (XDR) +ledgerext | TEXT | Extension common to all LedgerEntry types (XDR) (offerid) | PRIMARY KEY | ## trustlines @@ -91,8 +93,8 @@ tlimit | BIGINT NOT NULL DEFAULT 0 CHECK (tlimit >= 0) | limit balance | BIGINT NOT NULL DEFAULT 0 CHECK (balance >= 0) | flags | INT NOT NULL | lastmodified | INT NOT NULL | lastModifiedLedgerSeq -buyingliabilities | BIGINT CHECK (buyingliabilities >= 0) | -sellingliabilities | BIGINT CHECK (sellingliabilities >= 0) | +extension | TEXT | Extension specific to TrustLineEntry (XDR) +ledgerext | TEXT | Extension common to all LedgerEntry types (XDR) (accountid, issuer, assetcode) | PRIMARY KEY | ## accountdata @@ -107,6 +109,8 @@ accountid | VARCHAR(56) NOT NULL | (STRKEY) dataname | VARCHAR(88) NOT NULL | (BASE64) datavalue | VARCHAR(112) NOT NULL | (BASE64) lastmodified | INT NOT NULL | lastModifiedLedgerSeq +extension | TEXT | Extension specific to DataEntry (XDR) +ledgerext | TEXT | Extension common to all LedgerEntry types (XDR) (accountid, dataname) | PRIMARY KEY | ## txhistory diff --git a/src/database/Database.cpp b/src/database/Database.cpp index 272914ea36..79aafd3015 100644 --- a/src/database/Database.cpp +++ b/src/database/Database.cpp @@ -9,10 +9,13 @@ #include "main/Application.h" #include "main/Config.h" #include "overlay/StellarXDR.h" +#include "util/Decoder.h" #include "util/GlobalChecks.h" #include "util/Logging.h" #include "util/Timer.h" #include "util/types.h" +#include +#include #include "bucket/BucketManager.h" #include "herder/HerderPersistence.h" @@ -30,8 +33,10 @@ #include "medida/counter.h" #include "medida/metrics_registry.h" #include "medida/timer.h" +#include "xdr/Stellar-ledger-entries.h" #include +#include #ifdef USE_POSTGRES #include #endif @@ -57,7 +62,7 @@ bool Database::gDriversRegistered = false; // smallest schema version supported static unsigned long const MIN_SCHEMA_VERSION = 9; -static unsigned long const SCHEMA_VERSION = 12; +static unsigned long const SCHEMA_VERSION = 13; // These should always match our compiled version precisely, since we are // using a bundled version to get access to carray(). But in case someone @@ -252,6 +257,33 @@ Database::applySchemaUpgrade(unsigned long vers) // the accountbalances index around. mSession << "DROP INDEX IF EXISTS accountbalances"; break; + case 13: + if (!mApp.getConfig().MODE_USES_IN_MEMORY_LEDGER) + { + // Add columns for the LedgerEntry extension to each of + // the tables that stores a type of ledger entry. + addTextColumn("accounts", "ledgerext"); + addTextColumn("trustlines", "ledgerext"); + addTextColumn("accountdata", "ledgerext"); + addTextColumn("offers", "ledgerext"); + // Absorb the explicit columns of the extension fields of + // AccountEntry and TrustLineEntry into single opaque + // blobs of XDR each of which represents an entire extension. + convertAccountExtensionsToOpaqueXDR(); + convertTrustLineExtensionsToOpaqueXDR(); + // Neither earlier schema versions nor the one that we're upgrading + // to now had any extension columns in the offers or accountdata + // tables, but we add columns in this version, even though we're not + // going to use them for anything other than writing out opaque + // base64-encoded empty v0 XDR extensions, so that, as with the + // other LedgerEntry extensions, we'll be able to add such + // extensions in the future without bumping the database schema + // version, writing any upgrade code, or changing the SQL that reads + // and writes those tables. + addTextColumn("offers", "extension"); + addTextColumn("accountdata", "extension"); + } + break; default: throw std::runtime_error("Unknown DB schema version"); } @@ -277,6 +309,7 @@ Database::upgradeToCurrentSchema() std::to_string(SCHEMA_VERSION)); throw std::runtime_error(s); } + actBeforeDBSchemaUpgrade(); while (vers < SCHEMA_VERSION) { ++vers; @@ -289,6 +322,189 @@ Database::upgradeToCurrentSchema() assert(vers == SCHEMA_VERSION); } +void +Database::addTextColumn(std::string const& table, std::string const& column) +{ + std::string addColumnStr("ALTER TABLE " + table + " ADD " + column + + " TEXT;"); + CLOG(INFO, "Database") << "Adding column '" + << "' to table '" << table << "'"; + mSession << addColumnStr; +} + +void +Database::dropNullableColumn(std::string const& table, + std::string const& column) +{ + // SQLite doesn't give us a way of dropping a column with a single + // SQL command. If we need it in production, we could re-create the + // table without the column and drop the old one. Since we currently + // use SQLite only for testing and PostgreSQL in production, we simply + // leave the unused columm around in SQLite at the moment, and NULL + // out all of the cells in that column. + if (!isSqlite()) + { + std::string dropColumnStr("ALTER TABLE " + table + " DROP COLUMN " + + column); + CLOG(INFO, "Database") << "Dropping column '" << column + << "' from table '" << table << "'"; + + mSession << dropColumnStr; + } + else + { + std::string nullColumnStr("UPDATE " + table + " SET " + column + + " = NULL"); + CLOG(INFO, "Database") << "Setting all cells of column '" << column + << "' in table '" << table << "' to NULL"; + + mSession << nullColumnStr; + } +} + +std::string +Database::getOldLiabilitySelect(std::string const& table, + std::string const& fields) +{ + return fmt::format("SELECT {}, " + "buyingliabilities, sellingliabilities FROM {} WHERE " + "buyingliabilities IS NOT NULL OR " + "sellingliabilities IS NOT NULL", + fields, table); +} + +void +Database::convertAccountExtensionsToOpaqueXDR() +{ + addTextColumn("accounts", "extension"); + copyIndividualAccountExtensionFieldsToOpaqueXDR(); + dropNullableColumn("accounts", "buyingliabilities"); + dropNullableColumn("accounts", "sellingliabilities"); +} + +void +Database::convertTrustLineExtensionsToOpaqueXDR() +{ + addTextColumn("trustlines", "extension"); + copyIndividualTrustLineExtensionFieldsToOpaqueXDR(); + dropNullableColumn("trustlines", "buyingliabilities"); + dropNullableColumn("trustlines", "sellingliabilities"); +} + +void +Database::copyIndividualAccountExtensionFieldsToOpaqueXDR() +{ + std::string const tableStr = "accounts"; + + CLOG(INFO, "Database") << "Updating extension schema for " << tableStr; + + // + struct Fields + { + std::string mAccountID; + std::string mExtension; + }; + + std::string const fieldsStr = "accountid"; + std::string const selectStr = getOldLiabilitySelect(tableStr, fieldsStr); + auto makeFields = [](soci::row const& row) { + AccountEntry::_ext_t extension; + // getOldLiabilitySelect() places the buying and selling extension + // column names after the key field in the SQL select string. + extension.v(1); + extension.v1().liabilities.buying = row.get(1); + extension.v1().liabilities.selling = row.get(2); + return Fields{.mAccountID = row.get(0), + .mExtension = + decoder::encode_b64(xdr::xdr_to_opaque(extension))}; + }; + + std::string const updateStr = + "UPDATE accounts SET extension = :ext WHERE accountID = :id"; + auto prepUpdate = [](soci::statement& st_update, Fields const& data) { + st_update.exchange(soci::use(data.mExtension)), + st_update.exchange(soci::use(data.mAccountID)); + }; + + auto postUpdate = [](long long const affected_rows, Fields const& data) { + if (affected_rows != 1) + { + throw std::runtime_error(fmt::format( + "{}: updating account with account ID {} affected {} row(s) ", + __func__, data.mAccountID, affected_rows)); + } + }; + + size_t numUpdated = selectUpdateMap( + *this, selectStr, makeFields, updateStr, prepUpdate, postUpdate); + + CLOG(INFO, "Database") << __func__ << ": updated " << numUpdated + << " records(s) with liabilities in " << tableStr + << " table"; +} + +void +Database::copyIndividualTrustLineExtensionFieldsToOpaqueXDR() +{ + std::string const tableStr = "trustlines"; + + CLOG(INFO, "Database") << __func__ << ": updating extension schema for " + << tableStr; + + // + struct Fields + { + std::string mAccountID; + std::string mIssuerID; + std::string mAssetID; + std::string mExtension; + }; + + std::string const fieldsStr = "accountid, issuer, assetcode"; + std::string const selectStr = getOldLiabilitySelect(tableStr, fieldsStr); + auto makeFields = [](soci::row const& row) { + TrustLineEntry::_ext_t extension; + // getOldLiabilitySelect() places the buying and selling extension + // column names after the three key fields in the SQL select string. + extension.v(1); + extension.v1().liabilities.buying = row.get(3); + extension.v1().liabilities.selling = row.get(4); + return Fields{.mAccountID = row.get(0), + .mIssuerID = row.get(1), + .mAssetID = row.get(2), + .mExtension = + decoder::encode_b64(xdr::xdr_to_opaque(extension))}; + }; + + std::string const updateStr = + "UPDATE trustlines SET extension = :ext WHERE accountID = :id " + "AND issuer = :issuer_id AND assetcode = :asset_id"; + auto prepUpdate = [](soci::statement& st_update, Fields const& data) { + st_update.exchange(soci::use(data.mExtension)); + st_update.exchange(soci::use(data.mAccountID)); + st_update.exchange(soci::use(data.mIssuerID)); + st_update.exchange(soci::use(data.mAssetID)); + }; + + auto postUpdate = [](long long const affected_rows, Fields const& data) { + if (affected_rows != 1) + { + throw std::runtime_error(fmt::format( + "{}: updating trustline with account ID {}, issuer {}, and " + "asset {} affected {} row(s)", + __func__, data.mAccountID, data.mIssuerID, data.mAssetID, + affected_rows)); + } + }; + + size_t numUpdated = selectUpdateMap( + *this, selectStr, makeFields, updateStr, prepUpdate, postUpdate); + + CLOG(INFO, "Database") << __func__ << ": updated " << numUpdated + << " records(s) with liabilities in " << tableStr + << " table"; +} + void Database::putSchemaVersion(unsigned long vers) { diff --git a/src/database/Database.h b/src/database/Database.h index 2d809d2ca4..826cb8b0d8 100644 --- a/src/database/Database.h +++ b/src/database/Database.h @@ -7,11 +7,14 @@ #include "database/DatabaseTypeSpecificOperation.h" #include "medida/timer_context.h" #include "overlay/StellarXDR.h" +#include "util/Decoder.h" #include "util/NonCopyable.h" #include "util/Timer.h" +#include #include #include #include +#include namespace medida { @@ -103,11 +106,31 @@ class Database : NonMovableOrCopyable static void registerDrivers(); void applySchemaUpgrade(unsigned long vers); + // Convert the accounts table from using explicit entries for + // extension fields into storing the entire extension as opaque XDR. + void convertAccountExtensionsToOpaqueXDR(); + void copyIndividualAccountExtensionFieldsToOpaqueXDR(); + + std::string getOldLiabilitySelect(std::string const& table, + std::string const& fields); + void addTextColumn(std::string const& table, std::string const& column); + void dropNullableColumn(std::string const& table, + std::string const& column); + + // Convert the trustlines table from using explicit entries for + // extension fields into storing the entire extension as opaque XDR. + void convertTrustLineExtensionsToOpaqueXDR(); + void copyIndividualTrustLineExtensionFieldsToOpaqueXDR(); + public: // Instantiate object and connect to app.getConfig().DATABASE; // if there is a connection error, this will throw. Database(Application& app); + virtual ~Database() + { + } + // Return a crude meter of total queries to the db, for use in // overlay/LoadManager. medida::Meter& getQueryMeter(); @@ -196,6 +219,16 @@ class Database : NonMovableOrCopyable // Access the optional SOCI connection pool available for worker // threads. Throws an error if !canUsePool(). soci::connection_pool& getPool(); + + protected: + // Give clients the opportunity to perform operations on databases while + // they're still using old schemas (prior to the upgrade that occurs either + // immediately after database creation or after loading a version of + // stellar-core that introduces a new schema). + virtual void + actBeforeDBSchemaUpgrade() + { + } }; template @@ -230,4 +263,89 @@ class DBTimeExcluder : NonCopyable DBTimeExcluder(Application& mApp); ~DBTimeExcluder(); }; + +// Select a set of records using a client-defined query string, then map +// each record into an element of a client-defined datatype by applying a +// client-defined function (the records are accumulated in the "out" +// vector). +template +void +selectMap(Database& db, std::string const& selectStr, + std::function makeT, std::vector& out) +{ + soci::rowset rs = (db.getSession().prepare << selectStr); + + std::transform(rs.begin(), rs.end(), std::back_inserter(out), makeT); +} + +// Map each element in the given vector of a client-defined datatype into a +// SQL update command by applying a client-defined function, then send those +// update strings to the database. +// +// The "postUpdate" function receives the number of records affected +// by the given update, as well as the element of the client-defined +// datatype which generated that update. +template +void updateMap(Database& db, std::vector const& in, + std::string const& updateStr, + std::function prepUpdate, + std::function postUpdate); +template +void +updateMap(Database& db, std::vector const& in, std::string const& updateStr, + std::function prepUpdate, + std::function postUpdate) +{ + auto st_update = db.getPreparedStatement(updateStr).statement(); + + for (auto& recT : in) + { + prepUpdate(st_update, recT); + st_update.define_and_bind(); + st_update.execute(true); + auto affected_rows = st_update.get_affected_rows(); + st_update.clean_up(false); + postUpdate(affected_rows, recT); + } +} + +// The composition of updateMap() following selectMap(). +// +// Returns the number of records selected by selectMap() (all of which were +// then passed through updateMap() before the selectUpdateMap() call +// returned). +template +size_t +selectUpdateMap(Database& db, std::string const& selectStr, + std::function makeT, + std::string const& updateStr, + std::function prepUpdate, + std::function postUpdate) +{ + std::vector vecT; + + selectMap(db, selectStr, makeT, vecT); + updateMap(db, vecT, updateStr, prepUpdate, postUpdate); + + return vecT.size(); +} + +template +void +decodeOpaqueXDR(std::string const& in, T& out) +{ + std::vector opaque; + decoder::decode_b64(in, opaque); + xdr::xdr_from_opaque(opaque, out); +} + +template +void +decodeOpaqueXDR(std::string const& in, soci::indicator const& ind, T& out) +{ + if (ind == soci::i_ok) + { + decodeOpaqueXDR(in, out); + } +} } diff --git a/src/database/test/DatabaseTests.cpp b/src/database/test/DatabaseTests.cpp index 7077865faf..e8c8892482 100644 --- a/src/database/test/DatabaseTests.cpp +++ b/src/database/test/DatabaseTests.cpp @@ -4,16 +4,22 @@ #include "util/asio.h" #include "crypto/Hex.h" +#include "crypto/KeyUtils.h" #include "database/Database.h" +#include "ledger/LedgerTxn.h" +#include "ledger/test/LedgerTestUtils.h" #include "lib/catch.hpp" #include "main/Application.h" #include "main/Config.h" #include "test/TestUtils.h" #include "test/test.h" +#include "util/Decoder.h" #include "util/Logging.h" #include "util/Math.h" #include "util/Timer.h" #include "util/TmpDir.h" +#include "util/optional.h" +#include #include using namespace stellar; @@ -349,3 +355,345 @@ TEST_CASE("schema test", "[db]") auto av = db.getAppSchemaVersion(); REQUIRE(dbv == av); } + +class SchemaUpgradeTestApplication : public TestApplication +{ + public: + using PreUpgradeFunc = std::function; + + private: + class SchemaUpgradeTestDatabase : public Database + { + SchemaUpgradeTestApplication& mApp; + PreUpgradeFunc const mPreUpgradeFunc; + + public: + SchemaUpgradeTestDatabase(SchemaUpgradeTestApplication& app, + PreUpgradeFunc preUpgradeFunc) + : Database(app), mApp(app), mPreUpgradeFunc(preUpgradeFunc) + { + } + + virtual void + actBeforeDBSchemaUpgrade() override + { + if (mPreUpgradeFunc) + { + mPreUpgradeFunc(mApp); + } + } + }; + + PreUpgradeFunc const mPreUpgradeFunc; + + public: + SchemaUpgradeTestApplication(VirtualClock& clock, Config const& cfg, + PreUpgradeFunc preUpgradeFunc) + : TestApplication(clock, cfg), mPreUpgradeFunc(preUpgradeFunc) + { + } + + virtual std::unique_ptr + createDatabase() override + { + return std::make_unique(*this, + mPreUpgradeFunc); + } +}; + +// This tests upgrading from MIN_SCHEMA_VERSION to SCHEMA_VERSION by creating +// ledger entries with the old MIN_SCHEMA_VERSION after database creation, then +// validating their contents after the upgrade to SCHEMA_VERSION. +TEST_CASE("schema upgrade test", "[db]") +{ + using OptLiabilities = stellar::optional; + + auto addOneOldSchemaAccount = [](SchemaUpgradeTestApplication& app, + AccountEntry const& ae) { + auto& session = app.getDatabase().getSession(); + auto accountIDStr = KeyUtils::toStrKey(ae.accountID); + auto inflationDestStr = + ae.inflationDest ? KeyUtils::toStrKey(*ae.inflationDest) + : ""; + auto inflationDestInd = ae.inflationDest ? soci::i_ok : soci::i_null; + auto homeDomainStr = decoder::encode_b64(ae.homeDomain); + auto signersStr = decoder::encode_b64(xdr::xdr_to_opaque(ae.signers)); + auto thresholdsStr = decoder::encode_b64(ae.thresholds); + bool const liabilitiesPresent = (ae.ext.v() >= 1); + int64_t buyingLiabilities = + liabilitiesPresent ? ae.ext.v1().liabilities.buying : 0; + int64_t sellingLiabilities = + liabilitiesPresent ? ae.ext.v1().liabilities.selling : 0; + soci::indicator liabilitiesInd = + liabilitiesPresent ? soci::i_ok : soci::i_null; + + soci::transaction tx(session); + + // Use raw SQL to perform database operations, since we're writing in an + // old database format, and calling standard interfaces to create + // accounts or trustlines would use SQL corresponding to the new format + // to which we'll soon upgrade. + session << "INSERT INTO accounts ( " + "accountid, balance, seqnum, numsubentries, inflationdest," + "homedomain, thresholds, signers, flags, lastmodified, " + "buyingliabilities, sellingliabilities " + ") VALUES ( " + ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10, " + ":v11 " + ")", + soci::use(accountIDStr), soci::use(ae.balance), + soci::use(ae.seqNum), soci::use(ae.numSubEntries), + soci::use(inflationDestStr, inflationDestInd), + soci::use(homeDomainStr), soci::use(thresholdsStr), + soci::use(signersStr), soci::use(ae.flags), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()), + soci::use(buyingLiabilities, liabilitiesInd), + soci::use(sellingLiabilities, liabilitiesInd); + + tx.commit(); + }; + + auto addOneOldSchemaTrustLine = [](SchemaUpgradeTestApplication& app, + TrustLineEntry const& tl) { + auto& session = app.getDatabase().getSession(); + std::string accountIDStr, issuerStr, assetCodeStr; + getTrustLineStrings(tl.accountID, tl.asset, accountIDStr, issuerStr, + assetCodeStr); + int32_t assetType = tl.asset.type(); + bool const liabilitiesPresent = (tl.ext.v() >= 1); + int64_t buyingLiabilities = + liabilitiesPresent ? tl.ext.v1().liabilities.buying : 0; + int64_t sellingLiabilities = + liabilitiesPresent ? tl.ext.v1().liabilities.selling : 0; + soci::indicator liabilitiesInd = + liabilitiesPresent ? soci::i_ok : soci::i_null; + + soci::transaction tx(session); + + session << "INSERT INTO trustlines ( " + "accountid, assettype, issuer, assetcode," + "tlimit, balance, flags, lastmodified, " + "buyingliabilities, sellingliabilities " + ") VALUES ( " + ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9 " + ")", + soci::use(accountIDStr), soci::use(assetType), soci::use(issuerStr), + soci::use(assetCodeStr), soci::use(tl.limit), soci::use(tl.balance), + soci::use(tl.flags), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()), + soci::use(buyingLiabilities, liabilitiesInd), + soci::use(sellingLiabilities, liabilitiesInd); + + tx.commit(); + }; + + auto addOneOldSchemaDataEntry = [](SchemaUpgradeTestApplication& app, + DataEntry const& de) { + auto& session = app.getDatabase().getSession(); + auto accountIDStr = KeyUtils::toStrKey(de.accountID); + auto dataNameStr = decoder::encode_b64(de.dataName); + auto dataValueStr = decoder::encode_b64(de.dataValue); + + soci::transaction tx(session); + + session << "INSERT INTO accountdata ( " + "accountid, dataname, datavalue, lastmodified " + ") VALUES ( :id, :v1, :v2, :v3 )", + soci::use(accountIDStr), soci::use(dataNameStr), + soci::use(dataValueStr), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()); + + tx.commit(); + }; + + auto addOneOldSchemaOfferEntry = [](SchemaUpgradeTestApplication& app, + OfferEntry const& oe) { + auto& session = app.getDatabase().getSession(); + auto sellerIDStr = KeyUtils::toStrKey(oe.sellerID); + auto sellingStr = decoder::encode_b64(xdr::xdr_to_opaque(oe.selling)); + auto buyingStr = decoder::encode_b64(xdr::xdr_to_opaque(oe.buying)); + double price = double(oe.price.n) / double(oe.price.d); + + soci::transaction tx(session); + + session << "INSERT INTO offers ( " + "sellerid, offerid, sellingasset, buyingasset, " + "amount, pricen, priced, price, flags, lastmodified " + ") VALUES ( " + ":v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10 " + ")", + soci::use(sellerIDStr), soci::use(oe.offerID), + soci::use(sellingStr), soci::use(buyingStr), soci::use(oe.amount), + soci::use(oe.price.n), soci::use(oe.price.d), soci::use(price), + soci::use(oe.flags), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()); + + tx.commit(); + }; + + auto prepOldSchemaDB = + [addOneOldSchemaAccount, addOneOldSchemaTrustLine, + addOneOldSchemaDataEntry, + addOneOldSchemaOfferEntry](SchemaUpgradeTestApplication& app, + std::vector const& aes, + std::vector const& tls, + DataEntry const& de, OfferEntry const& oe) { + for (auto ae : aes) + { + addOneOldSchemaAccount(app, ae); + } + + for (auto tl : tls) + { + addOneOldSchemaTrustLine(app, tl); + } + + addOneOldSchemaDataEntry(app, de); + addOneOldSchemaOfferEntry(app, oe); + }; + + auto testOneDBMode = [prepOldSchemaDB](Config::TestDbMode const dbMode) { + // A vector of optional Liabilities entries, for each of which the test + // will generate a valid account. + auto const accOptLiabilities = { + nullopt(), + make_optional(Liabilities{12, 17}), + make_optional(Liabilities{4, 0}), + nullopt(), + make_optional(Liabilities{3, 0}), + make_optional(Liabilities{11, 11}), + make_optional(Liabilities{0, 0})}; + + // A vector of optional Liabilities entries, for each of which the test + // will generate a valid trustline. + auto const tlOptLiabilities = { + make_optional(Liabilities{1, 0}), + make_optional(Liabilities{0, 6}), + nullopt(), + make_optional(Liabilities{0, 0}), + make_optional(Liabilities{5, 8}), + nullopt()}; + + // Generate from each of the optional liabilities in accOptLiabilities a + // new valid account. + std::vector accountEntries; + std::transform( + accOptLiabilities.begin(), accOptLiabilities.end(), + std::back_inserter(accountEntries), [](OptLiabilities const& aol) { + AccountEntry ae = LedgerTestUtils::generateValidAccountEntry(); + if (aol) + { + ae.ext.v(1); + ae.ext.v1().liabilities = *aol; + } + else + { + ae.ext.v(0); + } + return ae; + }); + + // Generate from each of the optional liabilities in tlOptLiabilities a + // new valid trustline. + std::vector trustLineEntries; + std::transform(tlOptLiabilities.begin(), tlOptLiabilities.end(), + std::back_inserter(trustLineEntries), + [](OptLiabilities const& tlol) { + TrustLineEntry tl = + LedgerTestUtils::generateValidTrustLineEntry(); + if (tlol) + { + tl.ext.v(1); + tl.ext.v1().liabilities = *tlol; + } + else + { + tl.ext.v(0); + } + return tl; + }); + + // Create a data entry to test that its extension and ledger + // entry extension have the default (empty-XDR-union) values. + auto de = LedgerTestUtils::generateValidDataEntry(); + REQUIRE(de.ext.v() == 0); + REQUIRE(de.ext == DataEntry::_ext_t()); + + // Create an offer entry to test that its extension and ledger + // entry extension have the default (empty-XDR-union) values. + auto oe = LedgerTestUtils::generateValidOfferEntry(); + REQUIRE(oe.ext.v() == 0); + REQUIRE(oe.ext == OfferEntry::_ext_t()); + + // Create the application, with the code above that inserts old-schema + // accounts and trustlines into the database injected between database + // creation and upgrade. + Config const& cfg = getTestConfig(0, dbMode); + VirtualClock clock; + Application::pointer app = + createTestApplication( + clock, cfg, + [prepOldSchemaDB, accountEntries, trustLineEntries, de, + oe](SchemaUpgradeTestApplication& sapp) { + prepOldSchemaDB(sapp, accountEntries, trustLineEntries, de, + oe); + }); + app->start(); + + // Validate that the accounts and trustlines have the expected + // liabilities and ledger entry extensions now that the database upgrade + // has completed. + + LedgerTxn ltx(app->getLedgerTxnRoot()); + + for (auto ae : accountEntries) + { + LedgerEntry entry; + entry.data.type(ACCOUNT); + entry.data.account() = ae; + LedgerKey key = LedgerEntryKey(entry); + auto aeUpgraded = ltx.load(key); + REQUIRE(aeUpgraded.current() == entry); + } + + for (auto tl : trustLineEntries) + { + LedgerEntry entry; + entry.data.type(TRUSTLINE); + entry.data.trustLine() = tl; + LedgerKey key = LedgerEntryKey(entry); + auto tlUpgraded = ltx.load(key); + REQUIRE(tlUpgraded.current() == entry); + } + + { + LedgerEntry entry; + entry.data.type(DATA); + entry.data.data() = de; + LedgerKey key = LedgerEntryKey(entry); + auto deUpgraded = ltx.load(key); + REQUIRE(deUpgraded.current() == entry); + } + + { + LedgerEntry entry; + entry.data.type(OFFER); + entry.data.offer() = oe; + LedgerKey key = LedgerEntryKey(entry); + auto oeUpgraded = ltx.load(key); + REQUIRE(oeUpgraded.current() == entry); + } + }; + + for (auto dbMode : + {Config::TESTDB_IN_MEMORY_SQLITE, Config::TESTDB_ON_DISK_SQLITE +#ifdef USE_POSTGRES + , + Config::TESTDB_POSTGRESQL +#endif // USE_POSTGRES + }) + { + testOneDBMode(dbMode); + } +} diff --git a/src/ledger/LedgerTxn.h b/src/ledger/LedgerTxn.h index 15430e56d3..69a6af90ed 100644 --- a/src/ledger/LedgerTxn.h +++ b/src/ledger/LedgerTxn.h @@ -330,6 +330,10 @@ class WorstBestOfferIterator std::shared_ptr const& offerDescriptor() const; }; +void getTrustLineStrings(AccountID const& accountID, Asset const& asset, + std::string& accountIDStr, std::string& issuerStr, + std::string& assetCodeStr); + // An abstraction for an object that can be the parent of an AbstractLedgerTxn // (discussed below). Allows children to commit atomically to the parent. Has no // notion of a LedgerTxnEntry or LedgerTxnHeader (discussed respectively in diff --git a/src/ledger/LedgerTxnAccountSQL.cpp b/src/ledger/LedgerTxnAccountSQL.cpp index ba46db21ed..6ef78771ee 100644 --- a/src/ledger/LedgerTxnAccountSQL.cpp +++ b/src/ledger/LedgerTxnAccountSQL.cpp @@ -26,20 +26,21 @@ LedgerTxnRoot::Impl::loadAccount(LedgerKey const& key) const std::string inflationDest, homeDomain, thresholds, signers; soci::indicator inflationDestInd, signersInd; - Liabilities liabilities; - soci::indicator buyingLiabilitiesInd, sellingLiabilitiesInd; + std::string extensionStr; + soci::indicator extensionInd; + std::string ledgerExtStr; + soci::indicator ledgerExtInd; LedgerEntry le; le.data.type(ACCOUNT); auto& account = le.data.account(); - auto prep = - mDatabase.getPreparedStatement("SELECT balance, seqnum, numsubentries, " - "inflationdest, homedomain, thresholds, " - "flags, lastmodified, " - "buyingliabilities, sellingliabilities, " - "signers " - "FROM accounts WHERE accountid=:v1"); + auto prep = mDatabase.getPreparedStatement( + "SELECT balance, seqnum, numsubentries, " + "inflationdest, homedomain, thresholds, " + "flags, lastmodified, " + "signers, extension, " + "ledgerext FROM accounts WHERE accountid=:v1"); auto& st = prep.statement(); st.exchange(soci::into(account.balance)); st.exchange(soci::into(account.seqNum)); @@ -49,9 +50,9 @@ LedgerTxnRoot::Impl::loadAccount(LedgerKey const& key) const st.exchange(soci::into(thresholds)); st.exchange(soci::into(account.flags)); st.exchange(soci::into(le.lastModifiedLedgerSeq)); - st.exchange(soci::into(liabilities.buying, buyingLiabilitiesInd)); - st.exchange(soci::into(liabilities.selling, sellingLiabilitiesInd)); st.exchange(soci::into(signers, signersInd)); + st.exchange(soci::into(extensionStr, extensionInd)); + st.exchange(soci::into(ledgerExtStr, ledgerExtInd)); st.exchange(soci::use(actIDStrKey)); st.define_and_bind(); { @@ -87,12 +88,9 @@ LedgerTxnRoot::Impl::loadAccount(LedgerKey const& key) const }) == account.signers.end()); } - assert(buyingLiabilitiesInd == sellingLiabilitiesInd); - if (buyingLiabilitiesInd == soci::i_ok) - { - account.ext.v(1); - account.ext.v1().liabilities = liabilities; - } + decodeOpaqueXDR(extensionStr, extensionInd, account.ext); + + decodeOpaqueXDR(ledgerExtStr, ledgerExtInd, le.ext); return std::make_shared(std::move(le)); } @@ -145,9 +143,9 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation std::vector mSigners; std::vector mSignerInds; std::vector mLastModifieds; - std::vector mBuyingLiabilities; - std::vector mSellingLiabilities; - std::vector mLiabilitiesInds; + std::vector mExtensions; + std::vector mExtensionInds; + std::vector mLedgerExtensions; public: BulkUpsertAccountsOperation(Database& DB, @@ -166,9 +164,9 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation mSigners.reserve(entries.size()); mSignerInds.reserve(entries.size()); mLastModifieds.reserve(entries.size()); - mBuyingLiabilities.reserve(entries.size()); - mSellingLiabilities.reserve(entries.size()); - mLiabilitiesInds.reserve(entries.size()); + mExtensions.reserve(entries.size()); + mExtensionInds.reserve(entries.size()); + mLedgerExtensions.reserve(entries.size()); for (auto const& e : entries) { @@ -210,18 +208,18 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation if (account.ext.v() >= 1) { - mBuyingLiabilities.emplace_back( - account.ext.v1().liabilities.buying); - mSellingLiabilities.emplace_back( - account.ext.v1().liabilities.selling); - mLiabilitiesInds.emplace_back(soci::i_ok); + mExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(account.ext))); + mExtensionInds.emplace_back(soci::i_ok); } else { - mBuyingLiabilities.emplace_back(0); - mSellingLiabilities.emplace_back(0); - mLiabilitiesInds.emplace_back(soci::i_null); + mExtensions.emplace_back(""); + mExtensionInds.emplace_back(soci::i_null); } + + mLedgerExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(e.entry().ext))); } } @@ -232,7 +230,7 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation "INSERT INTO accounts ( " "accountid, balance, seqnum, numsubentries, inflationdest," "homedomain, thresholds, signers, flags, lastmodified, " - "buyingliabilities, sellingliabilities " + "extension, ledgerext " ") VALUES ( " ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10, :v11 " ") ON CONFLICT (accountid) DO UPDATE SET " @@ -245,8 +243,8 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation "signers = excluded.signers, " "flags = excluded.flags, " "lastmodified = excluded.lastmodified, " - "buyingliabilities = excluded.buyingliabilities, " - "sellingliabilities = excluded.sellingliabilities"; + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(mAccountIDs)); @@ -259,8 +257,8 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation st.exchange(soci::use(mSigners, mSignerInds)); st.exchange(soci::use(mFlags)); st.exchange(soci::use(mLastModifieds)); - st.exchange(soci::use(mBuyingLiabilities, mLiabilitiesInds)); - st.exchange(soci::use(mSellingLiabilities, mLiabilitiesInds)); + st.exchange(soci::use(mExtensions, mExtensionInds)); + st.exchange(soci::use(mLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("account"); @@ -284,8 +282,7 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation { std::string strAccountIDs, strBalances, strSeqNums, strSubEntryNums, strInflationDests, strFlags, strHomeDomains, strThresholds, - strSigners, strLastModifieds, strBuyingLiabilities, - strSellingLiabilities; + strSigners, strLastModifieds, strExtensions, strLedgerExtensions; PGconn* conn = pg->conn_; marshalToPGArray(conn, strAccountIDs, mAccountIDs); @@ -299,43 +296,42 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation marshalToPGArray(conn, strThresholds, mThresholds); marshalToPGArray(conn, strSigners, mSigners, &mSignerInds); marshalToPGArray(conn, strLastModifieds, mLastModifieds); - marshalToPGArray(conn, strBuyingLiabilities, mBuyingLiabilities, - &mLiabilitiesInds); - marshalToPGArray(conn, strSellingLiabilities, mSellingLiabilities, - &mLiabilitiesInds); - - std::string sql = - "WITH r AS (SELECT " - "unnest(:ids::TEXT[]), " - "unnest(:v1::BIGINT[]), " - "unnest(:v2::BIGINT[]), " - "unnest(:v3::INT[]), " - "unnest(:v4::TEXT[]), " - "unnest(:v5::TEXT[]), " - "unnest(:v6::TEXT[]), " - "unnest(:v7::TEXT[]), " - "unnest(:v8::INT[]), " - "unnest(:v9::INT[]), " - "unnest(:v10::BIGINT[]), " - "unnest(:v11::BIGINT[]) " - ")" - "INSERT INTO accounts ( " - "accountid, balance, seqnum, " - "numsubentries, inflationdest, homedomain, thresholds, signers, " - "flags, lastmodified, buyingliabilities, sellingliabilities " - ") SELECT * FROM r " - "ON CONFLICT (accountid) DO UPDATE SET " - "balance = excluded.balance, " - "seqnum = excluded.seqnum, " - "numsubentries = excluded.numsubentries, " - "inflationdest = excluded.inflationdest, " - "homedomain = excluded.homedomain, " - "thresholds = excluded.thresholds, " - "signers = excluded.signers, " - "flags = excluded.flags, " - "lastmodified = excluded.lastmodified, " - "buyingliabilities = excluded.buyingliabilities, " - "sellingliabilities = excluded.sellingliabilities"; + marshalToPGArray(conn, strExtensions, mExtensions, &mExtensionInds); + marshalToPGArray(conn, strLedgerExtensions, mLedgerExtensions); + + std::string sql = "WITH r AS (SELECT " + "unnest(:ids::TEXT[]), " + "unnest(:v1::BIGINT[]), " + "unnest(:v2::BIGINT[]), " + "unnest(:v3::INT[]), " + "unnest(:v4::TEXT[]), " + "unnest(:v5::TEXT[]), " + "unnest(:v6::TEXT[]), " + "unnest(:v7::TEXT[]), " + "unnest(:v8::INT[]), " + "unnest(:v9::INT[]), " + "unnest(:v10::TEXT[]), " + "unnest(:v11::TEXT[]) " + ")" + "INSERT INTO accounts ( " + "accountid, balance, seqnum, " + "numsubentries, inflationdest, homedomain, " + "thresholds, signers, " + "flags, lastmodified, extension, " + "ledgerext " + ") SELECT * FROM r " + "ON CONFLICT (accountid) DO UPDATE SET " + "balance = excluded.balance, " + "seqnum = excluded.seqnum, " + "numsubentries = excluded.numsubentries, " + "inflationdest = excluded.inflationdest, " + "homedomain = excluded.homedomain, " + "thresholds = excluded.thresholds, " + "signers = excluded.signers, " + "flags = excluded.flags, " + "lastmodified = excluded.lastmodified, " + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(strAccountIDs)); @@ -348,8 +344,8 @@ class BulkUpsertAccountsOperation : public DatabaseTypeSpecificOperation st.exchange(soci::use(strSigners)); st.exchange(soci::use(strFlags)); st.exchange(soci::use(strLastModifieds)); - st.exchange(soci::use(strBuyingLiabilities)); - st.exchange(soci::use(strSellingLiabilities)); + st.exchange(soci::use(strExtensions)); + st.exchange(soci::use(strLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("account"); @@ -499,9 +495,10 @@ class BulkLoadAccountsOperation int64_t balance; uint64_t seqNum; uint32_t numSubEntries, flags, lastModified; - Liabilities liabilities; - soci::indicator inflationDestInd, signersInd, buyingLiabilitiesInd, - sellingLiabilitiesInd; + std::string extension; + soci::indicator inflationDestInd, signersInd, extensionInd; + std::string ledgerExtension; + soci::indicator ledgerExtInd; st.exchange(soci::into(accountID)); st.exchange(soci::into(balance)); @@ -512,9 +509,9 @@ class BulkLoadAccountsOperation st.exchange(soci::into(thresholds)); st.exchange(soci::into(flags)); st.exchange(soci::into(lastModified)); - st.exchange(soci::into(liabilities.buying, buyingLiabilitiesInd)); - st.exchange(soci::into(liabilities.selling, sellingLiabilitiesInd)); + st.exchange(soci::into(extension, extensionInd)); st.exchange(soci::into(signers, signersInd)); + st.exchange(soci::into(ledgerExtension, ledgerExtInd)); st.define_and_bind(); { auto timer = mDb.getSelectTimer("account"); @@ -554,12 +551,7 @@ class BulkLoadAccountsOperation ae.flags = flags; le.lastModifiedLedgerSeq = lastModified; - assert(buyingLiabilitiesInd == sellingLiabilitiesInd); - if (buyingLiabilitiesInd == soci::i_ok) - { - ae.ext.v(1); - ae.ext.v1().liabilities = liabilities; - } + decodeOpaqueXDR(extension, extensionInd, ae.ext); if (signersInd == soci::i_ok) { @@ -573,6 +565,8 @@ class BulkLoadAccountsOperation }) == ae.signers.end()); } + decodeOpaqueXDR(ledgerExtension, ledgerExtInd, le.ext); + st.fetch(); } return res; @@ -604,7 +598,8 @@ class BulkLoadAccountsOperation std::string sql = "SELECT accountid, balance, seqnum, numsubentries, " "inflationdest, homedomain, thresholds, flags, lastmodified, " - "buyingliabilities, sellingliabilities, signers FROM accounts " + "extension, signers, ledgerext" + " FROM accounts " "WHERE accountid IN carray(?, ?, 'char*')"; auto prep = mDb.getPreparedStatement(sql); @@ -627,7 +622,6 @@ class BulkLoadAccountsOperation virtual std::vector doPostgresSpecificOperation(soci::postgresql_session_backend* pg) override { - std::string strAccountIDs; marshalToPGArray(pg->conn_, strAccountIDs, mAccountIDs); @@ -635,7 +629,8 @@ class BulkLoadAccountsOperation "WITH r AS (SELECT unnest(:v1::TEXT[])) " "SELECT accountid, balance, seqnum, numsubentries, " "inflationdest, homedomain, thresholds, flags, lastmodified, " - "buyingliabilities, sellingliabilities, signers FROM accounts " + "extension, signers, ledgerext" + " FROM accounts " "WHERE accountid IN (SELECT * FROM r)"; auto prep = mDb.getPreparedStatement(sql); diff --git a/src/ledger/LedgerTxnDataSQL.cpp b/src/ledger/LedgerTxnDataSQL.cpp index 6ce68ec289..715c88a738 100644 --- a/src/ledger/LedgerTxnDataSQL.cpp +++ b/src/ledger/LedgerTxnDataSQL.cpp @@ -24,18 +24,25 @@ LedgerTxnRoot::Impl::loadData(LedgerKey const& key) const std::string dataValue; soci::indicator dataValueIndicator; + std::string extensionStr; + soci::indicator extensionInd; + std::string ledgerExtStr; + soci::indicator ledgerExtInd; LedgerEntry le; le.data.type(DATA); DataEntry& de = le.data.data(); - std::string sql = "SELECT datavalue, lastmodified " + std::string sql = "SELECT datavalue, lastmodified, extension, " + "ledgerext " "FROM accountdata " "WHERE accountid= :id AND dataname= :dataname"; auto prep = mDatabase.getPreparedStatement(sql); auto& st = prep.statement(); st.exchange(soci::into(dataValue, dataValueIndicator)); st.exchange(soci::into(le.lastModifiedLedgerSeq)); + st.exchange(soci::into(extensionStr, extensionInd)); + st.exchange(soci::into(ledgerExtStr, ledgerExtInd)); st.exchange(soci::use(actIDStrKey)); st.exchange(soci::use(dataName)); st.define_and_bind(); @@ -54,6 +61,10 @@ LedgerTxnRoot::Impl::loadData(LedgerKey const& key) const } decoder::decode_b64(dataValue, de.dataValue); + decodeOpaqueXDR(extensionStr, extensionInd, de.ext); + + decodeOpaqueXDR(ledgerExtStr, ledgerExtInd, le.ext); + return std::make_shared(std::move(le)); } @@ -64,6 +75,8 @@ class BulkUpsertDataOperation : public DatabaseTypeSpecificOperation std::vector mDataNames; std::vector mDataValues; std::vector mLastModifieds; + std::vector mExtensions; + std::vector mLedgerExtensions; void accumulateEntry(LedgerEntry const& entry) @@ -75,6 +88,10 @@ class BulkUpsertDataOperation : public DatabaseTypeSpecificOperation mDataValues.emplace_back(decoder::encode_b64(data.dataValue)); mLastModifieds.emplace_back( unsignedToSigned(entry.lastModifiedLedgerSeq)); + mExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(data.ext))); + mLedgerExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(entry.ext))); } public: @@ -102,19 +119,25 @@ class BulkUpsertDataOperation : public DatabaseTypeSpecificOperation void doSociGenericOperation() { - std::string sql = "INSERT INTO accountdata ( " - "accountid, dataname, datavalue, lastmodified " - ") VALUES ( " - ":id, :v1, :v2, :v3 " - ") ON CONFLICT (accountid, dataname) DO UPDATE SET " - "datavalue = excluded.datavalue, " - "lastmodified = excluded.lastmodified "; + std::string sql = + "INSERT INTO accountdata ( " + "accountid, dataname, datavalue, lastmodified, extension, " + "ledgerext " + ") VALUES ( " + ":id, :v1, :v2, :v3, :v4, :v5 " + ") ON CONFLICT (accountid, dataname) DO UPDATE SET " + "datavalue = excluded.datavalue, " + "lastmodified = excluded.lastmodified, " + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(mAccountIDs)); st.exchange(soci::use(mDataNames)); st.exchange(soci::use(mDataValues)); st.exchange(soci::use(mLastModifieds)); + st.exchange(soci::use(mExtensions)); + st.exchange(soci::use(mLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("data"); @@ -136,31 +159,41 @@ class BulkUpsertDataOperation : public DatabaseTypeSpecificOperation doPostgresSpecificOperation(soci::postgresql_session_backend* pg) override { std::string strAccountIDs, strDataNames, strDataValues, - strLastModifieds; + strLastModifieds, strExtensions, strLedgerExtensions; PGconn* conn = pg->conn_; marshalToPGArray(conn, strAccountIDs, mAccountIDs); marshalToPGArray(conn, strDataNames, mDataNames); marshalToPGArray(conn, strDataValues, mDataValues); marshalToPGArray(conn, strLastModifieds, mLastModifieds); - std::string sql = "WITH r AS (SELECT " - "unnest(:ids::TEXT[]), " - "unnest(:v1::TEXT[]), " - "unnest(:v2::TEXT[]), " - "unnest(:v3::INT[]) " - ")" - "INSERT INTO accountdata ( " - "accountid, dataname, datavalue, lastmodified " - ") SELECT * FROM r " - "ON CONFLICT (accountid, dataname) DO UPDATE SET " - "datavalue = excluded.datavalue, " - "lastmodified = excluded.lastmodified "; + marshalToPGArray(conn, strExtensions, mExtensions); + marshalToPGArray(conn, strLedgerExtensions, mLedgerExtensions); + std::string sql = + "WITH r AS (SELECT " + "unnest(:ids::TEXT[]), " + "unnest(:v1::TEXT[]), " + "unnest(:v2::TEXT[]), " + "unnest(:v3::INT[]), " + "unnest(:v4::TEXT[]), " + "unnest(:v5::TEXT[]) " + ")" + "INSERT INTO accountdata ( " + "accountid, dataname, datavalue, lastmodified, extension, " + "ledgerext " + ") SELECT * FROM r " + "ON CONFLICT (accountid, dataname) DO UPDATE SET " + "datavalue = excluded.datavalue, " + "lastmodified = excluded.lastmodified, " + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(strAccountIDs)); st.exchange(soci::use(strDataNames)); st.exchange(soci::use(strDataValues)); st.exchange(soci::use(strLastModifieds)); + st.exchange(soci::use(strExtensions)); + st.exchange(soci::use(strLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("data"); @@ -311,11 +344,17 @@ class BulkLoadDataOperation { std::string accountID, dataName, dataValue; uint32_t lastModified; + std::string extension; + soci::indicator extensionInd; + std::string ledgerExtension; + soci::indicator ledgerExtInd; st.exchange(soci::into(accountID)); st.exchange(soci::into(dataName)); st.exchange(soci::into(dataValue)); st.exchange(soci::into(lastModified)); + st.exchange(soci::into(extension, extensionInd)); + st.exchange(soci::into(ledgerExtension, ledgerExtInd)); st.define_and_bind(); { auto timer = mDb.getSelectTimer("data"); @@ -335,6 +374,10 @@ class BulkLoadDataOperation decoder::decode_b64(dataValue, de.dataValue); le.lastModifiedLedgerSeq = lastModified; + decodeOpaqueXDR(extension, extensionInd, de.ext); + + decodeOpaqueXDR(ledgerExtension, ledgerExtInd, le.ext); + st.fetch(); } return res; @@ -376,10 +419,11 @@ class BulkLoadDataOperation "AS x " "INNER JOIN (SELECT rowid, value FROM carray(?, ?, 'char*') ORDER " "BY rowid) AS y ON x.rowid = y.rowid"; - std::string sql = - "WITH r AS (" + sqlJoin + - ") SELECT accountid, dataname, datavalue, lastmodified " - "FROM accountdata WHERE (accountid, dataname) IN r"; + std::string sql = "WITH r AS (" + sqlJoin + + ") SELECT accountid, dataname, datavalue, " + "lastmodified, extension, " + "ledgerext " + "FROM accountdata WHERE (accountid, dataname) IN r"; auto prep = mDb.getPreparedStatement(sql); auto be = prep.statement().get_backend(); @@ -412,7 +456,8 @@ class BulkLoadDataOperation std::string sql = "WITH r AS (SELECT unnest(:v1::TEXT[]), unnest(:v2::TEXT[])) " - "SELECT accountid, dataname, datavalue, lastmodified " + "SELECT accountid, dataname, datavalue, lastmodified, extension, " + "ledgerext " "FROM accountdata WHERE (accountid, dataname) IN (SELECT * FROM r)"; auto prep = mDb.getPreparedStatement(sql); diff --git a/src/ledger/LedgerTxnOfferSQL.cpp b/src/ledger/LedgerTxnOfferSQL.cpp index 372d185c7d..82c55d8394 100644 --- a/src/ledger/LedgerTxnOfferSQL.cpp +++ b/src/ledger/LedgerTxnOfferSQL.cpp @@ -31,7 +31,8 @@ LedgerTxnRoot::Impl::loadOffer(LedgerKey const& key) const std::string actIDStrKey = KeyUtils::toStrKey(key.offer().sellerID); std::string sql = "SELECT sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, flags, lastmodified " + "amount, pricen, priced, flags, lastmodified, extension, " + "ledgerext " "FROM offers " "WHERE sellerid= :id AND offerid= :offerid"; auto prep = mDatabase.getPreparedStatement(sql); @@ -54,8 +55,8 @@ LedgerTxnRoot::Impl::loadAllOffers() const { ZoneScoped; std::string sql = "SELECT sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, flags, lastmodified " - "FROM offers"; + "amount, pricen, priced, flags, lastmodified, extension, " + "ledgerext FROM offers"; auto prep = mDatabase.getPreparedStatement(sql); std::vector offers; @@ -75,8 +76,8 @@ LedgerTxnRoot::Impl::loadBestOffers(std::deque& offers, // price is an approximation of the actual n/d (truncated math, 15 digits) // ordering by offerid gives precendence to older offers for fairness std::string sql = "SELECT sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, flags, lastmodified " - "FROM offers " + "amount, pricen, priced, flags, lastmodified, extension, " + "ledgerext FROM offers " "WHERE sellingasset = :v1 AND buyingasset = :v2 " "ORDER BY price, offerid LIMIT :n"; @@ -116,16 +117,19 @@ LedgerTxnRoot::Impl::loadBestOffers(std::deque& offers, std::string sql = "WITH r1 AS " "(SELECT sellerid, offerid, sellingasset, buyingasset, amount, price, " - "pricen, priced, flags, lastmodified FROM offers " + "pricen, priced, flags, lastmodified, extension, " + "ledgerext FROM offers " "WHERE sellingasset = :v1 AND buyingasset = :v2 AND price > :v3 " "ORDER BY price, offerid LIMIT :v4), " "r2 AS " "(SELECT sellerid, offerid, sellingasset, buyingasset, amount, price, " - "pricen, priced, flags, lastmodified FROM offers " + "pricen, priced, flags, lastmodified, extension, " + "ledgerext FROM offers " "WHERE sellingasset = :v5 AND buyingasset = :v6 AND price = :v7 " "AND offerid >= :v8 ORDER BY price, offerid LIMIT :v9) " "SELECT sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, flags, lastmodified " + "amount, pricen, priced, flags, lastmodified, extension, " + "ledgerext " "FROM (SELECT * FROM r1 UNION ALL SELECT * FROM r2) AS res " "ORDER BY price, offerid LIMIT :v10"; @@ -205,7 +209,8 @@ LedgerTxnRoot::Impl::loadOffersByAccountAndAsset(AccountID const& accountID, { ZoneScoped; std::string sql = "SELECT sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, flags, lastmodified " + "amount, pricen, priced, flags, lastmodified, extension, " + "ledgerext " "FROM offers WHERE sellerid = :v1 AND " "(sellingasset = :v2 OR buyingasset = :v3)"; // Note: v2 == v3 but positional parameters are faster @@ -249,6 +254,10 @@ LedgerTxnRoot::Impl::loadOffers(StatementContext& prep, ZoneScoped; std::string actIDStrKey; std::string sellingAsset, buyingAsset; + std::string extensionStr; + soci::indicator extensionInd; + std::string ledgerExtStr; + soci::indicator ledgerExtInd; LedgerEntry le; le.data.type(OFFER); @@ -264,6 +273,8 @@ LedgerTxnRoot::Impl::loadOffers(StatementContext& prep, st.exchange(soci::into(oe.price.d)); st.exchange(soci::into(oe.flags)); st.exchange(soci::into(le.lastModifiedLedgerSeq)); + st.exchange(soci::into(extensionStr, extensionInd)); + st.exchange(soci::into(ledgerExtStr, ledgerExtInd)); st.define_and_bind(); st.execute(true); @@ -275,6 +286,10 @@ LedgerTxnRoot::Impl::loadOffers(StatementContext& prep, oe.selling = processAsset(sellingAsset); oe.buying = processAsset(buyingAsset); + decodeOpaqueXDR(extensionStr, extensionInd, oe.ext); + + decodeOpaqueXDR(ledgerExtStr, ledgerExtInd, le.ext); + offers.emplace_back(le); st.fetch(); } @@ -290,6 +305,10 @@ LedgerTxnRoot::Impl::loadOffers(StatementContext& prep) const std::string actIDStrKey; std::string sellingAsset, buyingAsset; + std::string extensionStr; + soci::indicator extensionInd; + std::string ledgerExtStr; + soci::indicator ledgerExtInd; LedgerEntry le; le.data.type(OFFER); @@ -305,6 +324,8 @@ LedgerTxnRoot::Impl::loadOffers(StatementContext& prep) const st.exchange(soci::into(oe.price.d)); st.exchange(soci::into(oe.flags)); st.exchange(soci::into(le.lastModifiedLedgerSeq)); + st.exchange(soci::into(extensionStr, extensionInd)); + st.exchange(soci::into(ledgerExtStr, ledgerExtInd)); st.define_and_bind(); st.execute(true); @@ -314,6 +335,10 @@ LedgerTxnRoot::Impl::loadOffers(StatementContext& prep) const oe.selling = processAsset(sellingAsset); oe.buying = processAsset(buyingAsset); + decodeOpaqueXDR(extensionStr, extensionInd, oe.ext); + + decodeOpaqueXDR(ledgerExtStr, ledgerExtInd, le.ext); + offers.emplace_back(le); st.fetch(); } @@ -334,6 +359,8 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation std::vector mPrices; std::vector mFlags; std::vector mLastModifieds; + std::vector mExtensions; + std::vector mLedgerExtensions; void accumulateEntry(LedgerEntry const& entry) @@ -358,6 +385,10 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation mFlags.emplace_back(unsignedToSigned(offer.flags)); mLastModifieds.emplace_back( unsignedToSigned(entry.lastModifiedLedgerSeq)); + mExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(offer.ext))); + mLedgerExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(entry.ext))); } public: @@ -375,6 +406,8 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation mPrices.reserve(entries.size()); mFlags.reserve(entries.size()); mLastModifieds.reserve(entries.size()); + mExtensions.reserve(entries.size()); + mLedgerExtensions.reserve(entries.size()); for (auto const& e : entries) { @@ -396,6 +429,8 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation mPrices.reserve(entries.size()); mFlags.reserve(entries.size()); mLastModifieds.reserve(entries.size()); + mExtensions.reserve(entries.size()); + mLedgerExtensions.reserve(entries.size()); for (auto const& e : entries) { @@ -407,21 +442,25 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation void doSociGenericOperation() { - std::string sql = "INSERT INTO offers ( " - "sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, price, flags, lastmodified " - ") VALUES ( " - ":v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10 " - ") ON CONFLICT (offerid) DO UPDATE SET " - "sellerid = excluded.sellerid, " - "sellingasset = excluded.sellingasset, " - "buyingasset = excluded.buyingasset, " - "amount = excluded.amount, " - "pricen = excluded.pricen, " - "priced = excluded.priced, " - "price = excluded.price, " - "flags = excluded.flags, " - "lastmodified = excluded.lastmodified "; + std::string sql = + "INSERT INTO offers ( " + "sellerid, offerid, sellingasset, buyingasset, " + "amount, pricen, priced, price, flags, lastmodified, extension, " + "ledgerext " + ") VALUES ( " + ":v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10, :v11, :v12 " + ") ON CONFLICT (offerid) DO UPDATE SET " + "sellerid = excluded.sellerid, " + "sellingasset = excluded.sellingasset, " + "buyingasset = excluded.buyingasset, " + "amount = excluded.amount, " + "pricen = excluded.pricen, " + "priced = excluded.priced, " + "price = excluded.price, " + "flags = excluded.flags, " + "lastmodified = excluded.lastmodified, " + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(mSellerIDs)); @@ -434,6 +473,8 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation st.exchange(soci::use(mPrices)); st.exchange(soci::use(mFlags)); st.exchange(soci::use(mLastModifieds)); + st.exchange(soci::use(mExtensions)); + st.exchange(soci::use(mLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("offer"); @@ -458,7 +499,7 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation std::string strSellerIDs, strOfferIDs, strSellingAssets, strBuyingAssets, strAmounts, strPriceNs, strPriceDs, strPrices, - strFlags, strLastModifieds; + strFlags, strLastModifieds, strExtensions, strLedgerExtensions; PGconn* conn = pg->conn_; marshalToPGArray(conn, strSellerIDs, mSellerIDs); @@ -473,33 +514,41 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation marshalToPGArray(conn, strPrices, mPrices); marshalToPGArray(conn, strFlags, mFlags); marshalToPGArray(conn, strLastModifieds, mLastModifieds); + marshalToPGArray(conn, strExtensions, mExtensions); + marshalToPGArray(conn, strLedgerExtensions, mLedgerExtensions); - std::string sql = "WITH r AS (SELECT " - "unnest(:v1::TEXT[]), " - "unnest(:v2::BIGINT[]), " - "unnest(:v3::TEXT[]), " - "unnest(:v4::TEXT[]), " - "unnest(:v5::BIGINT[]), " - "unnest(:v6::INT[]), " - "unnest(:v7::INT[]), " - "unnest(:v8::DOUBLE PRECISION[]), " - "unnest(:v9::INT[]), " - "unnest(:v10::INT[]) " - ")" - "INSERT INTO offers ( " - "sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, price, flags, lastmodified " - ") SELECT * from r " - "ON CONFLICT (offerid) DO UPDATE SET " - "sellerid = excluded.sellerid, " - "sellingasset = excluded.sellingasset, " - "buyingasset = excluded.buyingasset, " - "amount = excluded.amount, " - "pricen = excluded.pricen, " - "priced = excluded.priced, " - "price = excluded.price, " - "flags = excluded.flags, " - "lastmodified = excluded.lastmodified "; + std::string sql = + "WITH r AS (SELECT " + "unnest(:v1::TEXT[]), " + "unnest(:v2::BIGINT[]), " + "unnest(:v3::TEXT[]), " + "unnest(:v4::TEXT[]), " + "unnest(:v5::BIGINT[]), " + "unnest(:v6::INT[]), " + "unnest(:v7::INT[]), " + "unnest(:v8::DOUBLE PRECISION[]), " + "unnest(:v9::INT[]), " + "unnest(:v10::INT[]), " + "unnest(:v11::TEXT[]), " + "unnest(:v12::TEXT[]) " + ")" + "INSERT INTO offers ( " + "sellerid, offerid, sellingasset, buyingasset, " + "amount, pricen, priced, price, flags, lastmodified, extension, " + "ledgerext " + ") SELECT * from r " + "ON CONFLICT (offerid) DO UPDATE SET " + "sellerid = excluded.sellerid, " + "sellingasset = excluded.sellingasset, " + "buyingasset = excluded.buyingasset, " + "amount = excluded.amount, " + "pricen = excluded.pricen, " + "priced = excluded.priced, " + "price = excluded.price, " + "flags = excluded.flags, " + "lastmodified = excluded.lastmodified, " + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(strSellerIDs)); @@ -512,6 +561,8 @@ class BulkUpsertOffersOperation : public DatabaseTypeSpecificOperation st.exchange(soci::use(strPrices)); st.exchange(soci::use(strFlags)); st.exchange(soci::use(strLastModifieds)); + st.exchange(soci::use(strExtensions)); + st.exchange(soci::use(strLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("offer"); @@ -671,6 +722,10 @@ class BulkLoadOffersOperation int64_t amount; int64_t offerID; uint32_t flags, lastModified; + std::string extension; + soci::indicator extensionInd; + std::string ledgerExtension; + soci::indicator ledgerExtInd; Price price; st.exchange(soci::into(sellerID)); @@ -682,6 +737,8 @@ class BulkLoadOffersOperation st.exchange(soci::into(price.d)); st.exchange(soci::into(flags)); st.exchange(soci::into(lastModified)); + st.exchange(soci::into(extension, extensionInd)); + st.exchange(soci::into(ledgerExtension, ledgerExtInd)); st.define_and_bind(); { auto timer = mDb.getSelectTimer("offer"); @@ -710,6 +767,10 @@ class BulkLoadOffersOperation oe.price = price; oe.flags = flags; le.lastModifiedLedgerSeq = lastModified; + + decodeOpaqueXDR(extension, extensionInd, oe.ext); + + decodeOpaqueXDR(ledgerExtension, ledgerExtInd, le.ext); } st.fetch(); @@ -743,7 +804,8 @@ class BulkLoadOffersOperation { std::string sql = "SELECT sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, flags, lastmodified " + "amount, pricen, priced, flags, lastmodified, extension, " + "ledgerext " "FROM offers WHERE offerid IN carray(?, ?, 'int64')"; auto prep = mDb.getPreparedStatement(sql); @@ -772,7 +834,8 @@ class BulkLoadOffersOperation std::string sql = "WITH r AS (SELECT unnest(:v1::BIGINT[])) " "SELECT sellerid, offerid, sellingasset, buyingasset, " - "amount, pricen, priced, flags, lastmodified " + "amount, pricen, priced, flags, lastmodified, extension, " + "ledgerext " "FROM offers WHERE offerid IN (SELECT * FROM r)"; auto prep = mDb.getPreparedStatement(sql); auto& st = prep.statement(); diff --git a/src/ledger/LedgerTxnTrustLineSQL.cpp b/src/ledger/LedgerTxnTrustLineSQL.cpp index 4feb2de20f..d3329a0176 100644 --- a/src/ledger/LedgerTxnTrustLineSQL.cpp +++ b/src/ledger/LedgerTxnTrustLineSQL.cpp @@ -7,6 +7,8 @@ #include "database/Database.h" #include "database/DatabaseTypeSpecificOperation.h" #include "ledger/LedgerTxnImpl.h" +#include "util/Decoder.h" +#include "util/Logging.h" #include "util/XDROperators.h" #include "util/types.h" #include @@ -14,7 +16,7 @@ namespace stellar { -static void +void getTrustLineStrings(AccountID const& accountID, Asset const& asset, std::string& accountIDStr, std::string& issuerStr, std::string& assetCodeStr) @@ -53,24 +55,27 @@ LedgerTxnRoot::Impl::loadTrustLine(LedgerKey const& key) const getTrustLineStrings(key.trustLine().accountID, key.trustLine().asset, accountIDStr, issuerStr, assetStr); - Liabilities liabilities; - soci::indicator buyingLiabilitiesInd, sellingLiabilitiesInd; + std::string extensionStr; + soci::indicator extensionInd; + std::string ledgerExtStr; + soci::indicator ledgerExtInd; LedgerEntry le; le.data.type(TRUSTLINE); TrustLineEntry& tl = le.data.trustLine(); auto prep = mDatabase.getPreparedStatement( - "SELECT tlimit, balance, flags, lastmodified, buyingliabilities, " - "sellingliabilities FROM trustlines " + "SELECT tlimit, balance, flags, lastmodified, " + "extension, ledgerext" + " FROM trustlines " "WHERE accountid= :id AND issuer= :issuer AND assetcode= :asset"); auto& st = prep.statement(); st.exchange(soci::into(tl.limit)); st.exchange(soci::into(tl.balance)); st.exchange(soci::into(tl.flags)); st.exchange(soci::into(le.lastModifiedLedgerSeq)); - st.exchange(soci::into(liabilities.buying, buyingLiabilitiesInd)); - st.exchange(soci::into(liabilities.selling, sellingLiabilitiesInd)); + st.exchange(soci::into(extensionStr, extensionInd)); + st.exchange(soci::into(ledgerExtStr, ledgerExtInd)); st.exchange(soci::use(accountIDStr)); st.exchange(soci::use(issuerStr)); st.exchange(soci::use(assetStr)); @@ -87,12 +92,9 @@ LedgerTxnRoot::Impl::loadTrustLine(LedgerKey const& key) const tl.accountID = key.trustLine().accountID; tl.asset = key.trustLine().asset; - assert(buyingLiabilitiesInd == sellingLiabilitiesInd); - if (buyingLiabilitiesInd == soci::i_ok) - { - tl.ext.v(1); - tl.ext.v1().liabilities = liabilities; - } + decodeOpaqueXDR(extensionStr, extensionInd, tl.ext); + + decodeOpaqueXDR(ledgerExtStr, ledgerExtInd, le.ext); return std::make_shared(std::move(le)); } @@ -108,9 +110,9 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation std::vector mBalances; std::vector mFlags; std::vector mLastModifieds; - std::vector mBuyingLiabilities; - std::vector mSellingLiabilities; - std::vector mLiabilitiesInds; + std::vector mExtensions; + std::vector mExtensionInds; + std::vector mLedgerExtensions; public: BulkUpsertTrustLinesOperation(Database& DB, @@ -125,9 +127,9 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation mBalances.reserve(entries.size()); mFlags.reserve(entries.size()); mLastModifieds.reserve(entries.size()); - mBuyingLiabilities.reserve(entries.size()); - mSellingLiabilities.reserve(entries.size()); - mLiabilitiesInds.reserve(entries.size()); + mExtensions.reserve(entries.size()); + mExtensionInds.reserve(entries.size()); + mLedgerExtensions.reserve(entries.size()); for (auto const& e : entries) { @@ -151,17 +153,18 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation if (tl.ext.v() >= 1) { - mBuyingLiabilities.emplace_back(tl.ext.v1().liabilities.buying); - mSellingLiabilities.emplace_back( - tl.ext.v1().liabilities.selling); - mLiabilitiesInds.emplace_back(soci::i_ok); + mExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(tl.ext))); + mExtensionInds.emplace_back(soci::i_ok); } else { - mBuyingLiabilities.emplace_back(0); - mSellingLiabilities.emplace_back(0); - mLiabilitiesInds.emplace_back(soci::i_null); + mExtensions.emplace_back(""); + mExtensionInds.emplace_back(soci::i_null); } + + mLedgerExtensions.emplace_back( + decoder::encode_b64(xdr::xdr_to_opaque(e.entry().ext))); } } @@ -172,7 +175,7 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation "INSERT INTO trustlines ( " "accountid, assettype, issuer, assetcode," "tlimit, balance, flags, lastmodified, " - "buyingliabilities, sellingliabilities " + "extension, ledgerext " ") VALUES ( " ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9 " ") ON CONFLICT (accountid, issuer, assetcode) DO UPDATE SET " @@ -181,8 +184,8 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation "balance = excluded.balance, " "flags = excluded.flags, " "lastmodified = excluded.lastmodified, " - "buyingliabilities = excluded.buyingliabilities, " - "sellingliabilities = excluded.sellingliabilities "; + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(mAccountIDs)); @@ -193,8 +196,8 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation st.exchange(soci::use(mBalances)); st.exchange(soci::use(mFlags)); st.exchange(soci::use(mLastModifieds)); - st.exchange(soci::use(mBuyingLiabilities, mLiabilitiesInds)); - st.exchange(soci::use(mSellingLiabilities, mLiabilitiesInds)); + st.exchange(soci::use(mExtensions, mExtensionInds)); + st.exchange(soci::use(mLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("trustline"); @@ -219,8 +222,8 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation PGconn* conn = pg->conn_; std::string strAccountIDs, strAssetTypes, strIssuers, strAssetCodes, - strTlimits, strBalances, strFlags, strLastModifieds, - strBuyingLiabilities, strSellingLiabilities; + strTlimits, strBalances, strFlags, strLastModifieds, strExtensions, + strLedgerExtensions; marshalToPGArray(conn, strAccountIDs, mAccountIDs); marshalToPGArray(conn, strAssetTypes, mAssetTypes); @@ -230,10 +233,8 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation marshalToPGArray(conn, strBalances, mBalances); marshalToPGArray(conn, strFlags, mFlags); marshalToPGArray(conn, strLastModifieds, mLastModifieds); - marshalToPGArray(conn, strBuyingLiabilities, mBuyingLiabilities, - &mLiabilitiesInds); - marshalToPGArray(conn, strSellingLiabilities, mSellingLiabilities, - &mLiabilitiesInds); + marshalToPGArray(conn, strExtensions, mExtensions, &mExtensionInds); + marshalToPGArray(conn, strLedgerExtensions, mLedgerExtensions); std::string sql = "WITH r AS (SELECT " @@ -245,13 +246,13 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation "unnest(:v5::BIGINT[]), " "unnest(:v6::INT[]), " "unnest(:v7::INT[]), " - "unnest(:v8::BIGINT[]), " - "unnest(:v9::BIGINT[]) " + "unnest(:v8::TEXT[]), " + "unnest(:v9::TEXT[]) " ")" "INSERT INTO trustlines ( " "accountid, assettype, issuer, assetcode," "tlimit, balance, flags, lastmodified, " - "buyingliabilities, sellingliabilities " + "extension, ledgerext " ") SELECT * from r " "ON CONFLICT (accountid, issuer, assetcode) DO UPDATE SET " "assettype = excluded.assettype, " @@ -259,8 +260,8 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation "balance = excluded.balance, " "flags = excluded.flags, " "lastmodified = excluded.lastmodified, " - "buyingliabilities = excluded.buyingliabilities, " - "sellingliabilities = excluded.sellingliabilities "; + "extension = excluded.extension, " + "ledgerext = excluded.ledgerext"; auto prep = mDB.getPreparedStatement(sql); soci::statement& st = prep.statement(); st.exchange(soci::use(strAccountIDs)); @@ -271,8 +272,8 @@ class BulkUpsertTrustLinesOperation : public DatabaseTypeSpecificOperation st.exchange(soci::use(strBalances)); st.exchange(soci::use(strFlags)); st.exchange(soci::use(strLastModifieds)); - st.exchange(soci::use(strBuyingLiabilities)); - st.exchange(soci::use(strSellingLiabilities)); + st.exchange(soci::use(strExtensions)); + st.exchange(soci::use(strLedgerExtensions)); st.define_and_bind(); { auto timer = mDB.getUpsertTimer("trustline"); @@ -440,8 +441,10 @@ class BulkLoadTrustLinesOperation std::string accountID, assetCode, issuer; int64_t balance, limit; uint32_t assetType, flags, lastModified; - Liabilities liabilities; - soci::indicator buyingLiabilitiesInd, sellingLiabilitiesInd; + std::string extension; + soci::indicator extensionInd; + std::string ledgerExtension; + soci::indicator ledgerExtInd; st.exchange(soci::into(accountID)); st.exchange(soci::into(assetType)); @@ -451,8 +454,8 @@ class BulkLoadTrustLinesOperation st.exchange(soci::into(balance)); st.exchange(soci::into(flags)); st.exchange(soci::into(lastModified)); - st.exchange(soci::into(liabilities.buying, buyingLiabilitiesInd)); - st.exchange(soci::into(liabilities.selling, sellingLiabilitiesInd)); + st.exchange(soci::into(extension, extensionInd)); + st.exchange(soci::into(ledgerExtension, ledgerExtInd)); st.define_and_bind(); { auto timer = mDb.getSelectTimer("trust"); @@ -489,12 +492,9 @@ class BulkLoadTrustLinesOperation tl.flags = flags; le.lastModifiedLedgerSeq = lastModified; - assert(buyingLiabilitiesInd == sellingLiabilitiesInd); - if (buyingLiabilitiesInd == soci::i_ok) - { - tl.ext.v(1); - tl.ext.v1().liabilities = liabilities; - } + decodeOpaqueXDR(extension, extensionInd, tl.ext); + + decodeOpaqueXDR(ledgerExtension, ledgerExtInd, le.ext); st.fetch(); } @@ -553,19 +553,21 @@ class BulkLoadTrustLinesOperation cstrAssetCodes.emplace_back(mAssetCodes[i].c_str()); } - std::string sqlJoin = - "SELECT x.value, y.value, z.value FROM " - "(SELECT rowid, value FROM carray(?, ?, 'char*') ORDER BY rowid) " - "AS x " - "INNER JOIN (SELECT rowid, value FROM carray(?, ?, 'char*') ORDER " - "BY rowid) AS y ON x.rowid = y.rowid " - "INNER JOIN (SELECT rowid, value FROM carray(?, ?, 'char*') ORDER " - "BY rowid) AS z ON x.rowid = z.rowid"; + std::string sqlJoin = "SELECT x.value, y.value, z.value FROM " + "(SELECT rowid, value FROM carray(?, ?, " + "'char*') ORDER BY rowid) " + "AS x " + "INNER JOIN (SELECT rowid, value FROM " + "carray(?, ?, 'char*') ORDER " + "BY rowid) AS y ON x.rowid = y.rowid " + "INNER JOIN (SELECT rowid, value FROM " + "carray(?, ?, 'char*') ORDER " + "BY rowid) AS z ON x.rowid = z.rowid"; std::string sql = "WITH r AS (" + sqlJoin + ") SELECT accountid, assettype, assetcode, issuer, tlimit, " - "balance, flags, lastmodified, buyingliabilities, " - "sellingliabilities " + "balance, flags, lastmodified, " + "extension, ledgerext " "FROM trustlines WHERE (accountid, issuer, assetcode) IN r"; auto prep = mDb.getPreparedStatement(sql); @@ -603,11 +605,15 @@ class BulkLoadTrustLinesOperation marshalToPGArray(pg->conn_, strAssetCodes, mAssetCodes); auto prep = mDb.getPreparedStatement( - "WITH r AS (SELECT unnest(:v1::TEXT[]), unnest(:v2::TEXT[]), " - "unnest(:v3::TEXT[])) SELECT accountid, assettype, assetcode, " - "issuer, tlimit, balance, flags, lastmodified, buyingliabilities, " - "sellingliabilities FROM trustlines " - "WHERE (accountid, issuer, assetcode) IN (SELECT * FROM r)"); + "WITH r AS (SELECT unnest(:v1::TEXT[]), " + "unnest(:v2::TEXT[]), " + "unnest(:v3::TEXT[])) SELECT accountid, assettype, " + "assetcode, " + "issuer, tlimit, balance, flags, lastmodified, " + "extension, ledgerext" + " FROM trustlines " + "WHERE (accountid, issuer, assetcode) IN (SELECT * " + "FROM r)"); auto& st = prep.statement(); st.exchange(soci::use(strAccountIDs)); st.exchange(soci::use(strIssuers)); diff --git a/src/main/Application.h b/src/main/Application.h index 89ad88b4af..1fbc6f5fd5 100644 --- a/src/main/Application.h +++ b/src/main/Application.h @@ -282,11 +282,12 @@ class Application // copy made of `cfg` static pointer create(VirtualClock& clock, Config const& cfg, bool newDB = true); - template + template static std::shared_ptr - create(VirtualClock& clock, Config const& cfg, bool newDB = true) + create(VirtualClock& clock, Config const& cfg, Args&&... args, + bool newDB = true) { - auto ret = std::make_shared(clock, cfg); + auto ret = std::make_shared(clock, cfg, std::forward(args)...); ret->initialize(newDB); validateNetworkPassphrase(ret); diff --git a/src/main/ApplicationImpl.cpp b/src/main/ApplicationImpl.cpp index 37e120325e..bed5ee12b1 100644 --- a/src/main/ApplicationImpl.cpp +++ b/src/main/ApplicationImpl.cpp @@ -133,7 +133,7 @@ ApplicationImpl::ApplicationImpl(VirtualClock& clock, Config const& cfg) void ApplicationImpl::initialize(bool createNewDB) { - mDatabase = std::make_unique(*this); + mDatabase = createDatabase(); mPersistentState = std::make_unique(*this); mOverlayManager = createOverlayManager(); mLedgerManager = createLedgerManager(); @@ -895,6 +895,12 @@ ApplicationImpl::createLedgerManager() return LedgerManager::create(*this); } +std::unique_ptr +ApplicationImpl::createDatabase() +{ + return std::make_unique(*this); +} + AbstractLedgerTxnParent& ApplicationImpl::getLedgerTxnRoot() { @@ -902,4 +908,4 @@ ApplicationImpl::getLedgerTxnRoot() return mConfig.MODE_USES_IN_MEMORY_LEDGER ? *mNeverCommittingLedgerTxn : *mLedgerTxnRoot; } -} +} \ No newline at end of file diff --git a/src/main/ApplicationImpl.h b/src/main/ApplicationImpl.h index 202eb639e2..f4193b3178 100644 --- a/src/main/ApplicationImpl.h +++ b/src/main/ApplicationImpl.h @@ -202,5 +202,6 @@ class ApplicationImpl : public Application virtual std::unique_ptr createInvariantManager(); virtual std::unique_ptr createOverlayManager(); virtual std::unique_ptr createLedgerManager(); + virtual std::unique_ptr createDatabase(); }; } diff --git a/src/test/TestUtils.h b/src/test/TestUtils.h index 764e05bcd5..0d6d3482e8 100644 --- a/src/test/TestUtils.h +++ b/src/test/TestUtils.h @@ -65,15 +65,17 @@ class TestApplication : public ApplicationImpl std::unique_ptr createInvariantManager() override; }; -template ::value>::type> std::shared_ptr -createTestApplication(VirtualClock& clock, Config const& cfg, bool newDB = true) +createTestApplication(VirtualClock& clock, Config const& cfg, Args&&... args, + bool newDB = true) { Config c2(cfg); c2.adjust(); - auto app = Application::create(clock, c2, newDB); + auto app = Application::create( + clock, c2, std::forward(args)..., newDB); return app; } diff --git a/src/util/Decoder.h b/src/util/Decoder.h index 9f69c67664..bdd2e8985f 100644 --- a/src/util/Decoder.h +++ b/src/util/Decoder.h @@ -1,3 +1,5 @@ +#pragma once + // Copyright 2018 Stellar Development Foundation and contributors. Licensed // under the Apache License, Version 2.0. See the COPYING file at the root // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0