From 5801f31f822d74c9cd2c0af623e3ed52c85d5cea Mon Sep 17 00:00:00 2001 From: Terence Rokop Date: Mon, 29 Jun 2020 09:53:41 -0700 Subject: [PATCH] Test upgrades to database schema 13 Test upgrades to database schema 13, in which all extensions in all ledger entry types become opaque XDR as far as the database is concerned. The tests use the test-code-injection mechanism which allow them to run arbitrary code between the creation of a database (which is done in an old schema version, MIN_SCHEMA_VERSION) and the upgrading of that database to the current schema version (SCHEMA_VERSION). They create ledger entries of each of the four types, using raw SQL so that they can create them in the old format which the live production code no longer uses. After the database upgrade completes (which it has done by the time that createTestApplication() returns), the tests validate that the contents of the ledger entries are intact and logically unchanged (although their representations in the database have changed, with some columns having been created and some having been deleted, and some contents being encoded differently). --- src/database/test/DatabaseTests.cpp | 303 +++++++++++++++++++++++++++ src/ledger/LedgerTxn.h | 4 + src/ledger/LedgerTxnTrustLineSQL.cpp | 2 +- 3 files changed, 308 insertions(+), 1 deletion(-) diff --git a/src/database/test/DatabaseTests.cpp b/src/database/test/DatabaseTests.cpp index 7f852ec426..e8c8892482 100644 --- a/src/database/test/DatabaseTests.cpp +++ b/src/database/test/DatabaseTests.cpp @@ -4,16 +4,22 @@ #include "util/asio.h" #include "crypto/Hex.h" +#include "crypto/KeyUtils.h" #include "database/Database.h" +#include "ledger/LedgerTxn.h" +#include "ledger/test/LedgerTestUtils.h" #include "lib/catch.hpp" #include "main/Application.h" #include "main/Config.h" #include "test/TestUtils.h" #include "test/test.h" +#include "util/Decoder.h" #include "util/Logging.h" #include "util/Math.h" #include "util/Timer.h" #include "util/TmpDir.h" +#include "util/optional.h" +#include #include using namespace stellar; @@ -394,3 +400,300 @@ class SchemaUpgradeTestApplication : public TestApplication mPreUpgradeFunc); } }; + +// This tests upgrading from MIN_SCHEMA_VERSION to SCHEMA_VERSION by creating +// ledger entries with the old MIN_SCHEMA_VERSION after database creation, then +// validating their contents after the upgrade to SCHEMA_VERSION. +TEST_CASE("schema upgrade test", "[db]") +{ + using OptLiabilities = stellar::optional; + + auto addOneOldSchemaAccount = [](SchemaUpgradeTestApplication& app, + AccountEntry const& ae) { + auto& session = app.getDatabase().getSession(); + auto accountIDStr = KeyUtils::toStrKey(ae.accountID); + auto inflationDestStr = + ae.inflationDest ? KeyUtils::toStrKey(*ae.inflationDest) + : ""; + auto inflationDestInd = ae.inflationDest ? soci::i_ok : soci::i_null; + auto homeDomainStr = decoder::encode_b64(ae.homeDomain); + auto signersStr = decoder::encode_b64(xdr::xdr_to_opaque(ae.signers)); + auto thresholdsStr = decoder::encode_b64(ae.thresholds); + bool const liabilitiesPresent = (ae.ext.v() >= 1); + int64_t buyingLiabilities = + liabilitiesPresent ? ae.ext.v1().liabilities.buying : 0; + int64_t sellingLiabilities = + liabilitiesPresent ? ae.ext.v1().liabilities.selling : 0; + soci::indicator liabilitiesInd = + liabilitiesPresent ? soci::i_ok : soci::i_null; + + soci::transaction tx(session); + + // Use raw SQL to perform database operations, since we're writing in an + // old database format, and calling standard interfaces to create + // accounts or trustlines would use SQL corresponding to the new format + // to which we'll soon upgrade. + session << "INSERT INTO accounts ( " + "accountid, balance, seqnum, numsubentries, inflationdest," + "homedomain, thresholds, signers, flags, lastmodified, " + "buyingliabilities, sellingliabilities " + ") VALUES ( " + ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10, " + ":v11 " + ")", + soci::use(accountIDStr), soci::use(ae.balance), + soci::use(ae.seqNum), soci::use(ae.numSubEntries), + soci::use(inflationDestStr, inflationDestInd), + soci::use(homeDomainStr), soci::use(thresholdsStr), + soci::use(signersStr), soci::use(ae.flags), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()), + soci::use(buyingLiabilities, liabilitiesInd), + soci::use(sellingLiabilities, liabilitiesInd); + + tx.commit(); + }; + + auto addOneOldSchemaTrustLine = [](SchemaUpgradeTestApplication& app, + TrustLineEntry const& tl) { + auto& session = app.getDatabase().getSession(); + std::string accountIDStr, issuerStr, assetCodeStr; + getTrustLineStrings(tl.accountID, tl.asset, accountIDStr, issuerStr, + assetCodeStr); + int32_t assetType = tl.asset.type(); + bool const liabilitiesPresent = (tl.ext.v() >= 1); + int64_t buyingLiabilities = + liabilitiesPresent ? tl.ext.v1().liabilities.buying : 0; + int64_t sellingLiabilities = + liabilitiesPresent ? tl.ext.v1().liabilities.selling : 0; + soci::indicator liabilitiesInd = + liabilitiesPresent ? soci::i_ok : soci::i_null; + + soci::transaction tx(session); + + session << "INSERT INTO trustlines ( " + "accountid, assettype, issuer, assetcode," + "tlimit, balance, flags, lastmodified, " + "buyingliabilities, sellingliabilities " + ") VALUES ( " + ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9 " + ")", + soci::use(accountIDStr), soci::use(assetType), soci::use(issuerStr), + soci::use(assetCodeStr), soci::use(tl.limit), soci::use(tl.balance), + soci::use(tl.flags), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()), + soci::use(buyingLiabilities, liabilitiesInd), + soci::use(sellingLiabilities, liabilitiesInd); + + tx.commit(); + }; + + auto addOneOldSchemaDataEntry = [](SchemaUpgradeTestApplication& app, + DataEntry const& de) { + auto& session = app.getDatabase().getSession(); + auto accountIDStr = KeyUtils::toStrKey(de.accountID); + auto dataNameStr = decoder::encode_b64(de.dataName); + auto dataValueStr = decoder::encode_b64(de.dataValue); + + soci::transaction tx(session); + + session << "INSERT INTO accountdata ( " + "accountid, dataname, datavalue, lastmodified " + ") VALUES ( :id, :v1, :v2, :v3 )", + soci::use(accountIDStr), soci::use(dataNameStr), + soci::use(dataValueStr), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()); + + tx.commit(); + }; + + auto addOneOldSchemaOfferEntry = [](SchemaUpgradeTestApplication& app, + OfferEntry const& oe) { + auto& session = app.getDatabase().getSession(); + auto sellerIDStr = KeyUtils::toStrKey(oe.sellerID); + auto sellingStr = decoder::encode_b64(xdr::xdr_to_opaque(oe.selling)); + auto buyingStr = decoder::encode_b64(xdr::xdr_to_opaque(oe.buying)); + double price = double(oe.price.n) / double(oe.price.d); + + soci::transaction tx(session); + + session << "INSERT INTO offers ( " + "sellerid, offerid, sellingasset, buyingasset, " + "amount, pricen, priced, price, flags, lastmodified " + ") VALUES ( " + ":v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10 " + ")", + soci::use(sellerIDStr), soci::use(oe.offerID), + soci::use(sellingStr), soci::use(buyingStr), soci::use(oe.amount), + soci::use(oe.price.n), soci::use(oe.price.d), soci::use(price), + soci::use(oe.flags), + soci::use(app.getLedgerManager().getLastClosedLedgerNum()); + + tx.commit(); + }; + + auto prepOldSchemaDB = + [addOneOldSchemaAccount, addOneOldSchemaTrustLine, + addOneOldSchemaDataEntry, + addOneOldSchemaOfferEntry](SchemaUpgradeTestApplication& app, + std::vector const& aes, + std::vector const& tls, + DataEntry const& de, OfferEntry const& oe) { + for (auto ae : aes) + { + addOneOldSchemaAccount(app, ae); + } + + for (auto tl : tls) + { + addOneOldSchemaTrustLine(app, tl); + } + + addOneOldSchemaDataEntry(app, de); + addOneOldSchemaOfferEntry(app, oe); + }; + + auto testOneDBMode = [prepOldSchemaDB](Config::TestDbMode const dbMode) { + // A vector of optional Liabilities entries, for each of which the test + // will generate a valid account. + auto const accOptLiabilities = { + nullopt(), + make_optional(Liabilities{12, 17}), + make_optional(Liabilities{4, 0}), + nullopt(), + make_optional(Liabilities{3, 0}), + make_optional(Liabilities{11, 11}), + make_optional(Liabilities{0, 0})}; + + // A vector of optional Liabilities entries, for each of which the test + // will generate a valid trustline. + auto const tlOptLiabilities = { + make_optional(Liabilities{1, 0}), + make_optional(Liabilities{0, 6}), + nullopt(), + make_optional(Liabilities{0, 0}), + make_optional(Liabilities{5, 8}), + nullopt()}; + + // Generate from each of the optional liabilities in accOptLiabilities a + // new valid account. + std::vector accountEntries; + std::transform( + accOptLiabilities.begin(), accOptLiabilities.end(), + std::back_inserter(accountEntries), [](OptLiabilities const& aol) { + AccountEntry ae = LedgerTestUtils::generateValidAccountEntry(); + if (aol) + { + ae.ext.v(1); + ae.ext.v1().liabilities = *aol; + } + else + { + ae.ext.v(0); + } + return ae; + }); + + // Generate from each of the optional liabilities in tlOptLiabilities a + // new valid trustline. + std::vector trustLineEntries; + std::transform(tlOptLiabilities.begin(), tlOptLiabilities.end(), + std::back_inserter(trustLineEntries), + [](OptLiabilities const& tlol) { + TrustLineEntry tl = + LedgerTestUtils::generateValidTrustLineEntry(); + if (tlol) + { + tl.ext.v(1); + tl.ext.v1().liabilities = *tlol; + } + else + { + tl.ext.v(0); + } + return tl; + }); + + // Create a data entry to test that its extension and ledger + // entry extension have the default (empty-XDR-union) values. + auto de = LedgerTestUtils::generateValidDataEntry(); + REQUIRE(de.ext.v() == 0); + REQUIRE(de.ext == DataEntry::_ext_t()); + + // Create an offer entry to test that its extension and ledger + // entry extension have the default (empty-XDR-union) values. + auto oe = LedgerTestUtils::generateValidOfferEntry(); + REQUIRE(oe.ext.v() == 0); + REQUIRE(oe.ext == OfferEntry::_ext_t()); + + // Create the application, with the code above that inserts old-schema + // accounts and trustlines into the database injected between database + // creation and upgrade. + Config const& cfg = getTestConfig(0, dbMode); + VirtualClock clock; + Application::pointer app = + createTestApplication( + clock, cfg, + [prepOldSchemaDB, accountEntries, trustLineEntries, de, + oe](SchemaUpgradeTestApplication& sapp) { + prepOldSchemaDB(sapp, accountEntries, trustLineEntries, de, + oe); + }); + app->start(); + + // Validate that the accounts and trustlines have the expected + // liabilities and ledger entry extensions now that the database upgrade + // has completed. + + LedgerTxn ltx(app->getLedgerTxnRoot()); + + for (auto ae : accountEntries) + { + LedgerEntry entry; + entry.data.type(ACCOUNT); + entry.data.account() = ae; + LedgerKey key = LedgerEntryKey(entry); + auto aeUpgraded = ltx.load(key); + REQUIRE(aeUpgraded.current() == entry); + } + + for (auto tl : trustLineEntries) + { + LedgerEntry entry; + entry.data.type(TRUSTLINE); + entry.data.trustLine() = tl; + LedgerKey key = LedgerEntryKey(entry); + auto tlUpgraded = ltx.load(key); + REQUIRE(tlUpgraded.current() == entry); + } + + { + LedgerEntry entry; + entry.data.type(DATA); + entry.data.data() = de; + LedgerKey key = LedgerEntryKey(entry); + auto deUpgraded = ltx.load(key); + REQUIRE(deUpgraded.current() == entry); + } + + { + LedgerEntry entry; + entry.data.type(OFFER); + entry.data.offer() = oe; + LedgerKey key = LedgerEntryKey(entry); + auto oeUpgraded = ltx.load(key); + REQUIRE(oeUpgraded.current() == entry); + } + }; + + for (auto dbMode : + {Config::TESTDB_IN_MEMORY_SQLITE, Config::TESTDB_ON_DISK_SQLITE +#ifdef USE_POSTGRES + , + Config::TESTDB_POSTGRESQL +#endif // USE_POSTGRES + }) + { + testOneDBMode(dbMode); + } +} diff --git a/src/ledger/LedgerTxn.h b/src/ledger/LedgerTxn.h index 15430e56d3..69a6af90ed 100644 --- a/src/ledger/LedgerTxn.h +++ b/src/ledger/LedgerTxn.h @@ -330,6 +330,10 @@ class WorstBestOfferIterator std::shared_ptr const& offerDescriptor() const; }; +void getTrustLineStrings(AccountID const& accountID, Asset const& asset, + std::string& accountIDStr, std::string& issuerStr, + std::string& assetCodeStr); + // An abstraction for an object that can be the parent of an AbstractLedgerTxn // (discussed below). Allows children to commit atomically to the parent. Has no // notion of a LedgerTxnEntry or LedgerTxnHeader (discussed respectively in diff --git a/src/ledger/LedgerTxnTrustLineSQL.cpp b/src/ledger/LedgerTxnTrustLineSQL.cpp index b53e46ac90..d3329a0176 100644 --- a/src/ledger/LedgerTxnTrustLineSQL.cpp +++ b/src/ledger/LedgerTxnTrustLineSQL.cpp @@ -16,7 +16,7 @@ namespace stellar { -static void +void getTrustLineStrings(AccountID const& accountID, Asset const& asset, std::string& accountIDStr, std::string& issuerStr, std::string& assetCodeStr)