Test upgrades to database schema 13

Test upgrades to database schema 13, in which all
extensions in all ledger entry types become opaque XDR
as far as the database is concerned.

The tests use the test-code-injection mechanism which allow
them to run arbitrary code between the creation of a
database (which is done in an old schema version,
MIN_SCHEMA_VERSION) and the upgrading of that database to
the current schema version (SCHEMA_VERSION).  They create
ledger entries of each of the four types, using raw SQL so
that they can create them in the old format which the live
production code no longer uses.  After the database upgrade
completes (which it has done by the time that
createTestApplication() returns), the tests validate that
the contents of the ledger entries are intact and
logically unchanged (although their representations in
the database have changed, with some columns having been
created and some having been deleted, and some contents
being encoded differently).

src/database/test/DatabaseTests.cpp

@@ -4,16 +4,22 @@
 
 #include "util/asio.h"
 #include "crypto/Hex.h"
+#include "crypto/KeyUtils.h"
 #include "database/Database.h"
+#include "ledger/LedgerTxn.h"
+#include "ledger/test/LedgerTestUtils.h"
 #include "lib/catch.hpp"
 #include "main/Application.h"
 #include "main/Config.h"
 #include "test/TestUtils.h"
 #include "test/test.h"
+#include "util/Decoder.h"
 #include "util/Logging.h"
 #include "util/Math.h"
 #include "util/Timer.h"
 #include "util/TmpDir.h"
+#include "util/optional.h"
+#include <algorithm>
 #include <random>
 
 using namespace stellar;
@@ -394,3 +400,300 @@ class SchemaUpgradeTestApplication : public TestApplication
                                                            mPreUpgradeFunc);
     }
 };
+
+// This tests upgrading from MIN_SCHEMA_VERSION to SCHEMA_VERSION by creating
+// ledger entries with the old MIN_SCHEMA_VERSION after database creation, then
+// validating their contents after the upgrade to SCHEMA_VERSION.
+TEST_CASE("schema upgrade test", "[db]")
+{
+    using OptLiabilities = stellar::optional<Liabilities>;
+
+    auto addOneOldSchemaAccount = [](SchemaUpgradeTestApplication& app,
+                                     AccountEntry const& ae) {
+        auto& session = app.getDatabase().getSession();
+        auto accountIDStr = KeyUtils::toStrKey<PublicKey>(ae.accountID);
+        auto inflationDestStr =
+            ae.inflationDest ? KeyUtils::toStrKey<PublicKey>(*ae.inflationDest)
+                             : "";
+        auto inflationDestInd = ae.inflationDest ? soci::i_ok : soci::i_null;
+        auto homeDomainStr = decoder::encode_b64(ae.homeDomain);
+        auto signersStr = decoder::encode_b64(xdr::xdr_to_opaque(ae.signers));
+        auto thresholdsStr = decoder::encode_b64(ae.thresholds);
+        bool const liabilitiesPresent = (ae.ext.v() >= 1);
+        int64_t buyingLiabilities =
+            liabilitiesPresent ? ae.ext.v1().liabilities.buying : 0;
+        int64_t sellingLiabilities =
+            liabilitiesPresent ? ae.ext.v1().liabilities.selling : 0;
+        soci::indicator liabilitiesInd =
+            liabilitiesPresent ? soci::i_ok : soci::i_null;
+
+        soci::transaction tx(session);
+
+        // Use raw SQL to perform database operations, since we're writing in an
+        // old database format, and calling standard interfaces to create
+        // accounts or trustlines would use SQL corresponding to the new format
+        // to which we'll soon upgrade.
+        session << "INSERT INTO accounts ( "
+                   "accountid, balance, seqnum, numsubentries, inflationdest,"
+                   "homedomain, thresholds, signers, flags, lastmodified, "
+                   "buyingliabilities, sellingliabilities "
+                   ") VALUES ( "
+                   ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10, "
+                   ":v11 "
+                   ")",
+            soci::use(accountIDStr), soci::use(ae.balance),
+            soci::use(ae.seqNum), soci::use(ae.numSubEntries),
+            soci::use(inflationDestStr, inflationDestInd),
+            soci::use(homeDomainStr), soci::use(thresholdsStr),
+            soci::use(signersStr), soci::use(ae.flags),
+            soci::use(app.getLedgerManager().getLastClosedLedgerNum()),
+            soci::use(buyingLiabilities, liabilitiesInd),
+            soci::use(sellingLiabilities, liabilitiesInd);
+
+        tx.commit();
+    };
+
+    auto addOneOldSchemaTrustLine = [](SchemaUpgradeTestApplication& app,
+                                       TrustLineEntry const& tl) {
+        auto& session = app.getDatabase().getSession();
+        std::string accountIDStr, issuerStr, assetCodeStr;
+        getTrustLineStrings(tl.accountID, tl.asset, accountIDStr, issuerStr,
+                            assetCodeStr);
+        int32_t assetType = tl.asset.type();
+        bool const liabilitiesPresent = (tl.ext.v() >= 1);
+        int64_t buyingLiabilities =
+            liabilitiesPresent ? tl.ext.v1().liabilities.buying : 0;
+        int64_t sellingLiabilities =
+            liabilitiesPresent ? tl.ext.v1().liabilities.selling : 0;
+        soci::indicator liabilitiesInd =
+            liabilitiesPresent ? soci::i_ok : soci::i_null;
+
+        soci::transaction tx(session);
+
+        session << "INSERT INTO trustlines ( "
+                   "accountid, assettype, issuer, assetcode,"
+                   "tlimit, balance, flags, lastmodified, "
+                   "buyingliabilities, sellingliabilities "
+                   ") VALUES ( "
+                   ":id, :v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9 "
+                   ")",
+            soci::use(accountIDStr), soci::use(assetType), soci::use(issuerStr),
+            soci::use(assetCodeStr), soci::use(tl.limit), soci::use(tl.balance),
+            soci::use(tl.flags),
+            soci::use(app.getLedgerManager().getLastClosedLedgerNum()),
+            soci::use(buyingLiabilities, liabilitiesInd),
+            soci::use(sellingLiabilities, liabilitiesInd);
+
+        tx.commit();
+    };
+
+    auto addOneOldSchemaDataEntry = [](SchemaUpgradeTestApplication& app,
+                                       DataEntry const& de) {
+        auto& session = app.getDatabase().getSession();
+        auto accountIDStr = KeyUtils::toStrKey<PublicKey>(de.accountID);
+        auto dataNameStr = decoder::encode_b64(de.dataName);
+        auto dataValueStr = decoder::encode_b64(de.dataValue);
+
+        soci::transaction tx(session);
+
+        session << "INSERT INTO accountdata ( "
+                   "accountid, dataname, datavalue, lastmodified "
+                   ") VALUES ( :id, :v1, :v2, :v3 )",
+            soci::use(accountIDStr), soci::use(dataNameStr),
+            soci::use(dataValueStr),
+            soci::use(app.getLedgerManager().getLastClosedLedgerNum());
+
+        tx.commit();
+    };
+
+    auto addOneOldSchemaOfferEntry = [](SchemaUpgradeTestApplication& app,
+                                        OfferEntry const& oe) {
+        auto& session = app.getDatabase().getSession();
+        auto sellerIDStr = KeyUtils::toStrKey(oe.sellerID);
+        auto sellingStr = decoder::encode_b64(xdr::xdr_to_opaque(oe.selling));
+        auto buyingStr = decoder::encode_b64(xdr::xdr_to_opaque(oe.buying));
+        double price = double(oe.price.n) / double(oe.price.d);
+
+        soci::transaction tx(session);
+
+        session << "INSERT INTO offers ( "
+                   "sellerid, offerid, sellingasset, buyingasset, "
+                   "amount, pricen, priced, price, flags, lastmodified "
+                   ") VALUES ( "
+                   ":v1, :v2, :v3, :v4, :v5, :v6, :v7, :v8, :v9, :v10 "
+                   ")",
+            soci::use(sellerIDStr), soci::use(oe.offerID),
+            soci::use(sellingStr), soci::use(buyingStr), soci::use(oe.amount),
+            soci::use(oe.price.n), soci::use(oe.price.d), soci::use(price),
+            soci::use(oe.flags),
+            soci::use(app.getLedgerManager().getLastClosedLedgerNum());
+
+        tx.commit();
+    };
+
+    auto prepOldSchemaDB =
+        [addOneOldSchemaAccount, addOneOldSchemaTrustLine,
+         addOneOldSchemaDataEntry,
+         addOneOldSchemaOfferEntry](SchemaUpgradeTestApplication& app,
+                                    std::vector<AccountEntry> const& aes,
+                                    std::vector<TrustLineEntry> const& tls,
+                                    DataEntry const& de, OfferEntry const& oe) {
+            for (auto ae : aes)
+            {
+                addOneOldSchemaAccount(app, ae);
+            }
+
+            for (auto tl : tls)
+            {
+                addOneOldSchemaTrustLine(app, tl);
+            }
+
+            addOneOldSchemaDataEntry(app, de);
+            addOneOldSchemaOfferEntry(app, oe);
+        };
+
+    auto testOneDBMode = [prepOldSchemaDB](Config::TestDbMode const dbMode) {
+        // A vector of optional Liabilities entries, for each of which the test
+        // will generate a valid account.
+        auto const accOptLiabilities = {
+            nullopt<Liabilities>(),
+            make_optional<Liabilities>(Liabilities{12, 17}),
+            make_optional<Liabilities>(Liabilities{4, 0}),
+            nullopt<Liabilities>(),
+            make_optional<Liabilities>(Liabilities{3, 0}),
+            make_optional<Liabilities>(Liabilities{11, 11}),
+            make_optional<Liabilities>(Liabilities{0, 0})};
+
+        // A vector of optional Liabilities entries, for each of which the test
+        // will generate a valid trustline.
+        auto const tlOptLiabilities = {
+            make_optional<Liabilities>(Liabilities{1, 0}),
+            make_optional<Liabilities>(Liabilities{0, 6}),
+            nullopt<Liabilities>(),
+            make_optional<Liabilities>(Liabilities{0, 0}),
+            make_optional<Liabilities>(Liabilities{5, 8}),
+            nullopt<Liabilities>()};
+
+        // Generate from each of the optional liabilities in accOptLiabilities a
+        // new valid account.
+        std::vector<AccountEntry> accountEntries;
+        std::transform(
+            accOptLiabilities.begin(), accOptLiabilities.end(),
+            std::back_inserter(accountEntries), [](OptLiabilities const& aol) {
+                AccountEntry ae = LedgerTestUtils::generateValidAccountEntry();
+                if (aol)
+                {
+                    ae.ext.v(1);
+                    ae.ext.v1().liabilities = *aol;
+                }
+                else
+                {
+                    ae.ext.v(0);
+                }
+                return ae;
+            });
+
+        // Generate from each of the optional liabilities in tlOptLiabilities a
+        // new valid trustline.
+        std::vector<TrustLineEntry> trustLineEntries;
+        std::transform(tlOptLiabilities.begin(), tlOptLiabilities.end(),
+                       std::back_inserter(trustLineEntries),
+                       [](OptLiabilities const& tlol) {
+                           TrustLineEntry tl =
+                               LedgerTestUtils::generateValidTrustLineEntry();
+                           if (tlol)
+                           {
+                               tl.ext.v(1);
+                               tl.ext.v1().liabilities = *tlol;
+                           }
+                           else
+                           {
+                               tl.ext.v(0);
+                           }
+                           return tl;
+                       });
+
+        // Create a data entry to test that its extension and ledger
+        // entry extension have the default (empty-XDR-union) values.
+        auto de = LedgerTestUtils::generateValidDataEntry();
+        REQUIRE(de.ext.v() == 0);
+        REQUIRE(de.ext == DataEntry::_ext_t());
+
+        // Create an offer entry to test that its extension and ledger
+        // entry extension have the default (empty-XDR-union) values.
+        auto oe = LedgerTestUtils::generateValidOfferEntry();
+        REQUIRE(oe.ext.v() == 0);
+        REQUIRE(oe.ext == OfferEntry::_ext_t());
+
+        // Create the application, with the code above that inserts old-schema
+        // accounts and trustlines into the database injected between database
+        // creation and upgrade.
+        Config const& cfg = getTestConfig(0, dbMode);
+        VirtualClock clock;
+        Application::pointer app =
+            createTestApplication<SchemaUpgradeTestApplication,
+                                  SchemaUpgradeTestApplication::PreUpgradeFunc>(
+                clock, cfg,
+                [prepOldSchemaDB, accountEntries, trustLineEntries, de,
+                 oe](SchemaUpgradeTestApplication& sapp) {
+                    prepOldSchemaDB(sapp, accountEntries, trustLineEntries, de,
+                                    oe);
+                });
+        app->start();
+
+        // Validate that the accounts and trustlines have the expected
+        // liabilities and ledger entry extensions now that the database upgrade
+        // has completed.
+
+        LedgerTxn ltx(app->getLedgerTxnRoot());
+
+        for (auto ae : accountEntries)
+        {
+            LedgerEntry entry;
+            entry.data.type(ACCOUNT);
+            entry.data.account() = ae;
+            LedgerKey key = LedgerEntryKey(entry);
+            auto aeUpgraded = ltx.load(key);
+            REQUIRE(aeUpgraded.current() == entry);
+        }
+
+        for (auto tl : trustLineEntries)
+        {
+            LedgerEntry entry;
+            entry.data.type(TRUSTLINE);
+            entry.data.trustLine() = tl;
+            LedgerKey key = LedgerEntryKey(entry);
+            auto tlUpgraded = ltx.load(key);
+            REQUIRE(tlUpgraded.current() == entry);
+        }
+
+        {
+            LedgerEntry entry;
+            entry.data.type(DATA);
+            entry.data.data() = de;
+            LedgerKey key = LedgerEntryKey(entry);
+            auto deUpgraded = ltx.load(key);
+            REQUIRE(deUpgraded.current() == entry);
+        }
+
+        {
+            LedgerEntry entry;
+            entry.data.type(OFFER);
+            entry.data.offer() = oe;
+            LedgerKey key = LedgerEntryKey(entry);
+            auto oeUpgraded = ltx.load(key);
+            REQUIRE(oeUpgraded.current() == entry);
+        }
+    };
+
+    for (auto dbMode :
+         {Config::TESTDB_IN_MEMORY_SQLITE, Config::TESTDB_ON_DISK_SQLITE
+#ifdef USE_POSTGRES
+          ,
+          Config::TESTDB_POSTGRESQL
+#endif // USE_POSTGRES
+         })
+    {
+        testOneDBMode(dbMode);
+    }
+}

src/ledger/LedgerTxn.h

@@ -330,6 +330,10 @@ class WorstBestOfferIterator
     std::shared_ptr<OfferDescriptor const> const& offerDescriptor() const;
 };
 
+void getTrustLineStrings(AccountID const& accountID, Asset const& asset,
+                         std::string& accountIDStr, std::string& issuerStr,
+                         std::string& assetCodeStr);
+
 // An abstraction for an object that can be the parent of an AbstractLedgerTxn
 // (discussed below). Allows children to commit atomically to the parent. Has no
 // notion of a LedgerTxnEntry or LedgerTxnHeader (discussed respectively in

src/ledger/LedgerTxnTrustLineSQL.cpp

@@ -16,7 +16,7 @@
 namespace stellar
 {
 
-static void
+void
 getTrustLineStrings(AccountID const& accountID, Asset const& asset,
                     std::string& accountIDStr, std::string& issuerStr,
                     std::string& assetCodeStr)