Merge #1190

1190: Add database migration test r=KtorZ a=rvl Relates to #1177. # Overview Adds a migration test that can be run against different versions of the wallet. The idea is to use an api client to set up the database on a wallet server running one version. Then start up the wallet server on the current version and use the api client to check the database. All the possible upgrade paths from previous released versions to the current version _could_ be exhaustively tested. However, in this initial version, only the direct upgrade from each release to latest is tested. At present the testing actions are quite basic. It first restores a wallet, migrates, then lists the wallets. The migration tests are run as part of nightly CI. Co-authored-by: Rodney Lorrimar <rodney.lorrimar@iohk.io> Co-authored-by: KtorZ <matthias.benkort@gmail.com>
cardano-foundation · Jan 19, 2020 · a3b8164 · a3b8164
2 parents 5b06b37 + fd9f64a
commit a3b8164
Show file tree

Hide file tree

Showing 12 changed files with 639 additions and 8 deletions.
diff --git a/.buildkite/nightly.yml b/.buildkite/nightly.yml
@@ -18,6 +18,15 @@ steps:
     agents:
       system: x86_64-linux
 
+  - label: 'Database Migrations Test'
+    commands:
+      - "rm -rf state-migration-test*"
+      - "nix-build nix/migration-tests.nix -o migration-tests"
+      - "./migration-tests/runall.sh"
+    timeout_in_minutes: 60
+    agents:
+      system: x86_64-linux
+
   - block: 'Delete CI Caches'
 
   - label: 'Clean up CI cache'

diff --git a/default.nix b/default.nix
@@ -39,12 +39,14 @@ let
       haskellBuildUtils = iohkLib.haskellBuildUtils.package;
     };
 
-    # `tests` are the test suites which have been built
+    # `tests` are the test suites which have been built.
     tests = collectComponents "tests" isCardanoWallet haskellPackages;
+    # `checks` are the result of executing the tests.
+    checks = pkgs.recurseIntoAttrs (getPackageChecks (filterCardanoPackages haskellPackages));
+    # `benchmarks` are only built, not run.
     benchmarks = collectComponents "benchmarks" isCardanoWallet haskellPackages;
-    # `checks` are the result of executing the tests
-    checks = pkgs.recurseIntoAttrs (getPackageChecks
-       (filterCardanoPackages haskellPackages));
+    # `migration-tests` build previous releases then check if the database successfully upgrades.
+    migration-tests = import ./nix/migration-tests.nix { inherit system crossSystem config pkgs; };
 
     dockerImage = pkgs.callPackage ./nix/docker.nix {
       inherit (self) cardano-wallet-jormungandr;

diff --git a/lib/core/src/Cardano/Wallet.hs b/lib/core/src/Cardano/Wallet.hs
@@ -75,11 +75,13 @@ module Cardano.Wallet
     , walletSyncProgress
     , fetchRewardBalance
     , rollbackBlocks
+    , checkWalletIntegrity
     , ErrWalletAlreadyExists (..)
     , ErrNoSuchWallet (..)
     , ErrListUTxOStatistics (..)
     , ErrUpdatePassphrase (..)
     , ErrFetchRewards (..)
+    , ErrCheckWalletIntegrity (..)
 
     -- ** Address
     , listAddresses
@@ -264,6 +266,8 @@ import Cardano.Wallet.Transaction
     )
 import Control.DeepSeq
     ( NFData )
+import Control.Exception
+    ( Exception )
 import Control.Monad
     ( forM, forM_, when )
 import Control.Monad.IO.Class
@@ -524,6 +528,26 @@ createIcarusWallet ctx wid wname credentials = db & \DBLayer{..} -> do
     db = ctx ^. dbLayer @s @k
     (block0, bp, _) = ctx ^. genesisData
 
+-- | Check whether a wallet is in good shape when restarting a worker.
+checkWalletIntegrity
+    :: forall ctx s k. HasDBLayer s k ctx
+    => ctx
+    -> WalletId
+    -> BlockchainParameters
+    -> ExceptT ErrCheckWalletIntegrity IO ()
+checkWalletIntegrity ctx wid bp = db & \DBLayer{..} -> mapExceptT atomically $ do
+    cp <- withExceptT ErrCheckWalletIntegrityNoSuchWallet $ withNoSuchWallet wid $
+        readCheckpoint (PrimaryKey wid)
+    whenDifferentGenesis (blockchainParameters cp) bp $ throwE $
+        ErrCheckIntegrityDifferentGenesis
+            (getGenesisBlockHash bp)
+            (getGenesisBlockHash (blockchainParameters cp))
+  where
+    db = ctx ^. dbLayer @s @k
+    whenDifferentGenesis bp1 bp2 = when $
+        (bp1 ^. #getGenesisBlockHash /= bp2 ^. #getGenesisBlockHash) ||
+        (bp1 ^. #getGenesisBlockDate /= bp2 ^. #getGenesisBlockDate)
+
 -- | Retrieve the wallet state for the wallet with the given ID.
 readWallet
     :: forall ctx s k. HasDBLayer s k ctx
@@ -1558,6 +1582,13 @@ data ErrSelectForMigration
         -- ^ User attempted to migrate an empty wallet
     deriving (Eq, Show)
 
+data ErrCheckWalletIntegrity
+    = ErrCheckWalletIntegrityNoSuchWallet ErrNoSuchWallet
+    | ErrCheckIntegrityDifferentGenesis (Hash "Genesis") (Hash "Genesis")
+    deriving (Eq, Show)
+
+instance Exception ErrCheckWalletIntegrity
+
 {-------------------------------------------------------------------------------
                                    Utils
 -------------------------------------------------------------------------------}

diff --git a/lib/core/src/Cardano/Wallet/Api/Server.hs b/lib/core/src/Cardano/Wallet/Api/Server.hs
@@ -207,7 +207,7 @@ import Control.Arrow
 import Control.DeepSeq
     ( NFData )
 import Control.Exception
-    ( IOException, bracket, tryJust )
+    ( IOException, bracket, throwIO, tryJust )
 import Control.Exception.Lifted
     ( finally )
 import Control.Monad
@@ -1477,11 +1477,12 @@ registerWorker ctx wid = do
         Just worker ->
             Registry.insert re worker
   where
+    (_, bp, _) = ctx ^. genesisData
     re = ctx ^. workerRegistry
     df = ctx ^. dbFactory
     config = MkWorker
-        { workerBefore =
-            \_ _ -> return ()
+        { workerBefore = \ctx' _ ->
+            runExceptT (W.checkWalletIntegrity ctx' wid bp) >>= either throwIO pure
 
         , workerMain = \ctx' _ -> do
             -- FIXME:

diff --git a/lib/core/src/Cardano/Wallet/DB/Sqlite.hs b/lib/core/src/Cardano/Wallet/DB/Sqlite.hs
@@ -353,7 +353,7 @@ migrateManually tr defaultFieldValues =
         getCheckpointTableInfo <- Sqlite.prepare conn $ mconcat
             [ "SELECT sql FROM sqlite_master "
             , "WHERE type = 'table' "
-            , "AND name = '" <> fieldName field <> "';"
+            , "AND name = '" <> tableName field <> "';"
             ]
         row <- Sqlite.step getCheckpointTableInfo
             >> Sqlite.columns getCheckpointTableInfo

diff --git a/lib/jormungandr/cardano-wallet-jormungandr.cabal b/lib/jormungandr/cardano-wallet-jormungandr.cabal
@@ -306,3 +306,33 @@ benchmark latency
       Test.Integration.Jcli
       Cardano.Faucet
       Cardano.Wallet.Jormungandr.Launch
+
+executable migration-test
+  default-language:
+      Haskell2010
+  default-extensions:
+      NoImplicitPrelude
+      OverloadedStrings
+  ghc-options:
+      -threaded -rtsopts
+      -Wall
+      -O2
+  if (!flag(development))
+    ghc-options:
+      -Werror
+  build-depends:
+      base
+    , aeson
+    , cardano-wallet-core
+    , cardano-wallet-launcher
+    , iohk-monitoring
+    , lens
+    , lens-aeson
+    , process
+    , retry
+    , text
+    , wreq
+  hs-source-dirs:
+      test/migration
+  main-is:
+      migration-test.hs