Added asserts for revert reasons (moonbeam-foundation#2173)

tests/package.json

@@ -94,7 +94,7 @@
     "tcp-port-used": "^1.0.2",
     "ts-node": "^10.9.1",
     "typescript": "^4.9.3",
-    "web3": "^1.8.0"
+    "web3": "^1.8.2"
   },
   "devDependencies": {
     "@types/tcp-port-used": "^1.0.1"

tests/tests/test-precompile/test-precompile-proxy-non-transfer.ts

@@ -21,7 +21,7 @@ import {
   PRECOMPILE_AUTHOR_MAPPING_ADDRESS,
   PRECOMPILE_PROXY_ADDRESS,
 } from "../../util/constants";
-import { expectEVMResult } from "../../util/eth-transactions";
+import { expectEVMResult, extractRevertReason } from "../../util/eth-transactions";
 
 const AUTHOR_MAPPING_CONTRACT = getCompiled("AuthorMapping");
 const AUTHOR_MAPPING_INTERFACE = new ethers.utils.Interface(AUTHOR_MAPPING_CONTRACT.contract.abi);
@@ -63,7 +63,7 @@ describeDevMoonbeam("Proxy : Non transfer - Evm transfer", (context) => {
   it("should fail in simple evm transfer", async function () {
     const beforeCharlethBalance = BigInt(await context.web3.eth.getBalance(CHARLETH_ADDRESS));
     const {
-      result: { events },
+      result: { events, hash },
     } = await context.createBlock(
       createTransaction(context, {
         ...BALTATHAR_TRANSACTION_TEMPLATE,
@@ -73,6 +73,9 @@ describeDevMoonbeam("Proxy : Non transfer - Evm transfer", (context) => {
       })
     );
     expectEVMResult(events, "Revert");
+    const revertReason = await extractRevertReason(hash, context.ethers);
+    expect(revertReason).to.contain("CallFiltered");
+
     const afterCharlethBalance = BigInt(await context.web3.eth.getBalance(CHARLETH_ADDRESS));
     expect(afterCharlethBalance).to.be.eq(beforeCharlethBalance);
   });

tests/tests/test-precompile/test-precompile-proxy.ts

@@ -17,7 +17,7 @@ import {
   CONTRACT_PROXY_TYPE_STAKING,
   PRECOMPILE_PROXY_ADDRESS,
 } from "../../util/constants";
-import { expectEVMResult } from "../../util/eth-transactions";
+import { expectEVMResult, extractRevertReason } from "../../util/eth-transactions";
 import { web3EthCall } from "../../util/providers";
 
 const PROXY_CONTRACT_JSON = getCompiled("Proxy");
@@ -52,6 +52,9 @@ describeDevMoonbeam("Precompile - Proxy - add proxy fails if pre-existing proxy"
       })
     );
     expectEVMResult(result.events, "Revert");
+
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    expect(revertReason).to.contain("Cannot add more than one proxy");
   });
 });
 
@@ -103,6 +106,12 @@ describeDevMoonbeam("Precompile - Proxy - remove proxy fails if no existing prox
       })
     );
     expectEVMResult(result.events, "Revert");
+
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    // Full error expected
+    // Dispatched call failed with error: Module(ModuleError { index: 22, error: [1, 0, 0, 0],
+    // message: Some("NotFound") } )
+    expect(revertReason).to.contain("NotFound");
   });
 });
 
@@ -312,9 +321,10 @@ describeDevMoonbeam("Precompile - Proxy - is proxy - succeeds if exists", (conte
 
 describeDevMoonbeam("Pallet proxy - shouldn't accept unknown proxy", (context) => {
   it("shouldn't accept unknown proxy", async function () {
+    context.web3.eth.handleRevert = true;
     const beforeCharlethBalance = BigInt(await context.web3.eth.getBalance(CHARLETH_ADDRESS));
     const {
-      result: { events },
+      result: { events, hash },
     } = await context.createBlock(
       createTransaction(context, {
         ...BALTATHAR_TRANSACTION_TEMPLATE,
@@ -323,8 +333,10 @@ describeDevMoonbeam("Pallet proxy - shouldn't accept unknown proxy", (context) =
         value: 100,
       })
     );
-    // TODO: check revert reason
+
     expectEVMResult(events, "Revert");
+    const revertReason = await extractRevertReason(hash, context.ethers);
+    expect(revertReason).to.contain("Not proxy");
     const afterCharlethBalance = BigInt(await context.web3.eth.getBalance(CHARLETH_ADDRESS));
     expect(afterCharlethBalance - beforeCharlethBalance).to.be.eq(0n);
   });
@@ -398,7 +410,7 @@ describeDevMoonbeam("Pallet proxy - shouldn't accept removed proxy", (context) =
     expectEVMResult(events2, "Succeed");
 
     const {
-      result: { events: events3 },
+      result: { events: events3, hash: hash3 },
     } = await context.createBlock(
       createTransaction(context, {
         ...BALTATHAR_TRANSACTION_TEMPLATE,
@@ -408,6 +420,9 @@ describeDevMoonbeam("Pallet proxy - shouldn't accept removed proxy", (context) =
       })
     );
     expectEVMResult(events3, "Revert");
+
+    const revertReason = await extractRevertReason(hash3, context.ethers);
+    expect(revertReason).to.contain("Not proxy");
     const afterCharlethBalance = BigInt(await context.web3.eth.getBalance(CHARLETH_ADDRESS));
     expect(afterCharlethBalance - beforeCharlethBalance).to.be.eq(0n);
   });
@@ -432,7 +447,7 @@ describeDevMoonbeam("Pallet proxy - shouldn't accept instant for delayed proxy",
     expectEVMResult(events, "Succeed");
 
     const {
-      result: { events: events2 },
+      result: { events: events2, hash: hash2 },
     } = await context.createBlock(
       createTransaction(context, {
         ...BALTATHAR_TRANSACTION_TEMPLATE,
@@ -442,6 +457,8 @@ describeDevMoonbeam("Pallet proxy - shouldn't accept instant for delayed proxy",
       })
     );
     expectEVMResult(events2, "Revert");
+    const revertReason = await extractRevertReason(hash2, context.ethers);
+    expect(revertReason).to.contain("Unannounced");
     const afterCharlethBalance = BigInt(await context.web3.eth.getBalance(CHARLETH_ADDRESS));
     expect(afterCharlethBalance - beforeCharlethBalance).to.be.eq(0n);
   });

tests/tests/test-precompile/test-precompile-referenda-demo.ts

@@ -216,7 +216,8 @@ describeDevMoonbeam("Precompiles - Referenda Auto Upgrade Demo", (context) => {
         [trackName, setStorageCallIndex]
       );
       const { result } = await context.createBlock(contract.rawTx);
-      expectEVMResult(result.events, "Revert");
+      expectEVMResult(result.events, "Revert"); // No Revert reason to validate
+
       expect(
         (await context.polkadotApi.query.evm.accountCodes(contract.contractAddress)).toHex(),
         "Contract should not have been deployed"

tests/tests/test-precompile/test-precompile-referenda.ts

@@ -14,7 +14,7 @@ import { expectOk, expectSubstrateEvent, expectSubstrateEvents } from "../../uti
 
 import { describeDevMoonbeam } from "../../util/setup-dev-tests";
 import { createContract, createContractExecution } from "../../util/transactions";
-import { expectEVMResult } from "../../util/eth-transactions";
+import { expectEVMResult, extractRevertReason } from "../../util/eth-transactions";
 const debug = Debug("test:precompile-referenda");
 
 const REFERENDA_CONTRACT = getCompiled("Referenda");
@@ -79,6 +79,11 @@ describeDevMoonbeam("Precompiles - Referenda precompile", (context) => {
         })
       );
       expectEVMResult(block.result.events, "Revert");
+      const revertReason = await extractRevertReason(block.result.hash, context.ethers);
+      // Full Error expected:
+      // Dispatched call failed with error: Module(ModuleError { index: 42, error: [0, 0, 0, 0],
+      //     message: Some("NotOngoing") })
+      expect(revertReason).to.contain("NotOngoing");
     }
   });
 

tests/tests/test-precompile/test-precompile-xcm-utils.ts

@@ -12,7 +12,7 @@ import { BN } from "@polkadot/util";
 import type { XcmVersionedXcm } from "@polkadot/types/lookup";
 import { descendOriginFromAddress } from "../../util/xcm";
 import { ALITH_TRANSACTION_TEMPLATE, createTransaction } from "../../util/transactions";
-import { expectEVMResult } from "../../util/eth-transactions";
+import { expectEVMResult, extractRevertReason } from "../../util/eth-transactions";
 
 export const CLEAR_ORIGIN_WEIGHT = 8_514_000n;
 
@@ -239,6 +239,12 @@ describeDevMoonbeam(
         })
       );
       expectEVMResult(result.events, "Revert");
+
+      const revertReason = await extractRevertReason(result.hash, context.ethers);
+      // Full error expected:
+      // Dispatched call failed with error: Module(ModuleError { index: 0, error: [5, 0, 0, 0],
+      //  message: Some("CallFiltered") })
+      expect(revertReason).to.contain("CallFiltered");
     });
   },
   "Legacy",
@@ -285,6 +291,12 @@ describeDevMoonbeam(
         })
       );
       expectEVMResult(result.events, "Revert");
+
+      const revertReason = await extractRevertReason(result.hash, context.ethers);
+      // Full error expected:
+      // Dispatched call failed with error: Module(ModuleError { index: 0, error: [5, 0, 0, 0],
+      // message: Some("CallFiltered") })
+      expect(revertReason).to.contain("CallFiltered");
     });
   },
   "Legacy",
@@ -458,6 +470,11 @@ describeDevMoonbeamAllEthTxTypes("Precompiles - xcm utils", (context) => {
     // Verify the result
     // Expect success
     expectEVMResult(resultHere.events, "Revert");
+    const revertReason = await extractRevertReason(resultHere.hash, context.ethers);
+    // Full error expected:
+    // Dispatched call failed with error: Module(ModuleError { index: 28, error: [0, 0, 0, 0],
+    // message: Some("Unreachable") })
+    expect(revertReason).to.contain("Unreachable");
 
     // Try sending it with para relay view
     const { result: resultParaRelayView } = await context.createBlock(
@@ -474,6 +491,11 @@ describeDevMoonbeamAllEthTxTypes("Precompiles - xcm utils", (context) => {
     // Verify the result
     // Expect success
     expectEVMResult(resultParaRelayView.events, "Revert");
+    const revertReason2 = await extractRevertReason(resultHere.hash, context.ethers);
+    // Full error expected:
+    // Dispatched call failed with error: Module(ModuleError { index: 28, error: [0, 0, 0, 0],
+    // message: Some("Unreachable") })
+    expect(revertReason2).to.contain("Unreachable");
 
     // Try sending it with another para view (parents 1)
     const { result: resultParaOtherParaView } = await context.createBlock(
@@ -490,5 +512,11 @@ describeDevMoonbeamAllEthTxTypes("Precompiles - xcm utils", (context) => {
     // Verify the result
     // Expect success
     expectEVMResult(resultParaOtherParaView.events, "Revert");
+
+    const revertReason3 = await extractRevertReason(resultParaOtherParaView.hash, context.ethers);
+    // Full error expected:
+    // Dispatched call failed with error: Module(ModuleError { index: 28, error: [1, 0, 0, 0],
+    // message: Some("SendFailure") })
+    expect(revertReason3).to.contain("SendFailure");
   });
 });

tests/tests/test-randomness/test-randomness-babe-request.ts

@@ -11,7 +11,7 @@ import {
   PRECOMPILE_RANDOMNESS_ADDRESS,
 } from "../../util/constants";
 import { getCompiled } from "../../util/contracts";
-import { expectEVMResult } from "../../util/eth-transactions";
+import { expectEVMResult, extractRevertReason } from "../../util/eth-transactions";
 import { describeDevMoonbeam } from "../../util/setup-dev-tests";
 import { ALITH_TRANSACTION_TEMPLATE, createTransaction } from "../../util/transactions";
 
@@ -154,6 +154,12 @@ describeDevMoonbeam("Randomness Babe - Requesting a random number", (context) =>
     expect(result.successful).to.be.true;
     expectEVMResult(result.events, "Revert");
 
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    // Full error expected:
+    // Error in pallet_randomness: Module(ModuleError { index: 39, error: [3, 0, 0, 0],
+    // message: Some("CannotRequestMoreWordsThanMax") })
+    expect(revertReason).to.contain("CannotRequestMoreWordsThanMax");
+
     const randomnessRequests = await context.polkadotApi.query.randomness.requests.entries();
     expect(randomnessRequests.length).to.equal(0);
   });

tests/tests/test-randomness/test-randomness-vrf-lottery.ts

@@ -25,7 +25,7 @@ import {
   PRECOMPILE_RANDOMNESS_ADDRESS,
 } from "../../util/constants";
 import { getCompiled } from "../../util/contracts";
-import { expectEVMResult } from "../../util/eth-transactions";
+import { expectEVMResult, extractRevertReason } from "../../util/eth-transactions";
 import { describeDevMoonbeam, DevTestContext } from "../../util/setup-dev-tests";
 import {
   ALITH_TRANSACTION_TEMPLATE,
@@ -141,6 +141,12 @@ describeDevMoonbeam("Randomness VRF - Lottery Demo", (context) => {
       })
     );
     expectEVMResult(result.events, "Revert");
+
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    // Full error expected:
+    // Module(ModuleError { index: 39, error: [7, 0, 0, 0],
+    // message: Some("RequestCannotYetBeFulfilled") })
+    expect(revertReason).to.contain("RequestCannotYetBeFulfilled");
   });
 
   it("should be rolling the numbers", async function () {

tests/tests/test-randomness/test-randomness-vrf-request.ts

@@ -11,7 +11,7 @@ import {
   PRECOMPILE_RANDOMNESS_ADDRESS,
 } from "../../util/constants";
 import { getCompiled } from "../../util/contracts";
-import { expectEVMResult } from "../../util/eth-transactions";
+import { expectEVMResult, extractRevertReason } from "../../util/eth-transactions";
 import { describeDevMoonbeam } from "../../util/setup-dev-tests";
 import { ALITH_TRANSACTION_TEMPLATE, createTransaction } from "../../util/transactions";
 
@@ -157,6 +157,12 @@ describeDevMoonbeam("Randomness VRF - Requesting a random number", (context) =>
     expect(result.successful).to.be.true;
     expectEVMResult(result.events, "Revert");
 
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    // Full error expected:
+    // Error in pallet_randomness: Module(ModuleError { index: 39, error: [5, 0, 0, 0],
+    // message: Some("CannotRequestRandomnessBeforeMinDelay") })
+    expect(revertReason).to.contain("CannotRequestRandomnessBeforeMinDelay");
+
     const randomnessRequests = await context.polkadotApi.query.randomness.requests.entries();
     expect(randomnessRequests.length).to.equal(0);
   });
@@ -180,6 +186,12 @@ describeDevMoonbeam("Randomness VRF - Requesting a random number", (context) =>
     expect(result.successful).to.be.true;
     expectEVMResult(result.events, "Revert");
 
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    // Full error expected:
+    // Error in pallet_randomness: Module(ModuleError { index: 39, error: [4, 0, 0, 0],
+    // message: Some("CannotRequestRandomnessAfterMaxDelay") })
+    expect(revertReason).to.contain("CannotRequestRandomnessAfterMaxDelay");
+
     const randomnessRequests = await context.polkadotApi.query.randomness.requests.entries();
     expect(randomnessRequests.length).to.equal(0);
   });
@@ -204,6 +216,11 @@ describeDevMoonbeam("Randomness VRF - Requesting a random number", (context) =>
 
     expect(result.successful).to.be.true;
     expectEVMResult(result.events, "Revert");
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    // Full error expected:
+    // Error in pallet_randomness: Module(ModuleError { index: 39, error: [2, 0, 0, 0],
+    // message: Some("MustRequestAtLeastOneWord") })
+    expect(revertReason).to.contain("MustRequestAtLeastOneWord");
 
     const randomnessRequests = await context.polkadotApi.query.randomness.requests.entries();
     expect(randomnessRequests.length).to.equal(0);
@@ -229,6 +246,11 @@ describeDevMoonbeam("Randomness VRF - Requesting a random number", (context) =>
 
     expect(result.successful).to.be.true;
     expectEVMResult(result.events, "Revert");
+    const revertReason = await extractRevertReason(result.hash, context.ethers);
+    // Full error expected:
+    // Error in pallet_randomness: Module(ModuleError { index: 39, error: [3, 0, 0, 0],
+    // message: Some("CannotRequestMoreWordsThanMax") })
+    expect(revertReason).to.contain("CannotRequestMoreWordsThanMax");
 
     const randomnessRequests = await context.polkadotApi.query.randomness.requests.entries();
     expect(randomnessRequests.length).to.equal(0);

tests/util/eth-transactions.ts

@@ -7,6 +7,7 @@ import {
   EvmCoreErrorExitRevert,
   EvmCoreErrorExitFatal,
 } from "@polkadot/types/lookup";
+import { ethers } from "ethers";
 
 export type Errors = {
   Succeed: EvmCoreErrorExitSucceed["type"];
@@ -15,6 +16,20 @@ export type Errors = {
   Fatal: EvmCoreErrorExitFatal["type"];
 };
 
+export async function extractRevertReason(
+  responseHash: string,
+  ethers: ethers.providers.JsonRpcProvider
+) {
+  const tx = await ethers.getTransaction(responseHash);
+  try {
+    await ethers.call(tx, tx.blockNumber);
+    return null;
+  } catch (e) {
+    const jsonError = JSON.parse(e.error.body);
+    return jsonError.error.message.split("VM Exception while processing transaction: revert ")[1];
+  }
+}
+
 export function expectEVMResult<T extends Errors, Type extends keyof T>(
   events: EventRecord[],
   resultType: Type,