eth.getBlock sometimes does not return at all; sometimes produces "TypeError: Cannot create property 'gasLimit' on string"

[wbt]

This is a follow-up to #3568 in which I attempt to provide a reproducible example for that Issue and wind up creating a reproducible example for a different issue which seems like it might be related.

Steps to reproduce the behavior (commands for command-line shell)

1. Start Ganache (steps developed in v2.4.0) running in a new Ethereum workspace (so that block numbers line up).
2. mkdir blockArrayTest
3. cd blockArrayTest
4. truffle init
5. npm init [OK to accept defaults]
6. Edit truffle-config.js networks.development to point to Ganache (change host/port/network_id as appropriate)
7. Add file contracts/Counter.sol:

//This is a simplified bug-demo contract with no security
pragma solidity ^0.5.0;

contract Counter {
    int public value;
    event ValueUpdated(int oldVal, int newVal);

    constructor() public {
        value = 0;
    }

    function increment() public {
        emit ValueUpdated(value, ++value);
    }
}

8. Add file migrations/2_adjust_value.js:

var Counter = artifacts.require("Counter");
module.exports = function(deployer) {
    let contractInstance;
    deployer.deploy(Counter).
    then(function(instance) {
        contractInstance = instance;
        return contractInstance.increment();
    }).then(function() {
        return contractInstance.increment();
    }).then(function() {
        return contractInstance.increment();
    }).then(function() {
        return contractInstance.increment();
    }).then(function() {
        return contractInstance.increment();
    }).then(function() {
        return contractInstance.value();
    }).then(function(value) {
        console.log("Incremented counter to "+value); //should say 5
    }).catch(function(err) {
        console.log(err.message);
    });
};

9. truffle migrate --reset --network development
10. npm i @truffle/contract
11. npm i web3
12. Add file getEvents.js:

const TruffleContract= require("@truffle/contract");
const Web3 = require("web3");
let counterContract = TruffleContract(require("./build/contracts/Counter.json"));
let counterInstance;
//Connect to Ganache:
let provider = new Web3.providers.WebsocketProvider('ws://127.0.0.1:7545');
let web3 = new Web3(provider);
console.log("Using web3 version "+web3.version);
counterContract.setProvider(provider);
counterContract.deployed().
then(function(instance) {
    counterInstance = instance;
    counterInstance.ValueUpdated({fromBlock: '0', toBlock: 'latest'})
    .on("data", function (firedEvent) {
        myEventHandlingCallback(undefined, firedEvent);
    }).on("error", function(error) {
        myEventHandlingCallback(error, undefined);
    });
}).catch(function(err) {
    console.log(err);
});

myEventHandlingCallback = function(error, eventReceived) {
    let logMessage = "Received "+eventReceived.event+
    " event updating counter value from "+
    eventReceived.returnValues["oldVal"]+" to "+
    eventReceived.returnValues["newVal"]+
    " in tx "+eventReceived.transactionHash+
    " in block "+eventReceived.blockNumber;
    logMessage += " (0x"+eventReceived.blockNumber.toString(16)+")";

    //With this conditional, calling web3object.eth.getBlock for block 8
    //reports error
    //"Cannot create property 'gasLimit' on string '0x7'"
    //where the hex value in the error increases by one each time.
    //The error side of the callback is reached
    //"Error received in getBlock() callback"... is printed.
    //web3object.eth.getBlock for block 7 just never
    //reaches its callback; there is no error for that one.
    let condition1 =
    (eventReceived.blockNumber == '4') ||
    (eventReceived.blockNumber == '5') ||
    (eventReceived.blockNumber == '6') ||
    (eventReceived.blockNumber == '7') ||
    (eventReceived.blockNumber == '8');

    //With this filter, web3object.eth.getBlock for block 8 just never
    //reaches its callback. There is no error.
    let condition2 =
    (eventReceived.blockNumber == '5') ||
    (eventReceived.blockNumber == '6') ||
    (eventReceived.blockNumber == '7') ||
    (eventReceived.blockNumber == '8');

    if(condition1) {
        console.log(logMessage);
        getBlockInfo(web3, eventReceived.blockNumber).
        then(function(blockInfo) {
            //Do more meaningful processing, incorporating blockInfo
            console.log("Got information for block "+eventReceived.blockNumber);
        }).catch(function(error) {
            console.log("Error when fetching block info for block "+
            eventReceived.blockNumber+":",error);
        });
    }
};

/**
* Passing in web3 as an object to the function instead of relying on any
* global here was meant to help with debugging by making this more of a
* pure function.
* Having a wrapper with the callback syntax
* (a) helps with code maintainability in light of historic web3 breaking changes,
* (b) allows inspection of the returned data right here, in an attempt to
* eliminate any possible interference from other variables in the context
* where the data will be used.
*/
getBlockInfo = function(web3object, blockNumber) {
    return new Promise(function(resolve, reject) {
        web3object.eth.getBlock(blockNumber, function(error, blockData) {
            if(error) {
                console.log("Error received in getBlock() callback for block "+
                blockNumber+": ",error);
                reject(error);
            } else {
                //Omitted: Code for converting blockNumber to hex and
                //comparing that against the contents of blockData if not
                //an array or each element of an array if it is, and rejecting
                //if there is no match.
                //Additional logging statements also omitted.
                if(Array.isArray(blockData)) {
                    console.log("getBlock("+typeof blockNumber+" "+
                    blockNumber+") has an ARRAY return type of length "+
                    blockData.length+":",blockData);
                } else { //not strict, but true for this investigation
                    console.log("getBlock("+typeof blockNumber+" "+
                    blockNumber+") has an object return type as expected.");
                }
                resolve(blockData);
            }
        });
    });
};

13. node getEvents.js

Expected behavior

web3.eth.getBlock "Returns a block matching the block number or block hash."
The output should include these lines, missing the first two if condition2 is used:

getBlock(number 4) has an object return type as expected.
Got information for block 4
getBlock(number 5) has an object return type as expected.
Got information for block 5
getBlock(number 6) has an object return type as expected.
Got information for block 6
getBlock(number 7) has an object return type as expected.
Got information for block 7
getBlock(number 8) has an object return type as expected.
Got information for block 8

Actual behavior

The expected lines about block 8 are absent. (I think the second half of this comment is related.)
When using condition1 the output does NOT include the expected lines about block 7 and also includes the unexpected:

Error received in getBlock() callback for block 8: TypeError: Cannot create property 'gasLimit' on string '0x7'
at Method.outputBlockFormatter [as outputFormatter] (path\to\blockArrayTest\node_modules\web3-eth\node_modules\web3-core-helpers\src\formatters.js:298:20)
at Method.formatOutput (path\to\blockArrayTest\node_modules\web3-core-method\src\index.js:167:54)
at sendTxCallback (path\to\blockArrayTest\node_modules\web3-core-method\src\index.js:622:33)
at Object.callback (path\to\blockArrayTest\node_modules\web3-core-requestmanager\src\index.js:180:9)
at path\to\blockArrayTest\node_modules\web3-providers-ws\src\index.js:133:41
at Array.forEach ()
at WebsocketProvider._onMessage (path\to\blockArrayTest\node_modules\web3-providers-ws\src\index.js:118:69)
at W3CWebSocket._dispatchEvent [as dispatchEvent] (path\to\blockArrayTest\node_modules\yaeti\lib\EventTarget.js:115:12)
at W3CWebSocket.onMessage (path\to\blockArrayTest\node_modules@web3-js\websocket\lib\W3CWebSocket.js:234:14)
at WebSocketConnection. (path\to\blockArrayTest\node_modules@web3-js\websocket\lib\W3CWebSocket.js:205:19)
at WebSocketConnection.emit (events.js:159:13)
at WebSocketConnection.processFrame (path\to\blockArrayTest\node_modules@web3-js\websocket\lib\WebSocketConnection.js:554:26)
at path\to\blockArrayTest\node_modules@web3-js\websocket\lib\WebSocketConnection.js:323:40
at process._tickCallback (internal/process/next_tick.js:150:11)

Environment

npm -version 6.14.5
node -v v9.3.0
As seen in start of output: Using web3 version 1.2.8
Windows 10, using PowerShell for commands

[wbt]

I think I’ve solved it!

The shared memory which is at the root of the issue is in the WebsocketProvider's responseQueue object.

First, some relevant points on how requests work in web3: when a request is sent (examples: eth_subscribe, eth_getLogs, eth_getBlockByNumber), the "toPayload()" method of web3-core-requestmanager/src/jsonrpc is asked to bundle it up into a request. As part of this bundling, it assigns a sequential numeric id (Jsonrpc.messageId) which starts out as 0 and is incremented just before being included in the bundle (so the first id assigned is number 1.) This payload object, with the id, is passed into the provider, which uses the id from the request payload object as a Map key for storing information about what to do with the response. When the response comes back, it includes the id and the WebSocketProvider looks in the Map to figure out what it should do with that response.

The problem in the example code above is that there are multiple separate instances of the web3-core-requestmanager’s Jsonrpc object, with separate and overlapping sequences for the request ids, but a shared Provider queue expecting those ids to be unique.

When the code above calls eth_getBlock for block numbers 4-8, it’s using node_modules/web3-core-requestmanager/src/jsonrpc.js (file paths relative to the project root). Those requests for information about blocks 4-8 become request numbers 1-5 respectively.

However, previous to this, there are five other requests, which are using node_modules/@truffle/interface-adapter/node_modules/web3-core-requestmanager/src/jsonrpc.js (and NOT, as one might have guessed, node_modules/@truffle/contract/node_modules/web3-core-requestmanager/src/jsonrpc.js, which may start yet another sequence). In the example code above, those requests are:
1: net_version()
2: eth_getBlockByNumber(‘latest’, false)
3: <exactly the same as 2>
4: eth_getLogs ({fromBlock: '0x0', toBlock: 'latest', topics:
[ '0xb8ba758880724160775cc09f9aa6f15e3d6be6aed023b548a74a72981f806f63' ],
address: '0xe8ccf70aeed0b49908bcf2e4bf64b08b2ca72fef' })
5: eth_subscribe(‘logs’, <same object as prior request, without the fromBlock member>)

In this earlier list, requests 1-3 get a response back and are deleted from the responseQueue before the second set of requests is constructed, so there’s no conflict/overlap. However, requests 4 and 5 are still active in that responseQueue at the time the eth_getBlock requests with the same ids overwrites them. When a response comes in for the earlier request, it gets sent to the handler for the later request, and the handler for the later request is removed. Then, when the response for the later (eth_getBlock) request comes in to the WebsocketProvider, it gets ignored.

To see which jsonrpc.js file is being used for which request, you can go to those files and change the starting messageId values in the different files to different values, like 30/50/70 for the three respective files. Set this line to console.log("In web3's WebsocketProvider.prototype.send, setting id "+id+" to be a request for "+request.payload.method+"(",request.payload.params,")"); and set this line to similarly inspect the id variable associated with the returning response. You can also report deletion a few lines below if that helps with understanding the control flow. (As a side note, for defensive programming, it might be good to wrap this line in a check to see if that callback is actually defined.)

The workaround is to make sure the Provider object used for contracts is different from the one used for the web3 object. That is, in the sample code above, change

let provider = new Web3.providers.WebsocketProvider('ws://127.0.0.1:7545');
let web3 = new Web3(provider);
counterContract.setProvider(provider);

to

let web3 = new Web3(new Web3.providers.WebsocketProvider('ws://127.0.0.1:7545'));
counterContract.setProvider(new Web3.providers.WebsocketProvider('ws://127.0.0.1:7545'));

which should close both this and #3568.

[ryanio]

wow @wbt thanks for digging into this!