merge master and resolve conflicts

arbitrator/stylus/tests/hostio-test/src/main.rs

@@ -204,4 +204,37 @@ impl HostioTest {
     fn tx_origin() -> Result<Address> {
         Ok(tx::origin())
     }
+
+    fn storage_cache_bytes32() {
+        let key = B256::ZERO;
+        let val = B256::ZERO;
+        unsafe {
+            hostio::storage_cache_bytes32(key.as_ptr(), val.as_ptr());
+        }
+    }
+
+    fn pay_for_memory_grow(pages: U256) {
+        let pages: u16 = pages.try_into().unwrap();
+        unsafe {
+            hostio::pay_for_memory_grow(pages);
+        }
+    }
+
+    fn write_result_empty() {
+    }
+
+    fn write_result(size: U256) -> Result<Vec<u32>> {
+        let size: usize = size.try_into().unwrap();
+        let data = vec![0; size];
+        Ok(data)
+    }
+
+    fn read_args_no_args() {
+    }
+
+    fn read_args_one_arg(_arg1: U256) {
+    }
+
+    fn read_args_three_args(_arg1: U256, _arg2: U256, _arg3: U256) {
+    }
 }

arbitrator/wasm-libraries/user-host-trait/src/lib.rs

@@ -936,7 +936,7 @@ pub trait UserHost<DR: DataReader>: GasMeteredMachine {
     fn pay_for_memory_grow(&mut self, pages: u16) -> Result<(), Self::Err> {
         if pages == 0 {
             self.buy_ink(HOSTIO_INK)?;
-            return Ok(());
+            return trace!("pay_for_memory_grow", self, be!(pages), &[]);
         }
         let gas_cost = self.evm_api().add_pages(pages); // no sentry needed since the work happens after the hostio
         self.buy_gas(gas_cost)?;

arbnode/inbox_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
 	"github.com/offchainlabs/nitro/arbos/l2pricing"
 	"github.com/offchainlabs/nitro/arbutil"
+	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/execution/gethexec"
 	"github.com/offchainlabs/nitro/statetransfer"
 	"github.com/offchainlabs/nitro/util/arbmath"
@@ -44,7 +45,7 @@ func (w *execClientWrapper) FullSyncProgressMap() map[string]interface{} {
 }
 
 func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (*gethexec.ExecutionEngine, *TransactionStreamer, ethdb.Database, *core.BlockChain) {
-	chainConfig := params.ArbitrumDevTestChainConfig()
+	chainConfig := chaininfo.ArbitrumDevTestChainConfig()
 
 	initData := statetransfer.ArbosInitializationInfo{
 		Accounts: []statetransfer.AccountInitializationInfo{

arbnode/sync_monitor.go

@@ -146,11 +146,13 @@ func (s *SyncMonitor) FullSyncProgressMap() map[string]interface{} {
 		batchProcessed := s.inboxReader.GetLastReadBatchCount()
 		res["batchProcessed"] = batchProcessed
 
-		processedBatchMsgs, err := s.inboxReader.Tracker().GetBatchMessageCount(batchProcessed - 1)
-		if err != nil {
-			res["batchMetadataError"] = err.Error()
-		} else {
-			res["messageOfProcessedBatch"] = processedBatchMsgs
+		if batchProcessed > 0 {
+			processedBatchMsgs, err := s.inboxReader.Tracker().GetBatchMessageCount(batchProcessed - 1)
+			if err != nil {
+				res["batchMetadataError"] = err.Error()
+			} else {
+				res["messageOfProcessedBatch"] = processedBatchMsgs
+			}
 		}
 
 		l1reader := s.inboxReader.l1Reader

arbos/addressSet/addressSet_test.go

@@ -14,11 +14,11 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/params"
 
 	"github.com/offchainlabs/nitro/arbos/burn"
 	"github.com/offchainlabs/nitro/arbos/storage"
 	"github.com/offchainlabs/nitro/arbos/util"
+	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/util/colors"
 	"github.com/offchainlabs/nitro/util/testhelpers"
 )
@@ -27,7 +27,7 @@ func TestEmptyAddressSet(t *testing.T) {
 	sto := storage.NewMemoryBacked(burn.NewSystemBurner(nil, false))
 	Require(t, Initialize(sto))
 	aset := OpenAddressSet(sto)
-	version := params.ArbitrumDevTestParams().InitialArbOSVersion
+	version := chaininfo.ArbitrumDevTestParams().InitialArbOSVersion
 
 	if size(t, aset) != 0 {
 		Fail(t)
@@ -50,7 +50,7 @@ func TestAddressSet(t *testing.T) {
 	sto := storage.NewGeth(db, burn.NewSystemBurner(nil, false))
 	Require(t, Initialize(sto))
 	aset := OpenAddressSet(sto)
-	version := params.ArbitrumDevTestParams().InitialArbOSVersion
+	version := chaininfo.ArbitrumDevTestParams().InitialArbOSVersion
 
 	statedb, _ := (db).(*state.StateDB)
 	stateHashBeforeChanges := statedb.IntermediateRoot(false)
@@ -145,7 +145,7 @@ func TestAddressSetAllMembers(t *testing.T) {
 	sto := storage.NewGeth(db, burn.NewSystemBurner(nil, false))
 	Require(t, Initialize(sto))
 	aset := OpenAddressSet(sto)
-	version := params.ArbitrumDevTestParams().InitialArbOSVersion
+	version := chaininfo.ArbitrumDevTestParams().InitialArbOSVersion
 
 	addr1 := testhelpers.RandomAddress()
 	addr2 := testhelpers.RandomAddress()

arbos/arbosState/arbosstate.go

@@ -32,6 +32,7 @@ import (
 	"github.com/offchainlabs/nitro/arbos/retryables"
 	"github.com/offchainlabs/nitro/arbos/storage"
 	"github.com/offchainlabs/nitro/arbos/util"
+	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/util/testhelpers/env"
 )
 
@@ -128,7 +129,7 @@ func NewArbosMemoryBackedArbOSState() (*ArbosState, *state.StateDB) {
 		log.Crit("failed to init empty statedb", "error", err)
 	}
 	burner := burn.NewSystemBurner(nil, false)
-	chainConfig := params.ArbitrumDevTestChainConfig()
+	chainConfig := chaininfo.ArbitrumDevTestChainConfig()
 	newState, err := InitializeArbosState(statedb, burner, chainConfig, arbostypes.TestInitMessage)
 	if err != nil {
 		log.Crit("failed to open the ArbOS state", "error", err)

arbos/arbosState/initialization_test.go

@@ -13,10 +13,10 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/params"
 
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
 	"github.com/offchainlabs/nitro/arbos/burn"
+	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/statetransfer"
 	"github.com/offchainlabs/nitro/util/testhelpers"
 	"github.com/offchainlabs/nitro/util/testhelpers/env"
@@ -62,7 +62,7 @@ func tryMarshalUnmarshal(input *statetransfer.ArbosInitializationInfo, t *testin
 	raw := rawdb.NewMemoryDatabase()
 
 	initReader := statetransfer.NewMemoryInitDataReader(&initData)
-	chainConfig := params.ArbitrumDevTestChainConfig()
+	chainConfig := chaininfo.ArbitrumDevTestChainConfig()
 
 	cacheConfig := core.DefaultCacheConfigWithScheme(env.GetTestStateScheme())
 	stateroot, err := InitializeArbosInDatabase(raw, cacheConfig, initReader, chainConfig, arbostypes.TestInitMessage, 0, 0)

arbos/arbostypes/incomingmessage.go

@@ -16,6 +16,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 
 	"github.com/offchainlabs/nitro/arbos/util"
+	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/util/arbmath"
 )
 
@@ -265,7 +266,7 @@ type ParsedInitMessage struct {
 var DefaultInitialL1BaseFee = big.NewInt(50 * params.GWei)
 
 var TestInitMessage = &ParsedInitMessage{
-	ChainId:          params.ArbitrumDevTestChainConfig().ChainID,
+	ChainId:          chaininfo.ArbitrumDevTestChainConfig().ChainID,
 	InitialL1BaseFee: DefaultInitialL1BaseFee,
 }
 

arbos/block_processor.go

@@ -340,18 +340,6 @@ func ProduceBlockAdvanced(
 			return receipt, result, nil
 		})()
 
-		if tx.Type() == types.ArbitrumInternalTxType {
-			// ArbOS might have upgraded to a new version, so we need to refresh our state
-			state, err = arbosState.OpenSystemArbosState(statedb, nil, true)
-			if err != nil {
-				return nil, nil, err
-			}
-			// Update the ArbOS version in the header (if it changed)
-			extraInfo := types.DeserializeHeaderExtraInformation(header)
-			extraInfo.ArbOSFormatVersion = state.ArbOSVersion()
-			extraInfo.UpdateHeaderWithInfo(header)
-		}
-
 		// append the err, even if it is nil
 		hooks.TxErrors = append(hooks.TxErrors, err)
 
@@ -373,6 +361,18 @@ func ProduceBlockAdvanced(
 			continue
 		}
 
+		if tx.Type() == types.ArbitrumInternalTxType {
+			// ArbOS might have upgraded to a new version, so we need to refresh our state
+			state, err = arbosState.OpenSystemArbosState(statedb, nil, true)
+			if err != nil {
+				return nil, nil, err
+			}
+			// Update the ArbOS version in the header (if it changed)
+			extraInfo := types.DeserializeHeaderExtraInformation(header)
+			extraInfo.ArbOSFormatVersion = state.ArbOSVersion()
+			extraInfo.UpdateHeaderWithInfo(header)
+		}
+
 		if tx.Type() == types.ArbitrumInternalTxType && result.Err != nil {
 			return nil, nil, fmt.Errorf("failed to apply internal transaction: %w", result.Err)
 		}

arbos/l1pricing/l1pricing.go

@@ -21,6 +21,7 @@ import (
 	"github.com/offchainlabs/nitro/arbcompress"
 	"github.com/offchainlabs/nitro/arbos/storage"
 	"github.com/offchainlabs/nitro/arbos/util"
+	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/util/arbmath"
 	am "github.com/offchainlabs/nitro/util/arbmath"
 )
@@ -540,7 +541,7 @@ var randomNonce = binary.BigEndian.Uint64(crypto.Keccak256([]byte("Nonce"))[:8])
 var randomGasTipCap = new(big.Int).SetBytes(crypto.Keccak256([]byte("GasTipCap"))[:4])
 var randomGasFeeCap = new(big.Int).SetBytes(crypto.Keccak256([]byte("GasFeeCap"))[:4])
 var RandomGas = uint64(binary.BigEndian.Uint32(crypto.Keccak256([]byte("Gas"))[:4]))
-var randV = arbmath.BigMulByUint(params.ArbitrumOneChainConfig().ChainID, 3)
+var randV = arbmath.BigMulByUint(chaininfo.ArbitrumOneChainConfig().ChainID, 3)
 var randR = crypto.Keccak256Hash([]byte("R")).Big()
 var randS = crypto.Keccak256Hash([]byte("S")).Big()
 

arbos/l1pricing_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/offchainlabs/nitro/arbos/burn"
 	"github.com/offchainlabs/nitro/arbos/l1pricing"
 	"github.com/offchainlabs/nitro/arbos/util"
+	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/util/arbmath"
 )
 
@@ -317,7 +318,7 @@ func _withinOnePercent(v1, v2 *big.Int) bool {
 }
 
 func newMockEVMForTesting() *vm.EVM {
-	chainConfig := params.ArbitrumDevTestChainConfig()
+	chainConfig := chaininfo.ArbitrumDevTestChainConfig()
 	_, statedb := arbosState.NewArbosMemoryBackedArbOSState()
 	context := vm.BlockContext{
 		BlockNumber: big.NewInt(0),

broadcastclient/broadcastclient.go

@@ -130,9 +130,10 @@ type BroadcastClient struct {
 
 	chainId uint64
 
-	// Protects conn and shuttingDown
-	connMutex sync.Mutex
-	conn      net.Conn
+	// Protects conn, shuttingDown and compression
+	connMutex   sync.Mutex
+	conn        net.Conn
+	compression bool
 
 	retryCount atomic.Int64
 
@@ -299,7 +300,7 @@ func (bc *BroadcastClient) connect(ctx context.Context, nextSeqNum arbutil.Messa
 		return nil, nil
 	}
 
-	conn, br, _, err := timeoutDialer.Dial(ctx, bc.websocketUrl)
+	conn, br, hs, err := timeoutDialer.Dial(ctx, bc.websocketUrl)
 	if errors.Is(err, ErrIncorrectFeedServerVersion) || errors.Is(err, ErrIncorrectChainId) {
 		return nil, err
 	}
@@ -325,6 +326,24 @@ func (bc *BroadcastClient) connect(ctx context.Context, nextSeqNum arbutil.Messa
 		return nil, ErrMissingFeedServerVersion
 	}
 
+	compressionNegotiated := false
+	for _, ext := range hs.Extensions {
+		if ext.Equal(deflateExt) {
+			compressionNegotiated = true
+			break
+		}
+	}
+	if !compressionNegotiated && config.EnableCompression {
+		log.Warn("Compression was not negotiated when connecting to feed server.")
+	}
+	if compressionNegotiated && !config.EnableCompression {
+		err := conn.Close()
+		if err != nil {
+			return nil, fmt.Errorf("error closing connection when negotiated disabled extension: %w", err)
+		}
+		return nil, errors.New("error dialing feed server: negotiated compression ws extension, but it is disabled")
+	}
+
 	var earlyFrameData io.Reader
 	if br != nil {
 		// Depending on how long the client takes to read the response, there may be
@@ -339,6 +358,7 @@ func (bc *BroadcastClient) connect(ctx context.Context, nextSeqNum arbutil.Messa
 
 	bc.connMutex.Lock()
 	bc.conn = conn
+	bc.compression = compressionNegotiated
 	bc.connMutex.Unlock()
 	log.Info("Feed connected", "feedServerVersion", feedServerVersion, "chainId", chainId, "requestedSeqNum", nextSeqNum)
 
@@ -362,7 +382,7 @@ func (bc *BroadcastClient) startBackgroundReader(earlyFrameData io.Reader) {
 			var op ws.OpCode
 			var err error
 			config := bc.config()
-			msg, op, err = wsbroadcastserver.ReadData(ctx, bc.conn, earlyFrameData, config.Timeout, ws.StateClientSide, config.EnableCompression, flateReader)
+			msg, op, err = wsbroadcastserver.ReadData(ctx, bc.conn, earlyFrameData, config.Timeout, ws.StateClientSide, bc.compression, flateReader)
 			if err != nil {
 				if bc.isShuttingDown() {
 					return

broadcastclient/broadcastclient_test.go

@@ -30,43 +30,30 @@ import (
 	"github.com/offchainlabs/nitro/wsbroadcastserver"
 )
 
-func TestReceiveMessagesWithoutCompression(t *testing.T) {
+func TestReceiveMessages(t *testing.T) {
 	t.Parallel()
-	testReceiveMessages(t, false, false, false, false)
-}
-
-func TestReceiveMessagesWithCompression(t *testing.T) {
-	t.Parallel()
-	testReceiveMessages(t, true, true, false, false)
-}
-
-func TestReceiveMessagesWithServerOptionalCompression(t *testing.T) {
-	t.Parallel()
-	testReceiveMessages(t, true, true, false, false)
-}
-
-func TestReceiveMessagesWithServerOnlyCompression(t *testing.T) {
-	t.Parallel()
-	testReceiveMessages(t, false, true, false, false)
-}
-
-func TestReceiveMessagesWithClientOnlyCompression(t *testing.T) {
-	t.Parallel()
-	testReceiveMessages(t, true, false, false, false)
-}
-
-func TestReceiveMessagesWithRequiredCompression(t *testing.T) {
-	t.Parallel()
-	testReceiveMessages(t, true, true, true, false)
-}
-
-func TestReceiveMessagesWithRequiredCompressionButClientDisabled(t *testing.T) {
-	t.Parallel()
-	testReceiveMessages(t, false, true, true, true)
+	t.Run("withoutCompression", func(t *testing.T) {
+		testReceiveMessages(t, false, false, false, false)
+	})
+	t.Run("withServerOptionalCompression", func(t *testing.T) {
+		testReceiveMessages(t, true, true, false, false)
+	})
+	t.Run("withServerOnlyCompression", func(t *testing.T) {
+		testReceiveMessages(t, false, true, false, false)
+	})
+	t.Run("withClientOnlyCompression", func(t *testing.T) {
+		testReceiveMessages(t, true, false, false, false)
+	})
+	t.Run("withRequiredCompression", func(t *testing.T) {
+		testReceiveMessages(t, true, true, true, false)
+	})
+	t.Run("withRequiredCompressionButClientDisabled", func(t *testing.T) {
+		testReceiveMessages(t, false, true, true, true)
+	})
 }
 
 func testReceiveMessages(t *testing.T, clientCompression bool, serverCompression bool, serverRequire bool, expectNoMessagesReceived bool) {
-	t.Helper()
+	t.Parallel()
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()