Complete bulk memory proposal.

Signed-off-by: Takeshi Yoneda <takeshi@tetrate.io>

README.md

@@ -215,8 +215,8 @@ default. However, the following status covers what's currently possible with
 | sign-extension-ops                    |   ✅   |
 | multi-value                           |   ✅   |
 | JS-BigInt-integration                 |  N/A   |
-| reference-types                       |   ❌   |
-| bulk-memory-operations                |   ❌   |
+| reference-types                       |   👷‍♂️   |
+| bulk-memory-operations                |   ✅   |
 | simd                                  |   ❌   |
 
 Note: While the above are specified in a WebAssembly GitHub repository, they

internal/asm/arm64/impl.go

@@ -2062,6 +2062,25 @@ func (a *AssemblerImpl) EncodeConstToRegister(n *NodeImpl) (err error) {
 			0b01_000000 | byte(c),
 			0b110_10011,
 		})
+	case LSL:
+		if c == 0 {
+			err = errors.New("LSL with zero constant should be optimized out")
+			return
+		} else if c < 0 || c > 63 {
+			err = fmt.Errorf("LSL requires immediate to be within 0 to 63, but got %d", c)
+			return
+		}
+
+		// LSL(immediate) is an alias of UBFM
+		// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LSL--immediate---Logical-Shift-Left--immediate---an-alias-of-UBFM-
+		cb := byte(c)
+		a.Buf.Write([]byte{
+			(dstRegBits << 5) | dstRegBits,
+			(0b111111-cb)<<2 | dstRegBits>>3,
+			0b01_000000 | (64 - cb),
+			0b110_10011,
+		})
+
 	default:
 		return errorEncodingUnsupported(n)
 	}

internal/integration_test/asm/arm64_debug/impl_test.go

@@ -679,6 +679,10 @@ func TestAssemblerImpl_EncodeConstToRegister(t *testing.T) {
 			inst:   asm_arm64.MOVD,
 			consts: consts64,
 		},
+		{
+			inst:   asm_arm64.LSL,
+			consts: []int64{1, 2, 4, 16, 31, 32, 63},
+		},
 		{
 			inst:   asm_arm64.LSR,
 			consts: []int64{1, 2, 4, 16, 31, 32, 63},
@@ -694,7 +698,7 @@ func TestAssemblerImpl_EncodeConstToRegister(t *testing.T) {
 				t.Run(asm_arm64.RegisterName(r), func(t *testing.T) {
 					for _, c := range tc.consts {
 						var cs = []int64{c}
-						if tc.inst != asm_arm64.LSR && c != 0 {
+						if tc.inst != asm_arm64.LSR && tc.inst != asm_arm64.LSL && c != 0 {
 							cs = append(cs, -c)
 						}
 						for _, c := range cs {

internal/integration_test/post1_0/bulk-memory-operations/spec_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/tetratelabs/wazero"
 	"github.com/tetratelabs/wazero/internal/testing/require"
+	"github.com/tetratelabs/wazero/internal/wasmruntime"
 )
 
 // testCtx is an arbitrary, non-default context. Non-nil also prevents linter errors.
@@ -17,23 +18,185 @@ func TestBulkMemoryOperations_JIT(t *testing.T) {
 		t.Skip()
 	}
 	testBulkMemoryOperations(t, wazero.NewRuntimeConfigJIT)
+	testTableCopy(t, wazero.NewRuntimeConfigJIT)
+	testTableInit(t, wazero.NewRuntimeConfigJIT)
+	testElemDrop(t, wazero.NewRuntimeConfigJIT)
 }
 
 func TestBulkMemoryOperations_Interpreter(t *testing.T) {
 	testBulkMemoryOperations(t, wazero.NewRuntimeConfigInterpreter)
+	testTableCopy(t, wazero.NewRuntimeConfigInterpreter)
+	testTableInit(t, wazero.NewRuntimeConfigInterpreter)
+	testElemDrop(t, wazero.NewRuntimeConfigInterpreter)
 }
 
-// bulkMemoryOperationsWasm was compiled from testdata/bulk_memory_operations.wat
-//go:embed testdata/bulk_memory_operations.wasm
-var bulkMemoryOperationsWasm []byte
+var (
+	// bulkMemoryOperationsWasm was compiled from testdata/bulk_memory_operations.wat
+	//go:embed testdata/bulk_memory_operations.wasm
+	bulkMemoryOperationsWasm []byte
+	// tableCopyWasm was compiled from testdata/table_copy.wat
+	//go:embed testdata/table_copy.wasm
+	tableCopyWasm []byte
+	// tableInitWasm was compiled from testdata/table_init.wat
+	//go:embed testdata/table_init.wasm
+	tableInitWasm []byte
+	// elemDropWasm was compiled from testdata/elem_drop.wat
+	//go:embed testdata/elem_drop.wasm
+	elemDropWasm []byte
+)
 
-func testBulkMemoryOperations(t *testing.T, newRuntimeConfig func() *wazero.RuntimeConfig) {
+func requireErrorOnBulkMemoryFeatureDisabled(t *testing.T, newRuntimeConfig func() *wazero.RuntimeConfig, bin []byte) {
 	t.Run("disabled", func(t *testing.T) {
 		// bulk-memory-operations is disabled by default.
 		r := wazero.NewRuntimeWithConfig(newRuntimeConfig())
-		_, err := r.InstantiateModuleFromCode(testCtx, bulkMemoryOperationsWasm)
+		_, err := r.InstantiateModuleFromCode(testCtx, bin)
 		require.Error(t, err)
 	})
+}
+
+func testTableCopy(t *testing.T, newRuntimeConfig func() *wazero.RuntimeConfig) {
+	t.Run("table.copy", func(t *testing.T) {
+
+		requireErrorOnBulkMemoryFeatureDisabled(t, newRuntimeConfig, tableCopyWasm)
+
+		r := wazero.NewRuntimeWithConfig(newRuntimeConfig().WithFeatureBulkMemoryOperations(true))
+		mod, err := r.InstantiateModuleFromCode(testCtx, tableCopyWasm)
+		require.NoError(t, err)
+		defer mod.Close(testCtx)
+
+		// Non-overlapping copy.
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 3, 0, 3)
+		require.NoError(t, err)
+		res, err := mod.ExportedFunction("call").Call(testCtx, 3)
+		require.NoError(t, err)
+		require.Equal(t, uint64(0), res[0])
+		res, err = mod.ExportedFunction("call").Call(testCtx, 4)
+		require.NoError(t, err)
+		require.Equal(t, uint64(1), res[0])
+		res, err = mod.ExportedFunction("call").Call(testCtx, 5)
+		require.NoError(t, err)
+		require.Equal(t, uint64(2), res[0])
+
+		// src > dest with overlap
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 0, 1, 3)
+		require.NoError(t, err)
+		res, err = mod.ExportedFunction("call").Call(testCtx, 0)
+		require.NoError(t, err)
+		require.Equal(t, uint64(1), res[0])
+		res, err = mod.ExportedFunction("call").Call(testCtx, 1)
+		require.NoError(t, err)
+		require.Equal(t, uint64(2), res[0])
+		res, err = mod.ExportedFunction("call").Call(testCtx, 2)
+		require.NoError(t, err)
+		require.Equal(t, uint64(0), res[0])
+
+		// src < dest with overlap
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 2, 0, 3)
+		require.NoError(t, err)
+		res, err = mod.ExportedFunction("call").Call(testCtx, 2)
+		require.NoError(t, err)
+		require.Equal(t, uint64(1), res[0])
+		res, err = mod.ExportedFunction("call").Call(testCtx, 3)
+		require.NoError(t, err)
+		require.Equal(t, uint64(2), res[0])
+		res, err = mod.ExportedFunction("call").Call(testCtx, 4)
+		require.NoError(t, err)
+		require.Equal(t, uint64(0), res[0])
+
+		// Copying end at limit should be fine.
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 6, 8, 2)
+		require.NoError(t, err)
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 8, 6, 2)
+		require.NoError(t, err)
+
+		// Copying zero size at the end of region is valid.
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 10, 0, 0)
+		require.NoError(t, err)
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 0, 10, 0)
+		require.NoError(t, err)
+
+		// Out of bounds with size zero on outside of table.
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 11, 0, 0)
+		require.ErrorIs(t, err, wasmruntime.ErrRuntimeInvalidTableAccess)
+		_, err = mod.ExportedFunction("copy").Call(testCtx, 0, 11, 0)
+		require.ErrorIs(t, err, wasmruntime.ErrRuntimeInvalidTableAccess)
+	})
+}
+
+func testTableInit(t *testing.T, newRuntimeConfig func() *wazero.RuntimeConfig) {
+	t.Run("table.init", func(t *testing.T) {
+		requireErrorOnBulkMemoryFeatureDisabled(t, newRuntimeConfig, tableInitWasm)
+
+		r := wazero.NewRuntimeWithConfig(newRuntimeConfig().WithFeatureBulkMemoryOperations(true))
+		mod, err := r.InstantiateModuleFromCode(testCtx, tableInitWasm)
+		require.NoError(t, err)
+		defer mod.Close(testCtx)
+
+		// Out of bounds access should raise the runtime error.
+		_, err = mod.ExportedFunction("init").Call(testCtx, 2, 0, 2)
+		require.ErrorIs(t, err, wasmruntime.ErrRuntimeInvalidTableAccess)
+		// And the table still not initialized.
+		_, err = mod.ExportedFunction("call").Call(testCtx, 2)
+		require.ErrorIs(t, err, wasmruntime.ErrRuntimeInvalidTableAccess)
+
+		_, err = mod.ExportedFunction("init").Call(testCtx, 0, 1, 2)
+		require.NoError(t, err)
+		res, err := mod.ExportedFunction("call").Call(testCtx, 0)
+		require.NoError(t, err)
+		require.Equal(t, uint64(1), res[0])
+		res, err = mod.ExportedFunction("call").Call(testCtx, 1)
+		require.NoError(t, err)
+		require.Equal(t, uint64(0), res[0])
+
+		// Initialization ending at the limit should be fine.
+		_, err = mod.ExportedFunction("init").Call(testCtx, 1, 2, 2)
+		require.NoError(t, err)
+		// Also, zero length at the end also fine.
+		_, err = mod.ExportedFunction("init").Call(testCtx, 3, 0, 0)
+		require.NoError(t, err)
+		_, err = mod.ExportedFunction("init").Call(testCtx, 0, 4, 0)
+		require.NoError(t, err)
+
+		// Initialization out side of table with size zero should be trap.
+		_, err = mod.ExportedFunction("init").Call(testCtx, 4, 0, 0)
+		require.ErrorIs(t, err, wasmruntime.ErrRuntimeInvalidTableAccess)
+		// Same goes for element.
+		_, err = mod.ExportedFunction("init").Call(testCtx, 0, 5, 0)
+		require.ErrorIs(t, err, wasmruntime.ErrRuntimeInvalidTableAccess)
+	})
+}
+
+func testElemDrop(t *testing.T, newRuntimeConfig func() *wazero.RuntimeConfig) {
+	t.Run("elem.drop", func(t *testing.T) {
+		requireErrorOnBulkMemoryFeatureDisabled(t, newRuntimeConfig, elemDropWasm)
+
+		r := wazero.NewRuntimeWithConfig(newRuntimeConfig().WithFeatureBulkMemoryOperations(true))
+		mod, err := r.InstantiateModuleFromCode(testCtx, elemDropWasm)
+		require.NoError(t, err)
+		defer mod.Close(testCtx)
+
+		// Copying the passive element $a into index zero at the table.
+		_, err = mod.ExportedFunction("init_passive").Call(testCtx, 1)
+		require.NoError(t, err)
+		// Droppig same elements should be fine.
+		_, err = mod.ExportedFunction("drop_passive").Call(testCtx)
+		require.NoError(t, err)
+		_, err = mod.ExportedFunction("drop_passive").Call(testCtx)
+		require.NoError(t, err)
+
+		// Size zero init access to the size zero (dropped) elements should be ok.
+		_, err = mod.ExportedFunction("init_passive").Call(testCtx, 0)
+		require.NoError(t, err)
+
+		// Buf size must be zero for such dropped elements.
+		_, err = mod.ExportedFunction("init_passive").Call(testCtx, 1)
+		require.ErrorIs(t, err, wasmruntime.ErrRuntimeInvalidTableAccess)
+	})
+}
+
+func testBulkMemoryOperations(t *testing.T, newRuntimeConfig func() *wazero.RuntimeConfig) {
+	requireErrorOnBulkMemoryFeatureDisabled(t, newRuntimeConfig, bulkMemoryOperationsWasm)
+
 	t.Run("enabled", func(t *testing.T) {
 		r := wazero.NewRuntimeWithConfig(newRuntimeConfig().WithFeatureBulkMemoryOperations(true))
 

internal/integration_test/post1_0/bulk-memory-operations/testdata/elem_drop.wat

@@ -0,0 +1,16 @@
+(module
+  (table 1 funcref)
+  (func $f)
+  (elem $p funcref (ref.func $f))
+  (elem $a (table 0) (i32.const 0) func $f)
+
+  (func (export "drop_passive") (elem.drop $p))
+  (func (export "init_passive") (param $len i32)
+    (table.init $p (i32.const 0) (i32.const 0) (local.get $len))
+  )
+
+  (func (export "drop_active") (elem.drop $a))
+  (func (export "init_active") (param $len i32)
+    (table.init $a (i32.const 0) (i32.const 0) (local.get $len))
+  )
+)

internal/integration_test/post1_0/bulk-memory-operations/testdata/table_copy.wat

@@ -0,0 +1,17 @@
+(module
+  (table 10 funcref)
+  (elem (i32.const 0) $zero $one $two)
+  (func $zero (result i32) (i32.const 0))
+  (func $one (result i32) (i32.const 1))
+  (func $two (result i32) (i32.const 2))
+
+  (func (export "copy") (param i32 i32 i32)
+    (table.copy
+      (local.get 0)
+      (local.get 1)
+      (local.get 2)))
+
+  (func (export "call") (param i32) (result i32)
+    (call_indirect (result i32)
+      (local.get 0)))
+)

internal/integration_test/post1_0/bulk-memory-operations/testdata/table_init.wat

@@ -0,0 +1,18 @@
+(module
+  (table 3 funcref)
+  (elem funcref
+    (ref.func $zero) (ref.func $one) (ref.func $zero) (ref.func $one))
+
+  (func $zero (result i32) (i32.const 0))
+  (func $one (result i32) (i32.const 1))
+
+  (func (export "init") (param i32 i32 i32)
+    (table.init 0
+      (local.get 0)
+      (local.get 1)
+      (local.get 2)))
+
+  (func (export "call") (param i32) (result i32)
+    (call_indirect (result i32)
+      (local.get 0)))
+)

internal/integration_test/spectest/spectest.go

@@ -180,7 +180,7 @@ func (c command) expectedError() (err error) {
 		err = wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess
 	case "indirect call type mismatch", "indirect call":
 		err = wasmruntime.ErrRuntimeIndirectCallTypeMismatch
-	case "undefined element", "undefined":
+	case "undefined element", "undefined", "out of bounds table access":
 		err = wasmruntime.ErrRuntimeInvalidTableAccess
 	case "integer overflow":
 		err = wasmruntime.ErrRuntimeIntegerOverflow

internal/leb128/leb128_test.go

@@ -106,6 +106,7 @@ func TestDecodeUint32(t *testing.T) {
 		{bytes: []byte{0x00}, exp: 0},
 		{bytes: []byte{0x04}, exp: 4},
 		{bytes: []byte{0x01}, exp: 1},
+		{bytes: []byte{0x80, 0}, exp: 0},
 		{bytes: []byte{0x80, 0x7f}, exp: 16256},
 		{bytes: []byte{0xe5, 0x8e, 0x26}, exp: 624485},
 		{bytes: []byte{0x80, 0x80, 0x80, 0x4f}, exp: 165675008},

internal/modgen/modgen.go

@@ -379,7 +379,7 @@ func (g *generator) genElementSection() {
 	min := table.Min
 	for i := uint32(0); i < g.numElements; i++ {
 		// Elements can't exceed min of table.
-		indexes := make([]wasm.NullableIndex, g.nextRandom().Intn(int(min)+1))
+		indexes := make([]*wasm.Index, g.nextRandom().Intn(int(min)+1))
 		for i := range indexes {
 			v := uint32(g.nextRandom().Intn(numFuncs))
 			indexes[i] = &v

internal/modgen/modgen_test.go

@@ -587,7 +587,7 @@ func TestGenerator_startSection(t *testing.T) {
 }
 
 func TestGenerator_elementSection(t *testing.T) {
-	u32AsInitValue := func(in uint32) wasm.NullableIndex {
+	uint32Ptr := func(in uint32) *uint32 {
 		return &in
 	}
 
@@ -616,7 +616,7 @@ func TestGenerator_elementSection(t *testing.T) {
 				exps: []*wasm.ElementSegment{
 					{
 						OffsetExpr: &wasm.ConstantExpression{Opcode: wasm.OpcodeI32Const, Data: leb128.EncodeInt32(98)},
-						Init:       []wasm.NullableIndex{u32AsInitValue(0), u32AsInitValue(50)},
+						Init:       []*wasm.Index{uint32Ptr(0), uint32Ptr(50)},
 					},
 				},
 			},
@@ -630,16 +630,16 @@ func TestGenerator_elementSection(t *testing.T) {
 				exps: []*wasm.ElementSegment{
 					{
 						OffsetExpr: &wasm.ConstantExpression{Opcode: wasm.OpcodeI32Const, Data: leb128.EncodeInt32(0)},
-						Init:       []wasm.NullableIndex{u32AsInitValue(25), u32AsInitValue(75)},
+						Init:       []*wasm.Index{uint32Ptr(25), uint32Ptr(75)},
 					},
 					{
 						OffsetExpr: &wasm.ConstantExpression{Opcode: wasm.OpcodeI32Const, Data: leb128.EncodeInt32(99)},
-						Init:       []wasm.NullableIndex{u32AsInitValue(3)},
+						Init:       []*wasm.Index{uint32Ptr(3)},
 					},
 					{
 						OffsetExpr: &wasm.ConstantExpression{Opcode: wasm.OpcodeI32Const, Data: leb128.EncodeInt32(90)},
-						Init: []wasm.NullableIndex{u32AsInitValue(1), u32AsInitValue(2), u32AsInitValue(3), u32AsInitValue(4),
-							u32AsInitValue(5), u32AsInitValue(6), u32AsInitValue(7), u32AsInitValue(8), u32AsInitValue(9), u32AsInitValue(10)},
+						Init: []*wasm.Index{uint32Ptr(1), uint32Ptr(2), uint32Ptr(3), uint32Ptr(4),
+							uint32Ptr(5), uint32Ptr(6), uint32Ptr(7), uint32Ptr(8), uint32Ptr(9), uint32Ptr(10)},
 					},
 				},
 			},