Merge remote-tracking branch 'origin-vyper/master' into feat/schedule…

…r_optimization

docs/built-in-functions.rst

@@ -1090,3 +1090,6 @@ Utilities
 .. note::
 
     Issuing of the static call is *NOT* mode-dependent (that is, it is not removed from production code), although the compiler will issue a warning whenever ``print`` is used.
+
+.. warning::
+    In Vyper, as of v0.4.0, the order of argument evaluation of builtins is not defined. That means that the compiler may choose to reorder evaluation of arguments. For example, ``extract32(x(), y())`` may yield unexpected results if ``x()`` and ``y()`` both touch the same data. For this reason, it is best to avoid calling functions with side-effects inside of builtins. For more information, see `GHSA-g2xh-c426-v8mf <https://github.com/vyperlang/vyper/security/advisories/GHSA-g2xh-c426-v8mf>`_ and `issue #4019 <https://github.com/vyperlang/vyper/issues/4019>`_.

docs/index.rst

@@ -6,14 +6,17 @@
 Vyper
 #####
 
-Vyper is a contract-oriented, pythonic programming language that targets the `Ethereum Virtual Machine (EVM) <https://ethereum.org/learn/#ethereum-basics>`_.
+Vyper is a contract-oriented, Pythonic programming language that targets the `Ethereum Virtual Machine (EVM) <https://ethereum.org/learn/#ethereum-basics>`_.
+It prioritizes user safety, encourages clear coding practices via language design and efficient execution. In other words, Vyper code is safe, clear and efficient!
 
 Principles and Goals
 ====================
 
 * **Security**: It should be possible and natural to build secure smart-contracts in Vyper.
 * **Language and compiler simplicity**: The language and the compiler implementation should strive to be simple.
-* **Auditability**: Vyper code should be maximally human-readable. Furthermore, it should be maximally difficult to write misleading code. Simplicity for the reader is more important than simplicity for the writer, and simplicity for readers with low prior experience with Vyper (and low prior experience with programming in general) is particularly important.
+* **Auditability**: Vyper code should be maximally human-readable.
+  Furthermore, it should be maximally difficult to write misleading code.
+  Simplicity for the reader is more important than simplicity for the writer, and simplicity for readers with low prior experience with Vyper (and low prior experience with programming in general) is particularly important.
 
 Because of this Vyper provides the following features:
 

tests/functional/builtins/codegen/test_extract32.py

@@ -1,6 +1,7 @@
 import pytest
 
 from vyper.evm.opcodes import version_check
+from vyper.exceptions import CompilerPanic
 
 
 @pytest.mark.parametrize("location", ["storage", "transient"])
@@ -98,3 +99,50 @@ def foq(inp: Bytes[32]) -> address:
 
     with tx_failed():
         c.foq(b"crow" * 8)
+
+
+# to fix in future release
+@pytest.mark.xfail(raises=CompilerPanic, reason="risky overlap")
+def test_extract32_order_of_eval(get_contract):
+    extract32_code = """
+var:DynArray[Bytes[96], 1]
+
+@internal
+def bar() -> uint256:
+    self.var[0] = b'hellohellohellohellohellohellohello'
+    self.var.pop()
+    return 3
+
+@external
+def foo() -> bytes32:
+    self.var = [b'abcdefghijklmnopqrstuvwxyz123456789']
+    return extract32(self.var[0], self.bar(), output_type=bytes32)
+    """
+
+    c = get_contract(extract32_code)
+    assert c.foo() == b"defghijklmnopqrstuvwxyz123456789"
+
+
+# to fix in future release
+@pytest.mark.xfail(raises=CompilerPanic, reason="risky overlap")
+def test_extract32_order_of_eval_extcall(get_contract):
+    slice_code = """
+var:DynArray[Bytes[96], 1]
+
+interface Bar:
+    def bar() -> uint256: payable
+
+@external
+def bar() -> uint256:
+    self.var[0] = b'hellohellohellohellohellohellohello'
+    self.var.pop()
+    return 3
+
+@external
+def foo() -> bytes32:
+    self.var = [b'abcdefghijklmnopqrstuvwxyz123456789']
+    return extract32(self.var[0], extcall Bar(self).bar(), output_type=bytes32)
+    """
+
+    c = get_contract(slice_code)
+    assert c.foo() == b"defghijklmnopqrstuvwxyz123456789"

tests/functional/builtins/codegen/test_slice.py

@@ -5,7 +5,7 @@
 from vyper.compiler import compile_code
 from vyper.compiler.settings import OptimizationLevel, Settings
 from vyper.evm.opcodes import version_check
-from vyper.exceptions import ArgumentException, TypeMismatch
+from vyper.exceptions import ArgumentException, CompilerPanic, TypeMismatch
 
 _fun_bytes32_bounds = [(0, 32), (3, 29), (27, 5), (0, 5), (5, 3), (30, 2)]
 
@@ -562,3 +562,53 @@ def foo(cs: String[64]) -> uint256:
     c = get_contract(code)
     # ensure that counter was incremented only once
     assert c.foo(arg) == 1
+
+
+# to fix in future release
+@pytest.mark.xfail(raises=CompilerPanic, reason="risky overlap")
+def test_slice_order_of_eval(get_contract):
+    slice_code = """
+var:DynArray[Bytes[96], 1]
+
+interface Bar:
+    def bar() -> uint256: payable
+
+@external
+def bar() -> uint256:
+    self.var[0] = b'hellohellohellohellohellohellohello'
+    self.var.pop()
+    return 32
+
+@external
+def foo() -> Bytes[96]:
+    self.var = [b'abcdefghijklmnopqrstuvwxyz123456789']
+    return slice(self.var[0], 3, extcall Bar(self).bar())
+    """
+
+    c = get_contract(slice_code)
+    assert c.foo() == b"defghijklmnopqrstuvwxyz123456789"
+
+
+# to fix in future release
+@pytest.mark.xfail(raises=CompilerPanic, reason="risky overlap")
+def test_slice_order_of_eval2(get_contract):
+    slice_code = """
+var:DynArray[Bytes[96], 1]
+
+interface Bar:
+    def bar() -> uint256: payable
+
+@external
+def bar() -> uint256:
+    self.var[0] = b'hellohellohellohellohellohellohello'
+    self.var.pop()
+    return 3
+
+@external
+def foo() -> Bytes[96]:
+    self.var = [b'abcdefghijklmnopqrstuvwxyz123456789']
+    return slice(self.var[0], extcall Bar(self).bar(), 32)
+    """
+
+    c = get_contract(slice_code)
+    assert c.foo() == b"defghijklmnopqrstuvwxyz123456789"

tests/functional/codegen/types/test_array_indexing.py

@@ -1,5 +1,9 @@
 # TODO: rewrite the tests in type-centric way, parametrize array and indices types
 
+import pytest
+
+from vyper.exceptions import CompilerPanic
+
 
 def test_negative_ix_access(get_contract, tx_failed):
     # Arrays can't be accessed with negative indices
@@ -130,3 +134,76 @@ def foo():
     c.foo()
     for i in range(10):
         assert c.arr(i) == i
+
+
+# to fix in future release
+@pytest.mark.xfail(raises=CompilerPanic, reason="risky overlap")
+def test_array_index_overlap(get_contract):
+    code = """
+a: public(DynArray[DynArray[Bytes[96], 5], 5])
+
+@external
+def foo() -> Bytes[96]:
+    self.a.append([b'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'])
+    return self.a[0][self.bar()]
+
+
+@internal
+def bar() -> uint256:
+    self.a[0] = [b'yyy']
+    self.a.pop()
+    return 0
+    """
+    c = get_contract(code)
+    # tricky to get this right, for now we just panic instead of generating code
+    assert c.foo() == b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+
+# to fix in future release
+@pytest.mark.xfail(raises=CompilerPanic, reason="risky overlap")
+def test_array_index_overlap_extcall(get_contract):
+    code = """
+
+interface Bar:
+    def bar() -> uint256: payable
+
+a: public(DynArray[DynArray[Bytes[96], 5], 5])
+
+@external
+def foo() -> Bytes[96]:
+    self.a.append([b'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'])
+    return self.a[0][extcall Bar(self).bar()]
+
+
+@external
+def bar() -> uint256:
+    self.a[0] = [b'yyy']
+    self.a.pop()
+    return 0
+    """
+    c = get_contract(code)
+    assert c.foo() == b"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+
+# to fix in future release
+@pytest.mark.xfail(raises=CompilerPanic, reason="risky overlap")
+def test_array_index_overlap_extcall2(get_contract):
+    code = """
+interface B:
+    def calculate_index() -> uint256: nonpayable
+
+a: HashMap[uint256, DynArray[uint256, 5]]
+
+@external
+def bar() -> uint256:
+    self.a[0] = [2]
+    return self.a[0][extcall B(self).calculate_index()]
+
+@external
+def calculate_index() -> uint256:
+    self.a[0] = [1]
+    return 0
+    """
+    c = get_contract(code)
+
+    assert c.bar() == 1

tests/functional/codegen/types/test_dynamic_array.py

@@ -8,6 +8,7 @@
 from vyper.exceptions import (
     ArgumentException,
     ArrayIndexException,
+    CompilerPanic,
     ImmutableViolation,
     OverflowException,
     StackTooDeep,
@@ -1887,3 +1888,18 @@ def boo() -> uint256:
 
     c = get_contract(code)
     assert c.foo() == [1, 2, 3, 4]
+
+
+@pytest.mark.xfail(raises=CompilerPanic)
+def test_dangling_reference(get_contract, tx_failed):
+    code = """
+a: DynArray[DynArray[uint256, 5], 5]
+
+@external
+def foo():
+    self.a = [[1]]
+    self.a.pop().append(2)
+    """
+    c = get_contract(code)
+    with tx_failed():
+        c.foo()

vyper/ast/nodes.pyi

@@ -23,6 +23,7 @@ class VyperNode:
     end_col_offset: int = ...
     _metadata: dict = ...
     _original_node: Optional[VyperNode] = ...
+    _children: list[VyperNode] = ...
     def __init__(self, parent: Optional[VyperNode] = ..., **kwargs: Any) -> None: ...
     def __hash__(self) -> Any: ...
     def __eq__(self, other: Any) -> Any: ...

vyper/builtins/functions.py

@@ -29,6 +29,7 @@
     get_type_for_exact_size,
     ir_tuple_from_args,
     make_setter,
+    potential_overlap,
     promote_signed_int,
     sar,
     shl,
@@ -357,6 +358,9 @@ def build_IR(self, expr, args, kwargs, context):
             assert is_bytes32, src
             src = ensure_in_memory(src, context)
 
+        if potential_overlap(src, start) or potential_overlap(src, length):
+            raise CompilerPanic("risky overlap")
+
         with src.cache_when_complex("src") as (b1, src), start.cache_when_complex("start") as (
             b2,
             start,
@@ -862,6 +866,9 @@ def build_IR(self, expr, args, kwargs, context):
         bytez, index = args
         ret_type = kwargs["output_type"]
 
+        if potential_overlap(bytez, index):
+            raise CompilerPanic("risky overlap")
+
         def finalize(ret):
             annotation = "extract32"
             ret = IRnode.from_list(ret, typ=ret_type, annotation=annotation)

vyper/codegen/core.py

@@ -924,6 +924,26 @@ def potential_overlap(left, right):
     return False
 
 
+# similar to `potential_overlap()`, but compares left's _reads_ vs
+# right's _writes_.
+# TODO: `potential_overlap()` can probably be replaced by this function,
+# but all the cases need to be checked.
+def read_write_overlap(left, right):
+    if not isinstance(left, IRnode) or not isinstance(right, IRnode):
+        return False
+
+    if left.typ._is_prim_word and right.typ._is_prim_word:
+        return False
+
+    if len(left.referenced_variables & right.variable_writes) > 0:
+        return True
+
+    if len(left.referenced_variables) > 0 and right.contains_risky_call:
+        return True
+
+    return False
+
+
 # Create an x=y statement, where the types may be compound
 def make_setter(left, right, hi=None):
     check_assign(left, right)

vyper/codegen/expr.py

@@ -21,6 +21,7 @@
     make_setter,
     pop_dyn_array,
     potential_overlap,
+    read_write_overlap,
     sar,
     shl,
     shr,
@@ -40,6 +41,7 @@
     UnimplementedException,
     tag_exceptions,
 )
+from vyper.semantics.analysis.utils import get_expr_writes
 from vyper.semantics.types import (
     AddressT,
     BoolT,
@@ -86,6 +88,9 @@ def __init__(self, node, context, is_stmt=False):
             self.ir_node = fn()
             assert isinstance(self.ir_node, IRnode), self.ir_node
 
+        writes = set(access.variable for access in get_expr_writes(self.expr))
+        self.ir_node._writes = writes
+
         self.ir_node.annotation = self.expr.get("node_source_code")
         self.ir_node.ast_source = self.expr
 
@@ -352,6 +357,8 @@ def parse_Subscript(self):
 
         elif is_array_like(sub.typ):
             index = Expr.parse_value_expr(self.expr.slice, self.context)
+            if read_write_overlap(sub, index):
+                raise CompilerPanic("risky overlap")
 
         elif is_tuple_like(sub.typ):
             # should we annotate expr.slice in the frontend with the

vyper/codegen/ir_node.py

@@ -405,7 +405,8 @@ def unique_symbols(self):
         for arg in children:
             s = arg.unique_symbols
             non_uniques = ret.intersection(s)
-            assert len(non_uniques) == 0, f"non-unique symbols {non_uniques}"
+            if len(non_uniques) != 0:  # pragma: nocover
+                raise CompilerPanic(f"non-unique symbols {non_uniques}")
             ret |= s
         return ret
 
@@ -466,6 +467,18 @@ def referenced_variables(self):
 
         return ret
 
+    @cached_property
+    def variable_writes(self):
+        ret = getattr(self, "_writes", set())
+
+        for arg in self.args:
+            ret |= arg.variable_writes
+
+        if getattr(self, "is_self_call", False):
+            ret |= self.invoked_function_ir.func_ir.variable_writes
+
+        return ret
+
     @cached_property
     def contains_risky_call(self):
         ret = self.value in ("call", "delegatecall", "staticcall", "create", "create2")