Merge pull request #358 from Chia-Network/limit-pairs-atoms

make the upper limit of allocated atoms and pairs hard coded

benches/run-program.rs

@@ -209,11 +209,11 @@ fn run_program_benchmark(c: &mut Criterion) {
         ("hash-string", long_strings),
         ("hash-tree", large_tree::<16>),
         ("large-block", large_block),
-        ("loop_add", single_value::<4000000>),
-        ("loop_ior", single_value::<4000000>),
-        ("loop_not", single_value::<4000000>),
-        ("loop_sub", single_value::<4000000>),
-        ("loop_xor", single_value::<4000000>),
+        ("loop_add", single_value::<3675000>),
+        ("loop_ior", single_value::<3675000>),
+        ("loop_not", single_value::<3675000>),
+        ("loop_sub", single_value::<3675000>),
+        ("loop_xor", single_value::<3675000>),
         ("matrix-multiply", matrix::<50, 50>),
         ("point-pow", point_pow),
         ("pubkey-tree", large_tree::<10>),

src/allocator.rs

@@ -56,14 +56,11 @@ pub struct Allocator {
 
     // the atom_vec may not grow past this
     heap_limit: usize,
-
-    // the pair_vec may not grow past this
-    pair_limit: usize,
-
-    // the atom_vec may not grow past this
-    atom_limit: usize,
 }
 
+const MAX_NUM_ATOMS: usize = 62500000;
+const MAX_NUM_PAIRS: usize = 62500000;
+
 impl Default for Allocator {
     fn default() -> Self {
         Self::new()
@@ -72,29 +69,18 @@ impl Default for Allocator {
 
 impl Allocator {
     pub fn new() -> Self {
-        Self::new_limited(
-            u32::MAX as usize,
-            i32::MAX as usize,
-            (i32::MAX - 1) as usize,
-        )
+        Self::new_limited(u32::MAX as usize)
     }
 
-    pub fn new_limited(heap_limit: usize, pair_limit: usize, atom_limit: usize) -> Self {
+    pub fn new_limited(heap_limit: usize) -> Self {
         // we have a maximum of 4 GiB heap, because pointers are 32 bit unsigned
         assert!(heap_limit <= u32::MAX as usize);
-        // the atoms and pairs share a single 32 bit address space, where
-        // negative numbers are atoms and positive numbers are pairs. That's why
-        // we have one more slot for pairs than atoms
-        assert!(pair_limit <= i32::MAX as usize);
-        assert!(atom_limit < i32::MAX as usize);
 
         let mut r = Self {
             u8_vec: Vec::new(),
             pair_vec: Vec::new(),
             atom_vec: Vec::new(),
             heap_limit,
-            pair_limit,
-            atom_limit,
         };
         r.u8_vec.reserve(1024 * 1024);
         r.atom_vec.reserve(256);
@@ -136,7 +122,7 @@ impl Allocator {
         if (self.heap_limit - start as usize) < v.len() {
             return err(self.null(), "out of memory");
         }
-        if self.atom_vec.len() == self.atom_limit {
+        if self.atom_vec.len() == MAX_NUM_ATOMS {
             return err(self.null(), "too many atoms");
         }
         self.u8_vec.extend_from_slice(v);
@@ -159,7 +145,7 @@ impl Allocator {
 
     pub fn new_pair(&mut self, first: NodePtr, rest: NodePtr) -> Result<NodePtr, EvalErr> {
         let r = self.pair_vec.len() as i32;
-        if self.pair_vec.len() == self.pair_limit {
+        if self.pair_vec.len() == MAX_NUM_PAIRS {
             return err(self.null(), "too many pairs");
         }
         self.pair_vec.push(IntPair { first, rest });
@@ -170,7 +156,7 @@ impl Allocator {
         if node.0 >= 0 {
             return err(node, "(internal error) substr expected atom, got pair");
         }
-        if self.atom_vec.len() == self.atom_limit {
+        if self.atom_vec.len() == MAX_NUM_ATOMS {
             return err(self.null(), "too many atoms");
         }
         let atom = self.atom_vec[(-node.0 - 1) as usize];
@@ -192,7 +178,7 @@ impl Allocator {
     }
 
     pub fn new_concat(&mut self, new_size: usize, nodes: &[NodePtr]) -> Result<NodePtr, EvalErr> {
-        if self.atom_vec.len() == self.atom_limit {
+        if self.atom_vec.len() == MAX_NUM_ATOMS {
             return err(self.null(), "too many atoms");
         }
         let start = self.u8_vec.len();
@@ -471,7 +457,7 @@ fn test_allocate_pair() {
 
 #[test]
 fn test_allocate_heap_limit() {
-    let mut a = Allocator::new_limited(6, i32::MAX as usize, (i32::MAX - 1) as usize);
+    let mut a = Allocator::new_limited(6);
     // we can't allocate 6 bytes
     assert_eq!(a.new_atom(b"foobar").unwrap_err().1, "out of memory");
     // but 5 is OK
@@ -480,7 +466,7 @@ fn test_allocate_heap_limit() {
 
 #[test]
 fn test_allocate_atom_limit() {
-    let mut a = Allocator::new_limited(u32::MAX as usize, i32::MAX as usize, 5);
+    let mut a = Allocator::new();
     // we can allocate 5 atoms total
     // keep in mind that we always have 2 pre-allocated atoms for null and one,
     // so with a limit of 5, we only have 3 slots left at this point.
@@ -490,17 +476,28 @@ fn test_allocate_atom_limit() {
 
     // the 4th fails and ensure not to append atom to the stack
     assert_eq!(a.u8_vec.len(), 10);
+    for _ in 3..MAX_NUM_ATOMS - 2 {
+        // exhaust the numebr of atoms allowed to be allocated
+        let _ = a.new_atom(b"foo").unwrap();
+    }
     assert_eq!(a.new_atom(b"foobar").unwrap_err().1, "too many atoms");
-    assert_eq!(a.u8_vec.len(), 10);
+
+    // the pre-allocated null() and one() also count against the limit, and they
+    // use 0 and 1 bytes respectively
+    assert_eq!(a.u8_vec.len(), MAX_NUM_ATOMS * 3 - 3 - 2);
 }
 
 #[test]
 fn test_allocate_pair_limit() {
-    let mut a = Allocator::new_limited(u32::MAX as usize, 1, (i32::MAX - 1) as usize);
+    let mut a = Allocator::new();
     let atom = a.new_atom(b"foo").unwrap();
     // one pair is OK
     let _pair1 = a.new_pair(atom, atom).unwrap();
-    // but not 2
+    for _ in 1..MAX_NUM_PAIRS {
+        // exhaust the numebr of pairs allowed to be allocated
+        let _ = a.new_pair(atom, atom).unwrap();
+    }
+
     assert_eq!(a.new_pair(atom, atom).unwrap_err().1, "too many pairs");
 }
 
@@ -601,7 +598,7 @@ fn test_sexp() {
 
 #[test]
 fn test_concat_limit() {
-    let mut a = Allocator::new_limited(9, i32::MAX as usize, (i32::MAX - 1) as usize);
+    let mut a = Allocator::new_limited(9);
     let atom1 = a.new_atom(b"f").unwrap();
     let atom2 = a.new_atom(b"o").unwrap();
     let atom3 = a.new_atom(b"o").unwrap();

tests/run-programs.py

@@ -7,13 +7,47 @@
 import platform
 from colorama import init, Fore, Style
 from run import run_clvm
-from os.path import isfile
+from os.path import isfile, splitext, basename
 
 init()
 ret = 0
 
-for fn in glob.glob('programs/large-atom-*.hex.invalid'):
-
+expect = {
+    "args-add": "cost exceeded",
+    "args-all": "cost exceeded",
+    "args-and": "cost exceeded",
+    "args-any": "cost exceeded",
+    "args-cat": "cost exceeded",
+    "args-mul": "cost exceeded",
+    "args-or": "cost exceeded",
+    "args-point_add": "cost exceeded",
+    "args-sha": "cost exceeded",
+    "args-sub": "cost exceeded",
+    "args-unknown-1": "cost exceeded",
+    "args-unknown-2": "cost exceeded",
+    "args-unknown-3": "cost exceeded",
+    "args-unknown-4": "cost exceeded",
+    "args-unknown-5": "too many pairs",
+    "args-unknown-6": "too many pairs",
+    "args-unknown-7": "too many pairs",
+    "args-unknown-8": "too many pairs",
+    "args-unknown-9": "too many pairs",
+    "args-xor": "cost exceeded",
+    "recursive-add": "cost exceeded",
+    "recursive-ash": "cost exceeded",
+    "recursive-cat": "cost exceeded",
+    "recursive-cons": "too many pairs",
+    "recursive-div": "cost exceeded",
+    "recursive-lsh": "cost exceeded",
+    "recursive-mul": "cost exceeded",
+    "recursive-not": "cost exceeded",
+    "recursive-pubkey": "cost exceeded",
+    "recursive-sub": "too many pairs",
+    "softfork-1": "cost exceeded",
+    "softfork-2": "cost exceeded",
+}
+
+for fn in glob.glob("programs/large-atom-*.hex.invalid"):
     try:
         print(fn)
         run_clvm(fn)
@@ -23,80 +57,92 @@
         print(Fore.GREEN + f"OK: expected: {e}" + Style.RESET_ALL)
 
 
-for fn in glob.glob('programs/*.clvm'):
-
-    hexname = fn[:-4] + 'hex'
+for fn in glob.glob("programs/*.clvm"):
+    hexname = fn[:-4] + "hex"
     if isfile(hexname):
         continue
-    with open(hexname, 'w+') as out:
+    with open(hexname, "w+") as out:
         print(f"compiling {fn}")
-        proc = subprocess.Popen(['opc', fn], stdout=out)
+        proc = subprocess.Popen(["opc", fn], stdout=out)
         proc.wait()
 
-for fn in glob.glob('programs/*.env'):
-    hexenv = fn + 'hex'
+for fn in glob.glob("programs/*.env"):
+    hexenv = fn + "hex"
     if isfile(hexenv):
         continue
-    with open(hexenv, 'w+') as out:
+    with open(hexenv, "w+") as out:
         print(f"compiling {fn}")
-        proc = subprocess.Popen(['opc', fn], stdout=out)
+        proc = subprocess.Popen(["opc", fn], stdout=out)
         proc.wait()
 
-for hexname in sorted(glob.glob('programs/*.hex')):
-
-    hexenv = hexname[:-3] + 'envhex'
+for hexname in sorted(glob.glob("programs/*.hex")):
+    hexenv = hexname[:-3] + "envhex"
 
-    command = ['./run.py', hexname, hexenv]
+    command = ["./run.py", hexname, hexenv]
 
     # prepend the size command, to measure RSS
-    if platform.system() == 'Darwin':
-        command = ['/usr/bin/time', '-l'] + command;
-    if platform.system() == 'Linux':
-        command = ['/usr/bin/time'] + command;
+    if platform.system() == "Darwin":
+        command = ["/usr/bin/time", "-l"] + command
+    if platform.system() == "Linux":
+        command = ["/usr/bin/time"] + command
 
-    print(' '.join(command))
+    print(" ".join(command))
     start = time.perf_counter()
     proc = subprocess.run(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
-    output = proc.stderr.decode('UTF-8')
-    output += proc.stdout.decode('UTF-8')
+    output = proc.stderr.decode("UTF-8")
+    output += proc.stdout.decode("UTF-8")
     end = time.perf_counter()
 
-    if 'FAIL: cost exceeded' not in output:
+    test = splitext(basename(hexname))[0]
+    expected_error = expect[test]
+    if f"FAIL: {expected_error}" not in output:
         ret += 1
-        print(Fore.RED + '\nTEST FAILURE: expected cost to be exceeded\n' + Style.RESET_ALL)
+        print(
+            Fore.RED
+            + f'\nTEST FAILURE: expected "{expected_error}"\n'
+            + Style.RESET_ALL
+        )
         print(output)
 
-    print(Fore.YELLOW + ('  Runtime: %0.2f s' % (end - start)) + Style.RESET_ALL)
+    print(Fore.YELLOW + ("  Runtime: %0.2f s" % (end - start)) + Style.RESET_ALL)
 
     # parse RSS (MacOS and Linux only)
     size = None
-    if platform.system() == 'Darwin':
-        for l in output.split('\n'):
-            if 'maximum resident set size' not in l:
+    if platform.system() == "Darwin":
+        for l in output.split("\n"):
+            if "maximum resident set size" not in l:
                 continue
-            val, key = l.strip().split('  ', 1)
-            if key == 'maximum resident set size':
+            val, key = l.strip().split("  ", 1)
+            if key == "maximum resident set size":
                 size = int(val) / 1024 / 1024
-    if platform.system() == 'Linux':
+    if platform.system() == "Linux":
         # example output:
         # 10.49user 0.32system 0:10.84elapsed 99%CPU (0avgtext+0avgdata 189920maxresident)k
-        for l in output.split('\n'):
-            if 'maxresident)k' not in l:
+        for l in output.split("\n"):
+            if "maxresident)k" not in l:
                 continue
-            size = int(l.split('maxresident)k')[0].split(' ')[-1]) / 1024
+            size = int(l.split("maxresident)k")[0].split(" ")[-1]) / 1024
     if size != None:
-        print(Fore.YELLOW + ('  Resident Size: %d MiB' % size) + Style.RESET_ALL)
+        print(Fore.YELLOW + ("  Resident Size: %d MiB" % size) + Style.RESET_ALL)
 
         if size > 2300:
             ret += 1
-            print(Fore.RED + '\nTEST FAILURE: Max memory use exceeded (limit: 2300 MB)\n' + Style.RESET_ALL)
+            print(
+                Fore.RED
+                + "\nTEST FAILURE: Max memory use exceeded (limit: 2300 MB)\n"
+                + Style.RESET_ALL
+            )
 
     # cost 10923314721 roughly corresponds to 11 seconds
     if end - start > 11:
         ret += 1
-        print(Fore.RED + '\nTEST FAILURE: Time exceeded: %f (limit: 11)\n' % (end - start) + Style.RESET_ALL)
+        print(
+            Fore.RED
+            + "\nTEST FAILURE: Time exceeded: %f (limit: 11)\n" % (end - start)
+            + Style.RESET_ALL
+        )
 
 if ret:
-    print(Fore.RED + f'\n   There were {ret} failures!\n' + Style.RESET_ALL)
+    print(Fore.RED + f"\n   There were {ret} failures!\n" + Style.RESET_ALL)
 
 sys.exit(ret)

wheel/src/api.rs

@@ -27,7 +27,7 @@ pub fn run_serialized_chia_program(
     flags: u32,
 ) -> PyResult<(u64, LazyNode)> {
     let mut allocator = if flags & LIMIT_HEAP != 0 {
-        Allocator::new_limited(500000000, 62500000, 62500000)
+        Allocator::new_limited(500000000)
     } else {
         Allocator::new()
     };