From 1afc32478c2142ff772ec8cd1733f872c0ac5e4f Mon Sep 17 00:00:00 2001 From: Ryan Macnak Date: Tue, 21 Mar 2023 23:06:27 +0000 Subject: [PATCH] [stable][vm, gc] Add missing promotion of Finalizer external size. Remove race incrementing external size. When a FinalizerEntry's target gets promoted, the associated external size needs to also get promoted. We were handling the cases where the FinalizerEntry itself was either already old or remained new, but not the case where it was promoted. Failing to promote the external size meant that when the finalizer was collected, external size was subtraced from old space that was still being attributed to new space, so the old space total external size became negative. TEST=ci Bug: https://github.com/dart-lang/sdk/issues/50537 Change-Id: I83316636dea13415f38343212157c32f5ef19857 Cherry-pick: https://dart-review.googlesource.com/c/sdk/+/272350 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/289941 Reviewed-by: Alexander Markov --- .../tests/vm/dart/splay_c_finalizer_test.dart | 85 +++++ runtime/tests/vm/dart/splay_common.dart | 290 +++++++++++++++++ .../vm/dart/splay_dart_finalizer_test.dart | 92 ++++++ .../tests/vm/dart/splay_ephemeron_test.dart | 296 +----------------- runtime/tests/vm/dart/splay_test.dart | 296 +----------------- runtime/tests/vm/dart/splay_weak_test.dart | 103 ++++++ .../split_aot_kernel_generation_test.dart | 2 +- .../vm/dart_2/splay_c_finalizer_test.dart | 89 ++++++ runtime/tests/vm/dart_2/splay_common.dart | 295 +++++++++++++++++ .../vm/dart_2/splay_dart_finalizer_test.dart | 96 ++++++ .../tests/vm/dart_2/splay_ephemeron_test.dart | 291 +---------------- runtime/tests/vm/dart_2/splay_test.dart | 291 +---------------- runtime/tests/vm/dart_2/splay_weak_test.dart | 107 +++++++ .../split_aot_kernel_generation_test.dart | 2 +- runtime/vm/heap/pages.h | 17 +- runtime/vm/heap/scavenger.cc | 97 +++--- runtime/vm/heap/scavenger.h | 20 +- ...nalizer_external_size_accounting_test.dart | 48 +++ ...nalizer_external_size_accounting_test.dart | 50 +++ 19 files changed, 1370 insertions(+), 1197 deletions(-) create mode 100644 runtime/tests/vm/dart/splay_c_finalizer_test.dart create mode 100644 runtime/tests/vm/dart/splay_common.dart create mode 100644 runtime/tests/vm/dart/splay_dart_finalizer_test.dart create mode 100644 runtime/tests/vm/dart/splay_weak_test.dart create mode 100644 runtime/tests/vm/dart_2/splay_c_finalizer_test.dart create mode 100644 runtime/tests/vm/dart_2/splay_common.dart create mode 100644 runtime/tests/vm/dart_2/splay_dart_finalizer_test.dart create mode 100644 runtime/tests/vm/dart_2/splay_weak_test.dart create mode 100644 tests/ffi/finalizer_external_size_accounting_test.dart create mode 100644 tests/ffi_2/finalizer_external_size_accounting_test.dart diff --git a/runtime/tests/vm/dart/splay_c_finalizer_test.dart b/runtime/tests/vm/dart/splay_c_finalizer_test.dart new file mode 100644 index 000000000000..2654dd53657d --- /dev/null +++ b/runtime/tests/vm/dart/splay_c_finalizer_test.dart @@ -0,0 +1,85 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it +// continuously makes small changes to a large, long-lived data structure, +// stressing lots of combinations of references between new-gen and old-gen +// objects, and between marked and unmarked objects. + +// VMOptions= +// VMOptions=--no_concurrent_mark --no_concurrent_sweep +// VMOptions=--no_concurrent_mark --concurrent_sweep +// VMOptions=--no_concurrent_mark --use_compactor +// VMOptions=--no_concurrent_mark --use_compactor --force_evacuation +// VMOptions=--concurrent_mark --no_concurrent_sweep +// VMOptions=--concurrent_mark --concurrent_sweep +// VMOptions=--concurrent_mark --use_compactor +// VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 +// VMOptions=--verify_before_gc +// VMOptions=--verify_after_gc +// VMOptions=--verify_before_gc --verify_after_gc +// VMOptions=--verify_store_buffer +// VMOptions=--verify_after_marking +// VMOptions=--stress_write_barrier_elimination +// VMOptions=--old_gen_heap_size=150 + +import "dart:ffi"; +import "dart:io"; + +import "splay_common.dart"; + +void main() { + if (Platform.isWindows) { + print("No malloc via self process lookup on Windows"); + return; + } + + // Split across turns so finalizers can run. + FinalizerSplay().mainAsync(); +} + +class FinalizerSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object? value) => new FinalizerNode(key, value); +} + +final libc = DynamicLibrary.process(); +typedef MallocForeign = Pointer Function(IntPtr size); +typedef MallocNative = Pointer Function(int size); +final malloc = libc.lookupFunction('malloc'); +typedef FreeForeign = Void Function(Pointer); +final free = libc.lookup>('free'); +final freeFinalizer = NativeFinalizer(free); + +class Leaf implements Finalizable { + final Pointer memory; + + Leaf(String tag) : memory = malloc(15) { + if (memory == nullptr) { + throw OutOfMemoryError(); + } + freeFinalizer.attach(this, memory, detach: this, externalSize: 15); + } +} + +class Payload { + Payload(this.left, this.right); + var left, right; + + static generate(depth, tag) { + if (depth == 0) return new Leaf(tag); + return new Payload(generate(depth - 1, tag), generate(depth - 1, tag)); + } +} + +class FinalizerNode extends Node { + FinalizerNode(num key, Object? value) : super(key, value); + + Node? left, right; +} diff --git a/runtime/tests/vm/dart/splay_common.dart b/runtime/tests/vm/dart/splay_common.dart new file mode 100644 index 000000000000..937eb87d9041 --- /dev/null +++ b/runtime/tests/vm/dart/splay_common.dart @@ -0,0 +1,290 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:async"; +import "dart:math"; + +abstract class Node { + Node(this.key, this.value); + final num key; + final Object? value; + + Node? get left; + set left(Node? value); + Node? get right; + set right(Node? value); + + /** + * Performs an ordered traversal of the subtree starting here. + */ + void traverse(void f(Node n)) { + Node? current = this; + while (current != null) { + Node? left = current.left; + if (left != null) left.traverse(f); + f(current); + current = current.right; + } + } +} + +class Leaf { + Leaf(String tag) + : string = "String for key $tag in leaf node", + array = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] {} + String string; + List array; +} + +abstract class Splay { + newPayload(int depth, String tag); + Node newNode(num key, Object? value); + + // Configuration. + static final int kTreeSize = 8000; + static final int kTreeModifications = 80; + static final int kTreePayloadDepth = 5; + + Random rnd = new Random(12345); + + // Insert new node with a unique key. + num insertNewNode() { + num key; + do { + key = rnd.nextDouble(); + } while (find(key) != null); + insert(key, newPayload(kTreePayloadDepth, key.toString())); + return key; + } + + void setup() { + for (int i = 0; i < kTreeSize; i++) insertNewNode(); + } + + void tearDown() { + // Allow the garbage collector to reclaim the memory + // used by the splay tree no matter how we exit the + // tear down function. + List keys = exportKeys(); + // tree = null; + + // Verify that the splay tree has the right size. + int length = keys.length; + if (length != kTreeSize) throw new Error("Splay tree has wrong size"); + + // Verify that the splay tree has sorted, unique keys. + for (int i = 0; i < length - 1; i++) { + if (keys[i] >= keys[i + 1]) throw new Error("Splay tree not sorted"); + } + } + + void exercise() { + // Replace a few nodes in the splay tree. + for (int i = 0; i < kTreeModifications; i++) { + num key = insertNewNode(); + Node? greatest = findGreatestLessThan(key); + if (greatest == null) + remove(key); + else + remove(greatest.key); + } + } + + void main() { + setup(); + final sw = Stopwatch()..start(); + while (sw.elapsedMilliseconds < 2000) { + exercise(); + } + tearDown(); + } + + void mainAsync() { + setup(); + final sw = Stopwatch()..start(); + step() { + if (sw.elapsedMilliseconds < 2000) { + exercise(); + Timer.run(step); + } else { + tearDown(); + } + } + + Timer.run(step); + } + + /** + * A splay tree is a self-balancing binary search tree with the additional + * property that recently accessed elements are quick to access again. + * It performs basic operations such as insertion, look-up and removal + * in O(log(n)) amortized time. + */ + + /** + * Inserts a node into the tree with the specified [key] and value if + * the tree does not already contain a node with the specified key. If + * the value is inserted, it becomes the root of the tree. + */ + void insert(num key, value) { + if (isEmpty) { + root = newNode(key, value); + return; + } + // Splay on the key to move the last node on the search path for + // the key to the root of the tree. + splay(key); + if (root!.key == key) return; + Node node = newNode(key, value); + if (key > root!.key) { + node.left = root; + node.right = root!.right; + root!.right = null; + } else { + node.right = root; + node.left = root!.left; + root!.left = null; + } + root = node; + } + + /** + * Removes a node with the specified key from the tree if the tree + * contains a node with this key. The removed node is returned. If + * [key] is not found, an exception is thrown. + */ + Node remove(num key) { + if (isEmpty) throw new Error('Key not found: $key'); + splay(key); + if (root!.key != key) throw new Error('Key not found: $key'); + Node removed = root!; + if (root!.left == null) { + root = root!.right; + } else { + Node? right = root!.right; + root = root!.left; + // Splay to make sure that the new root has an empty right child. + splay(key); + // Insert the original right child as the right child of the new + // root. + root!.right = right; + } + return removed; + } + + /** + * Returns the node having the specified [key] or null if the tree doesn't + * contain a node with the specified [key]. + */ + Node? find(num key) { + if (isEmpty) return null; + splay(key); + return root!.key == key ? root : null; + } + + /** + * Returns the Node having the maximum key value. + */ + Node? findMax([Node? start]) { + if (isEmpty) return null; + Node current = null == start ? root! : start; + while (current.right != null) current = current.right!; + return current; + } + + /** + * Returns the Node having the maximum key value that + * is less than the specified [key]. + */ + Node? findGreatestLessThan(num key) { + if (isEmpty) return null; + // Splay on the key to move the node with the given key or the last + // node on the search path to the top of the tree. + splay(key); + // Now the result is either the root node or the greatest node in + // the left subtree. + if (root!.key < key) return root; + if (root!.left != null) return findMax(root!.left); + return null; + } + + /** + * Perform the splay operation for the given key. Moves the node with + * the given key to the top of the tree. If no node has the given + * key, the last node on the search path is moved to the top of the + * tree. This is the simplified top-down splaying algorithm from: + * "Self-adjusting Binary Search Trees" by Sleator and Tarjan + */ + void splay(num key) { + if (isEmpty) return; + // Create a dummy node. The use of the dummy node is a bit + // counter-intuitive: The right child of the dummy node will hold + // the L tree of the algorithm. The left child of the dummy node + // will hold the R tree of the algorithm. Using a dummy node, left + // and right will always be nodes and we avoid special cases. + final Node dummy = newNode(0, null); + Node left = dummy; + Node right = dummy; + Node current = root!; + while (true) { + if (key < current.key) { + if (current.left == null) break; + if (key < current.left!.key) { + // Rotate right. + Node tmp = current.left!; + current.left = tmp.right; + tmp.right = current; + current = tmp; + if (current.left == null) break; + } + // Link right. + right.left = current; + right = current; + current = current.left!; + } else if (key > current.key) { + if (current.right == null) break; + if (key > current.right!.key) { + // Rotate left. + Node tmp = current.right!; + current.right = tmp.left; + tmp.left = current; + current = tmp; + if (current.right == null) break; + } + // Link left. + left.right = current; + left = current; + current = current.right!; + } else { + break; + } + } + // Assemble. + left.right = current.left; + right.left = current.right; + current.left = dummy.right; + current.right = dummy.left; + root = current; + } + + /** + * Returns a list with all the keys of the tree. + */ + List exportKeys() { + List result = []; + if (!isEmpty) root!.traverse((Node node) => result.add(node.key)); + return result; + } + + // Tells whether the tree is empty. + bool get isEmpty => null == root; + + // Pointer to the root node of the tree. + Node? root; +} + +class Error implements Exception { + const Error(this.message); + final String message; +} diff --git a/runtime/tests/vm/dart/splay_dart_finalizer_test.dart b/runtime/tests/vm/dart/splay_dart_finalizer_test.dart new file mode 100644 index 000000000000..3e7929882996 --- /dev/null +++ b/runtime/tests/vm/dart/splay_dart_finalizer_test.dart @@ -0,0 +1,92 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it +// continuously makes small changes to a large, long-lived data structure, +// stressing lots of combinations of references between new-gen and old-gen +// objects, and between marked and unmarked objects. + +// VMOptions= +// VMOptions=--no_concurrent_mark --no_concurrent_sweep +// VMOptions=--no_concurrent_mark --concurrent_sweep +// VMOptions=--no_concurrent_mark --use_compactor +// VMOptions=--no_concurrent_mark --use_compactor --force_evacuation +// VMOptions=--concurrent_mark --no_concurrent_sweep +// VMOptions=--concurrent_mark --concurrent_sweep +// VMOptions=--concurrent_mark --use_compactor +// VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 +// VMOptions=--verify_before_gc +// VMOptions=--verify_after_gc +// VMOptions=--verify_before_gc --verify_after_gc +// VMOptions=--verify_store_buffer +// VMOptions=--verify_after_marking +// VMOptions=--stress_write_barrier_elimination +// VMOptions=--old_gen_heap_size=300 + +import "splay_common.dart"; + +void main() { + // Split across turns so finalizers can run. + FinalizerSplay().mainAsync(); +} + +class FinalizerSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object? value) => new FinalizerNode(key, value); +} + +final payloadLeft = {}; +final payloadRight = {}; +finalizePayload(Object token) { + payloadLeft.remove(token); + payloadRight.remove(token); +} + +final payloadFinalizer = new Finalizer(finalizePayload); + +class Payload { + Payload(left, right) { + this.left = left; + this.right = right; + payloadFinalizer.attach(this, token, detach: this); + } + var token = new Object(); + get left => payloadLeft[token]; + set left(value) => payloadLeft[token] = value; + get right => payloadRight[token]; + set right(value) => payloadRight[token] = value; + + static generate(depth, tag) { + if (depth == 0) return new Leaf(tag); + return new Payload(generate(depth - 1, tag), generate(depth - 1, tag)); + } +} + +final nodeLeft = {}; +final nodeRight = {}; +finalizeNode(Object token) { + nodeLeft.remove(token); + nodeRight.remove(token); +} + +final nodeFinalizer = new Finalizer(finalizeNode); + +class FinalizerNode extends Node { + FinalizerNode(num key, Object? value) : super(key, value) { + this.left = null; + this.right = null; + nodeFinalizer.attach(this, token, detach: this); + } + + var token = new Object(); + Node? get left => nodeLeft[token]; + set left(Node? value) => nodeLeft[token] = value; + Node? get right => nodeRight[token]; + set right(Node? value) => nodeRight[token] = value; +} diff --git a/runtime/tests/vm/dart/splay_ephemeron_test.dart b/runtime/tests/vm/dart/splay_ephemeron_test.dart index d7d3f524161a..d7d2f1d5103d 100644 --- a/runtime/tests/vm/dart/splay_ephemeron_test.dart +++ b/runtime/tests/vm/dart/splay_ephemeron_test.dart @@ -2,8 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -// This test is a copy of the Splay benchmark that is run with a variety of -// different GC options. It makes for a good GC stress test because it +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it // continuously makes small changes to a large, long-lived data structure, // stressing lots of combinations of references between new-gen and old-gen // objects, and between marked and unmarked objects. @@ -31,102 +31,17 @@ // VMOptions=--stress_write_barrier_elimination // VMOptions=--old_gen_heap_size=150 -import "dart:math"; -import 'package:benchmark_harness/benchmark_harness.dart'; +import "splay_common.dart"; void main() { - Splay.main(); + EphemeronSplay().main(); } -class Splay extends BenchmarkBase { - const Splay() : super("Splay"); - - // Configuration. - static final int kTreeSize = 8000; - static final int kTreeModifications = 80; - static final int kTreePayloadDepth = 5; - - static SplayTree? tree; - - static Random rnd = new Random(12345); - - // Insert new node with a unique key. - static num insertNewNode() { - num key; - final localTree = tree!; - do { - key = rnd.nextDouble(); - } while (localTree.find(key) != null); - Payload payload = Payload.generate(kTreePayloadDepth, key.toString()); - localTree.insert(key, payload); - return key; - } - - static void mysetup() { - tree = new SplayTree(); - for (int i = 0; i < kTreeSize; i++) insertNewNode(); - } - - static void tearDown() { - // Allow the garbage collector to reclaim the memory - // used by the splay tree no matter how we exit the - // tear down function. - List keys = tree!.exportKeys(); - tree = null; - - // Verify that the splay tree has the right size. - int length = keys.length; - if (length != kTreeSize) throw new Error("Splay tree has wrong size"); - - // Verify that the splay tree has sorted, unique keys. - for (int i = 0; i < length - 1; i++) { - if (keys[i] >= keys[i + 1]) throw new Error("Splay tree not sorted"); - } - } - - void warmup() { - exercise(); - } - - void exercise() { - // Replace a few nodes in the splay tree. - final localTree = tree!; - for (int i = 0; i < kTreeModifications; i++) { - num key = insertNewNode(); - Node? greatest = localTree.findGreatestLessThan(key); - if (greatest == null) { - localTree.remove(key); - } else { - localTree.remove(greatest.key); - } - } - } - - static void main() { - mysetup(); - // Don't use benchmark_harness - exercise runtime is not stable (so - // benchmark_harness measuring approach is meaningless for such benchmark - // anyway). - final sw = Stopwatch()..start(); - final benchmark = new Splay(); - while (sw.elapsedMilliseconds < 2000) { - benchmark.exercise(); - } - tearDown(); - } +class EphemeronSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object? value) => new EphemeronNode(key, value); } - -class Leaf { - Leaf(String tag) : - string = "String for key $tag in leaf node", - array = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] - {} - String string; - List array; -} - - class Payload { Payload(left, right) { this.left = left; @@ -151,188 +66,8 @@ class Payload { } } -class Error implements Exception { - const Error(this.message); - final String message; -} - - -/** - * A splay tree is a self-balancing binary search tree with the additional - * property that recently accessed elements are quick to access again. - * It performs basic operations such as insertion, look-up and removal - * in O(log(n)) amortized time. - */ -class SplayTree { - SplayTree(); - - /** - * Inserts a node into the tree with the specified [key] and value if - * the tree does not already contain a node with the specified key. If - * the value is inserted, it becomes the root of the tree. - */ - void insert(num key, value) { - if (isEmpty) { - root = new Node(key, value); - return; - } - // Splay on the key to move the last node on the search path for - // the key to the root of the tree. - splay(key); - if (root!.key == key) return; - Node node = new Node(key, value); - if (key > root!.key) { - node.left = root; - node.right = root!.right; - root!.right = null; - } else { - node.right = root; - node.left = root!.left; - root!.left = null; - } - root = node; - } - - /** - * Removes a node with the specified key from the tree if the tree - * contains a node with this key. The removed node is returned. If - * [key] is not found, an exception is thrown. - */ - Node remove(num key) { - if (isEmpty) throw new Error('Key not found: $key'); - splay(key); - if (root!.key != key) throw new Error('Key not found: $key'); - Node removed = root!; - if (root!.left == null) { - root = root!.right; - } else { - Node? right = root!.right; - root = root!.left; - // Splay to make sure that the new root has an empty right child. - splay(key); - // Insert the original right child as the right child of the new - // root. - root!.right = right; - } - return removed; - } - - /** - * Returns the node having the specified [key] or null if the tree doesn't - * contain a node with the specified [key]. - */ - Node? find(num key) { - if (isEmpty) return null; - splay(key); - return root!.key == key ? root : null; - } - - /** - * Returns the Node having the maximum key value. - */ - Node? findMax([Node? start]) { - if (isEmpty) return null; - Node current = null == start ? root! : start; - while (current.right != null) current = current.right!; - return current; - } - - /** - * Returns the Node having the maximum key value that - * is less than the specified [key]. - */ - Node? findGreatestLessThan(num key) { - if (isEmpty) return null; - // Splay on the key to move the node with the given key or the last - // node on the search path to the top of the tree. - splay(key); - // Now the result is either the root node or the greatest node in - // the left subtree. - if (root!.key < key) return root; - if (root!.left != null) return findMax(root!.left); - return null; - } - - /** - * Perform the splay operation for the given key. Moves the node with - * the given key to the top of the tree. If no node has the given - * key, the last node on the search path is moved to the top of the - * tree. This is the simplified top-down splaying algorithm from: - * "Self-adjusting Binary Search Trees" by Sleator and Tarjan - */ - void splay(num key) { - if (isEmpty) return; - // Create a dummy node. The use of the dummy node is a bit - // counter-intuitive: The right child of the dummy node will hold - // the L tree of the algorithm. The left child of the dummy node - // will hold the R tree of the algorithm. Using a dummy node, left - // and right will always be nodes and we avoid special cases. - final Node dummy = new Node(0, null); - Node left = dummy; - Node right = dummy; - Node current = root!; - while (true) { - if (key < current.key) { - if (current.left == null) break; - if (key < current.left!.key) { - // Rotate right. - Node tmp = current.left!; - current.left = tmp.right; - tmp.right = current; - current = tmp; - if (current.left == null) break; - } - // Link right. - right.left = current; - right = current; - current = current.left!; - } else if (key > current.key) { - if (current.right == null) break; - if (key > current.right!.key) { - // Rotate left. - Node tmp = current.right!; - current.right = tmp.left; - tmp.left = current; - current = tmp; - if (current.right == null) break; - } - // Link left. - left.right = current; - left = current; - current = current.right!; - } else { - break; - } - } - // Assemble. - left.right = current.left; - right.left = current.right; - current.left = dummy.right; - current.right = dummy.left; - root = current; - } - - /** - * Returns a list with all the keys of the tree. - */ - List exportKeys() { - List result = []; - if (!isEmpty) root!.traverse((Node node) => result.add(node.key)); - return result; - } - - // Tells whether the tree is empty. - bool get isEmpty => null == root; - - // Pointer to the root node of the tree. - Node? root; -} - - -class Node { - Node(this.key, this.value); - final num key; - final Object? value; +class EphemeronNode extends Node { + EphemeronNode(num key, Object? value) : super(key, value); // This ordering of fields is delibrate: one key is visited before the expando // and one after. @@ -344,17 +79,4 @@ class Node { set left(Node? value) => expando[leftKey] = value; Node? get right => expando[rightKey]; set right(Node? value) => expando[rightKey] = value; - - /** - * Performs an ordered traversal of the subtree starting here. - */ - void traverse(void f(Node n)) { - Node? current = this; - while (current != null) { - Node? left = current.left; - if (left != null) left.traverse(f); - f(current); - current = current.right; - } - } } diff --git a/runtime/tests/vm/dart/splay_test.dart b/runtime/tests/vm/dart/splay_test.dart index 207921fc0b86..0e5ee189cb97 100644 --- a/runtime/tests/vm/dart/splay_test.dart +++ b/runtime/tests/vm/dart/splay_test.dart @@ -2,8 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -// This test is a copy of the Splay benchmark that is run with a variety of -// different GC options. It makes for a good GC stress test because it +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it // continuously makes small changes to a large, long-lived data structure, // stressing lots of combinations of references between new-gen and old-gen // objects, and between marked and unmarked objects. @@ -34,102 +34,17 @@ // VMOptions=--no_load_cse --no_dead_store_elimination // VMOptions=--test_il_serialization -import "dart:math"; -import 'package:benchmark_harness/benchmark_harness.dart'; +import "splay_common.dart"; void main() { - Splay.main(); + StrongSplay().main(); } -class Splay extends BenchmarkBase { - const Splay() : super("Splay"); - - // Configuration. - static final int kTreeSize = 8000; - static final int kTreeModifications = 80; - static final int kTreePayloadDepth = 5; - - static SplayTree? tree; - - static Random rnd = new Random(12345); - - // Insert new node with a unique key. - static num insertNewNode() { - num key; - final localTree = tree!; - do { - key = rnd.nextDouble(); - } while (localTree.find(key) != null); - Payload payload = Payload.generate(kTreePayloadDepth, key.toString()); - localTree.insert(key, payload); - return key; - } - - static void mysetup() { - tree = new SplayTree(); - for (int i = 0; i < kTreeSize; i++) insertNewNode(); - } - - static void tearDown() { - // Allow the garbage collector to reclaim the memory - // used by the splay tree no matter how we exit the - // tear down function. - List keys = tree!.exportKeys(); - tree = null; - - // Verify that the splay tree has the right size. - int length = keys.length; - if (length != kTreeSize) throw new Error("Splay tree has wrong size"); - - // Verify that the splay tree has sorted, unique keys. - for (int i = 0; i < length - 1; i++) { - if (keys[i] >= keys[i + 1]) throw new Error("Splay tree not sorted"); - } - } - - void warmup() { - exercise(); - } - - void exercise() { - // Replace a few nodes in the splay tree. - final localTree = tree!; - for (int i = 0; i < kTreeModifications; i++) { - num key = insertNewNode(); - Node? greatest = localTree.findGreatestLessThan(key); - if (greatest == null) { - localTree.remove(key); - } else { - localTree.remove(greatest.key); - } - } - } - - static void main() { - mysetup(); - // Don't use benchmark_harness - exercise runtime is not stable (so - // benchmark_harness measuring approach is meaningless for such benchmark - // anyway). - final sw = Stopwatch()..start(); - final benchmark = new Splay(); - while (sw.elapsedMilliseconds < 2000) { - benchmark.exercise(); - } - tearDown(); - } +class StrongSplay extends Splay { + Object newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object? value) => new StrongNode(key, value); } - -class Leaf { - Leaf(String tag) : - string = "String for key $tag in leaf node", - array = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] - {} - String string; - List array; -} - - class Payload { Payload(this.left, this.right); var left, right; @@ -141,201 +56,8 @@ class Payload { } } -class Error implements Exception { - const Error(this.message); - final String message; -} - - -/** - * A splay tree is a self-balancing binary search tree with the additional - * property that recently accessed elements are quick to access again. - * It performs basic operations such as insertion, look-up and removal - * in O(log(n)) amortized time. - */ -class SplayTree { - SplayTree(); - - /** - * Inserts a node into the tree with the specified [key] and value if - * the tree does not already contain a node with the specified key. If - * the value is inserted, it becomes the root of the tree. - */ - void insert(num key, value) { - if (isEmpty) { - root = new Node(key, value); - return; - } - // Splay on the key to move the last node on the search path for - // the key to the root of the tree. - splay(key); - if (root!.key == key) return; - Node node = new Node(key, value); - if (key > root!.key) { - node.left = root; - node.right = root!.right; - root!.right = null; - } else { - node.right = root; - node.left = root!.left; - root!.left = null; - } - root = node; - } - - /** - * Removes a node with the specified key from the tree if the tree - * contains a node with this key. The removed node is returned. If - * [key] is not found, an exception is thrown. - */ - Node remove(num key) { - if (isEmpty) throw new Error('Key not found: $key'); - splay(key); - if (root!.key != key) throw new Error('Key not found: $key'); - Node removed = root!; - if (root!.left == null) { - root = root!.right; - } else { - Node? right = root!.right; - root = root!.left; - // Splay to make sure that the new root has an empty right child. - splay(key); - // Insert the original right child as the right child of the new - // root. - root!.right = right; - } - return removed; - } - - /** - * Returns the node having the specified [key] or null if the tree doesn't - * contain a node with the specified [key]. - */ - Node? find(num key) { - if (isEmpty) return null; - splay(key); - return root!.key == key ? root : null; - } - - /** - * Returns the Node having the maximum key value. - */ - Node? findMax([Node? start]) { - if (isEmpty) return null; - Node current = null == start ? root! : start; - while (current.right != null) current = current.right!; - return current; - } - - /** - * Returns the Node having the maximum key value that - * is less than the specified [key]. - */ - Node? findGreatestLessThan(num key) { - if (isEmpty) return null; - // Splay on the key to move the node with the given key or the last - // node on the search path to the top of the tree. - splay(key); - // Now the result is either the root node or the greatest node in - // the left subtree. - if (root!.key < key) return root; - if (root!.left != null) return findMax(root!.left); - return null; - } - - /** - * Perform the splay operation for the given key. Moves the node with - * the given key to the top of the tree. If no node has the given - * key, the last node on the search path is moved to the top of the - * tree. This is the simplified top-down splaying algorithm from: - * "Self-adjusting Binary Search Trees" by Sleator and Tarjan - */ - void splay(num key) { - if (isEmpty) return; - // Create a dummy node. The use of the dummy node is a bit - // counter-intuitive: The right child of the dummy node will hold - // the L tree of the algorithm. The left child of the dummy node - // will hold the R tree of the algorithm. Using a dummy node, left - // and right will always be nodes and we avoid special cases. - final Node dummy = new Node(0, null); - Node left = dummy; - Node right = dummy; - Node current = root!; - while (true) { - if (key < current.key) { - if (current.left == null) break; - if (key < current.left!.key) { - // Rotate right. - Node tmp = current.left!; - current.left = tmp.right; - tmp.right = current; - current = tmp; - if (current.left == null) break; - } - // Link right. - right.left = current; - right = current; - current = current.left!; - } else if (key > current.key) { - if (current.right == null) break; - if (key > current.right!.key) { - // Rotate left. - Node tmp = current.right!; - current.right = tmp.left; - tmp.left = current; - current = tmp; - if (current.right == null) break; - } - // Link left. - left.right = current; - left = current; - current = current.right!; - } else { - break; - } - } - // Assemble. - left.right = current.left; - right.left = current.right; - current.left = dummy.right; - current.right = dummy.left; - root = current; - } - - /** - * Returns a list with all the keys of the tree. - */ - List exportKeys() { - List result = []; - if (!isEmpty) root!.traverse((Node node) => result.add(node.key)); - return result; - } - - // Tells whether the tree is empty. - bool get isEmpty => null == root; - - // Pointer to the root node of the tree. - Node? root; -} - - -class Node { - Node(this.key, this.value); - final num key; - final Object? value; +class StrongNode extends Node { + StrongNode(num key, Object? value) : super(key, value); Node? left, right; - - /** - * Performs an ordered traversal of the subtree starting here. - */ - void traverse(void f(Node n)) { - Node? current = this; - while (current != null) { - Node? left = current.left; - if (left != null) left.traverse(f); - f(current); - current = current.right; - } - } } diff --git a/runtime/tests/vm/dart/splay_weak_test.dart b/runtime/tests/vm/dart/splay_weak_test.dart new file mode 100644 index 000000000000..32f8445d7d05 --- /dev/null +++ b/runtime/tests/vm/dart/splay_weak_test.dart @@ -0,0 +1,103 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it +// continuously makes small changes to a large, long-lived data structure, +// stressing lots of combinations of references between new-gen and old-gen +// objects, and between marked and unmarked objects. + +// VMOptions= +// VMOptions=--no_concurrent_mark --no_concurrent_sweep +// VMOptions=--no_concurrent_mark --concurrent_sweep +// VMOptions=--no_concurrent_mark --use_compactor +// VMOptions=--no_concurrent_mark --use_compactor --force_evacuation +// VMOptions=--concurrent_mark --no_concurrent_sweep +// VMOptions=--concurrent_mark --concurrent_sweep +// VMOptions=--concurrent_mark --use_compactor +// VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 +// VMOptions=--verify_before_gc +// VMOptions=--verify_after_gc +// VMOptions=--verify_before_gc --verify_after_gc +// VMOptions=--verify_store_buffer +// VMOptions=--verify_after_marking +// VMOptions=--stress_write_barrier_elimination +// VMOptions=--old_gen_heap_size=150 + +import "splay_common.dart"; + +void main() { + WeakSplay().main(); +} + +class WeakSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object? value) => new WeakNode(key, value); +} + +class Payload { + Payload(left, right) { + this.left = left; + this.right = right; + } + + // This ordering of fields is delibrate: one strong reference visited before + // the weak reference and one after. + var leftWeak; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var leftStrong; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var rightStrong; + var rightWeak; + + get left => leftWeak?.target; + set left(value) { + leftWeak = new WeakReference(value as Object); + // Indirection: chance for WeakRef to be scanned before target is marked. + leftStrong = [[value]]; + } + + get right => rightWeak?.target; + set right(value) { + rightWeak = new WeakReference(value as Object); + // Indirection: chance for WeakRef to be scanned before target is marked. + rightStrong = [[value]]; + } + + static generate(depth, tag) { + if (depth == 0) return new Leaf(tag); + return new Payload(generate(depth - 1, tag), generate(depth - 1, tag)); + } +} + +class WeakNode extends Node { + WeakNode(num key, Object? value) : super(key, value); + + // This ordering of fields is delibrate: one strong reference visited before + // the weak reference and one after. + var leftWeak; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var leftStrong; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var rightStrong; + var rightWeak; + + Node? get left => leftWeak?.target; + set left(Node? value) { + leftWeak = value == null ? null : new WeakReference(value); + // Indirection: chance for WeakRef to be scanned before target is marked. + leftStrong = [[value]]; + } + + Node? get right => rightWeak?.target; + set right(Node? value) { + rightWeak = value == null ? null : new WeakReference(value); + // Indirection: chance for WeakRef to be scanned before target is marked. + rightStrong = [[value]]; + } +} diff --git a/runtime/tests/vm/dart/split_aot_kernel_generation_test.dart b/runtime/tests/vm/dart/split_aot_kernel_generation_test.dart index 1f8d33b0f361..4d9ce89cc363 100644 --- a/runtime/tests/vm/dart/split_aot_kernel_generation_test.dart +++ b/runtime/tests/vm/dart/split_aot_kernel_generation_test.dart @@ -2,7 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -// OtherResources=splay_test.dart +// OtherResources=splay_test.dart splay_common.dart // Tests AOT kernel generation split into 2 steps using '--from-dill' option. diff --git a/runtime/tests/vm/dart_2/splay_c_finalizer_test.dart b/runtime/tests/vm/dart_2/splay_c_finalizer_test.dart new file mode 100644 index 000000000000..0c3301141c61 --- /dev/null +++ b/runtime/tests/vm/dart_2/splay_c_finalizer_test.dart @@ -0,0 +1,89 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it +// continuously makes small changes to a large, long-lived data structure, +// stressing lots of combinations of references between new-gen and old-gen +// objects, and between marked and unmarked objects. + +// This file is copied into another directory and the default opt out scheme of +// CFE using the pattern 'vm/dart_2' doesn't work, so opt it out explicitly. +// @dart=2.9 + +// VMOptions= +// VMOptions=--no_concurrent_mark --no_concurrent_sweep +// VMOptions=--no_concurrent_mark --concurrent_sweep +// VMOptions=--no_concurrent_mark --use_compactor +// VMOptions=--no_concurrent_mark --use_compactor --force_evacuation +// VMOptions=--concurrent_mark --no_concurrent_sweep +// VMOptions=--concurrent_mark --concurrent_sweep +// VMOptions=--concurrent_mark --use_compactor +// VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 +// VMOptions=--verify_before_gc +// VMOptions=--verify_after_gc +// VMOptions=--verify_before_gc --verify_after_gc +// VMOptions=--verify_store_buffer +// VMOptions=--verify_after_marking +// VMOptions=--stress_write_barrier_elimination +// VMOptions=--old_gen_heap_size=150 + +import "dart:ffi"; +import "dart:io"; + +import "splay_common.dart"; + +void main() { + if (Platform.isWindows) { + print("No malloc via self process lookup on Windows"); + return; + } + + // Split across turns so finalizers can run. + FinalizerSplay().mainAsync(); +} + +class FinalizerSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object value) => new FinalizerNode(key, value); +} + +final libc = DynamicLibrary.process(); +typedef MallocForeign = Pointer Function(IntPtr size); +typedef MallocNative = Pointer Function(int size); +final malloc = libc.lookupFunction('malloc'); +typedef FreeForeign = Void Function(Pointer); +final free = libc.lookup>('free'); +final freeFinalizer = NativeFinalizer(free); + +class Leaf implements Finalizable { + final Pointer memory; + + Leaf(String tag) : memory = malloc(15) { + if (memory == nullptr) { + throw OutOfMemoryError(); + } + freeFinalizer.attach(this, memory, detach: this, externalSize: 15); + } +} + +class Payload { + Payload(this.left, this.right); + var left, right; + + static generate(depth, tag) { + if (depth == 0) return new Leaf(tag); + return new Payload(generate(depth - 1, tag), generate(depth - 1, tag)); + } +} + +class FinalizerNode extends Node { + FinalizerNode(num key, Object value) : super(key, value); + + Node left, right; +} diff --git a/runtime/tests/vm/dart_2/splay_common.dart b/runtime/tests/vm/dart_2/splay_common.dart new file mode 100644 index 000000000000..aa918874ecb7 --- /dev/null +++ b/runtime/tests/vm/dart_2/splay_common.dart @@ -0,0 +1,295 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// This file is copied into another directory and the default opt out scheme of +// CFE using the pattern 'vm/dart_2' doesn't work, so opt it out explicitly. +// @dart=2.9 + +import "dart:async"; +import "dart:math"; + +abstract class Node { + Node(this.key, this.value); + final num key; + final Object value; + + Node get left; + set left(Node value); + Node get right; + set right(Node value); + + /** + * Performs an ordered traversal of the subtree starting here. + */ + void traverse(void f(Node n)) { + Node current = this; + while (current != null) { + Node left = current.left; + if (left != null) left.traverse(f); + f(current); + current = current.right; + } + } +} + +class Leaf { + Leaf(String tag) { + string = "String for key $tag in leaf node"; + array = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + } + String string; + List array; +} + +abstract class Splay { + newPayload(int depth, String tag); + Node newNode(num key, Object value); + + // Configuration. + static final int kTreeSize = 8000; + static final int kTreeModifications = 80; + static final int kTreePayloadDepth = 5; + + Random rnd = new Random(12345); + + // Insert new node with a unique key. + num insertNewNode() { + num key; + do { + key = rnd.nextDouble(); + } while (find(key) != null); + insert(key, newPayload(kTreePayloadDepth, key.toString())); + return key; + } + + void setup() { + for (int i = 0; i < kTreeSize; i++) insertNewNode(); + } + + void tearDown() { + // Allow the garbage collector to reclaim the memory + // used by the splay tree no matter how we exit the + // tear down function. + List keys = exportKeys(); + // tree = null; + + // Verify that the splay tree has the right size. + int length = keys.length; + if (length != kTreeSize) throw new Error("Splay tree has wrong size"); + + // Verify that the splay tree has sorted, unique keys. + for (int i = 0; i < length - 1; i++) { + if (keys[i] >= keys[i + 1]) throw new Error("Splay tree not sorted"); + } + } + + void exercise() { + // Replace a few nodes in the splay tree. + for (int i = 0; i < kTreeModifications; i++) { + num key = insertNewNode(); + Node greatest = findGreatestLessThan(key); + if (greatest == null) + remove(key); + else + remove(greatest.key); + } + } + + void main() { + setup(); + final sw = Stopwatch()..start(); + while (sw.elapsedMilliseconds < 2000) { + exercise(); + } + tearDown(); + } + + void mainAsync() { + setup(); + final sw = Stopwatch()..start(); + step() { + if (sw.elapsedMilliseconds < 2000) { + exercise(); + Timer.run(step); + } else { + tearDown(); + } + } + + Timer.run(step); + } + + /** + * A splay tree is a self-balancing binary search tree with the additional + * property that recently accessed elements are quick to access again. + * It performs basic operations such as insertion, look-up and removal + * in O(log(n)) amortized time. + */ + + /** + * Inserts a node into the tree with the specified [key] and value if + * the tree does not already contain a node with the specified key. If + * the value is inserted, it becomes the root of the tree. + */ + void insert(num key, value) { + if (isEmpty) { + root = newNode(key, value); + return; + } + // Splay on the key to move the last node on the search path for + // the key to the root of the tree. + splay(key); + if (root.key == key) return; + Node node = newNode(key, value); + if (key > root.key) { + node.left = root; + node.right = root.right; + root.right = null; + } else { + node.right = root; + node.left = root.left; + root.left = null; + } + root = node; + } + + /** + * Removes a node with the specified key from the tree if the tree + * contains a node with this key. The removed node is returned. If + * [key] is not found, an exception is thrown. + */ + Node remove(num key) { + if (isEmpty) throw new Error('Key not found: $key'); + splay(key); + if (root.key != key) throw new Error('Key not found: $key'); + Node removed = root; + if (root.left == null) { + root = root.right; + } else { + Node right = root.right; + root = root.left; + // Splay to make sure that the new root has an empty right child. + splay(key); + // Insert the original right child as the right child of the new + // root. + root.right = right; + } + return removed; + } + + /** + * Returns the node having the specified [key] or null if the tree doesn't + * contain a node with the specified [key]. + */ + Node find(num key) { + if (isEmpty) return null; + splay(key); + return root.key == key ? root : null; + } + + /** + * Returns the Node having the maximum key value. + */ + Node findMax([Node start]) { + if (isEmpty) return null; + Node current = null == start ? root : start; + while (current.right != null) current = current.right; + return current; + } + + /** + * Returns the Node having the maximum key value that + * is less than the specified [key]. + */ + Node findGreatestLessThan(num key) { + if (isEmpty) return null; + // Splay on the key to move the node with the given key or the last + // node on the search path to the top of the tree. + splay(key); + // Now the result is either the root node or the greatest node in + // the left subtree. + if (root.key < key) return root; + if (root.left != null) return findMax(root.left); + return null; + } + + /** + * Perform the splay operation for the given key. Moves the node with + * the given key to the top of the tree. If no node has the given + * key, the last node on the search path is moved to the top of the + * tree. This is the simplified top-down splaying algorithm from: + * "Self-adjusting Binary Search Trees" by Sleator and Tarjan + */ + void splay(num key) { + if (isEmpty) return; + // Create a dummy node. The use of the dummy node is a bit + // counter-intuitive: The right child of the dummy node will hold + // the L tree of the algorithm. The left child of the dummy node + // will hold the R tree of the algorithm. Using a dummy node, left + // and right will always be nodes and we avoid special cases. + final Node dummy = newNode(null, null); + Node left = dummy; + Node right = dummy; + Node current = root; + while (true) { + if (key < current.key) { + if (current.left == null) break; + if (key < current.left.key) { + // Rotate right. + Node tmp = current.left; + current.left = tmp.right; + tmp.right = current; + current = tmp; + if (current.left == null) break; + } + // Link right. + right.left = current; + right = current; + current = current.left; + } else if (key > current.key) { + if (current.right == null) break; + if (key > current.right.key) { + // Rotate left. + Node tmp = current.right; + current.right = tmp.left; + tmp.left = current; + current = tmp; + if (current.right == null) break; + } + // Link left. + left.right = current; + left = current; + current = current.right; + } else { + break; + } + } + // Assemble. + left.right = current.left; + right.left = current.right; + current.left = dummy.right; + current.right = dummy.left; + root = current; + } + + /** + * Returns a list with all the keys of the tree. + */ + List exportKeys() { + List result = []; + if (!isEmpty) root.traverse((Node node) => result.add(node.key)); + return result; + } + + // Tells whether the tree is empty. + bool get isEmpty => null == root; + + // Pointer to the root node of the tree. + Node root; +} + +class Error implements Exception { + const Error(this.message); + final String message; +} diff --git a/runtime/tests/vm/dart_2/splay_dart_finalizer_test.dart b/runtime/tests/vm/dart_2/splay_dart_finalizer_test.dart new file mode 100644 index 000000000000..6db608389a03 --- /dev/null +++ b/runtime/tests/vm/dart_2/splay_dart_finalizer_test.dart @@ -0,0 +1,96 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it +// continuously makes small changes to a large, long-lived data structure, +// stressing lots of combinations of references between new-gen and old-gen +// objects, and between marked and unmarked objects. + +// This file is copied into another directory and the default opt out scheme of +// CFE using the pattern 'vm/dart_2' doesn't work, so opt it out explicitly. +// @dart=2.9 + +// VMOptions= +// VMOptions=--no_concurrent_mark --no_concurrent_sweep +// VMOptions=--no_concurrent_mark --concurrent_sweep +// VMOptions=--no_concurrent_mark --use_compactor +// VMOptions=--no_concurrent_mark --use_compactor --force_evacuation +// VMOptions=--concurrent_mark --no_concurrent_sweep +// VMOptions=--concurrent_mark --concurrent_sweep +// VMOptions=--concurrent_mark --use_compactor +// VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 +// VMOptions=--verify_before_gc +// VMOptions=--verify_after_gc +// VMOptions=--verify_before_gc --verify_after_gc +// VMOptions=--verify_store_buffer +// VMOptions=--verify_after_marking +// VMOptions=--stress_write_barrier_elimination +// VMOptions=--old_gen_heap_size=300 + +import "splay_common.dart"; + +void main() { + // Split across turns so finalizers can run. + FinalizerSplay().mainAsync(); +} + +class FinalizerSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object value) => new FinalizerNode(key, value); +} + +final payloadLeft = {}; +final payloadRight = {}; +finalizePayload(Object token) { + payloadLeft.remove(token); + payloadRight.remove(token); +} + +final payloadFinalizer = new Finalizer(finalizePayload); + +class Payload { + Payload(left, right) { + this.left = left; + this.right = right; + payloadFinalizer.attach(this, token, detach: this); + } + var token = new Object(); + get left => payloadLeft[token]; + set left(value) => payloadLeft[token] = value; + get right => payloadRight[token]; + set right(value) => payloadRight[token] = value; + + static generate(depth, tag) { + if (depth == 0) return new Leaf(tag); + return new Payload(generate(depth - 1, tag), generate(depth - 1, tag)); + } +} + +final nodeLeft = {}; +final nodeRight = {}; +finalizeNode(Object token) { + nodeLeft.remove(token); + nodeRight.remove(token); +} + +final nodeFinalizer = new Finalizer(finalizeNode); + +class FinalizerNode extends Node { + FinalizerNode(num key, Object value) : super(key, value) { + this.left = null; + this.right = null; + nodeFinalizer.attach(this, token, detach: this); + } + + var token = new Object(); + Node get left => nodeLeft[token]; + set left(Node value) => nodeLeft[token] = value; + Node get right => nodeRight[token]; + set right(Node value) => nodeRight[token] = value; +} diff --git a/runtime/tests/vm/dart_2/splay_ephemeron_test.dart b/runtime/tests/vm/dart_2/splay_ephemeron_test.dart index 9e8da8861139..0f435fef89af 100644 --- a/runtime/tests/vm/dart_2/splay_ephemeron_test.dart +++ b/runtime/tests/vm/dart_2/splay_ephemeron_test.dart @@ -2,8 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -// This test is a copy of the Splay benchmark that is run with a variety of -// different GC options. It makes for a good GC stress test because it +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it // continuously makes small changes to a large, long-lived data structure, // stressing lots of combinations of references between new-gen and old-gen // objects, and between marked and unmarked objects. @@ -35,97 +35,17 @@ // VMOptions=--stress_write_barrier_elimination // VMOptions=--old_gen_heap_size=150 -import "dart:math"; -import 'package:benchmark_harness/benchmark_harness.dart'; +import "splay_common.dart"; void main() { - Splay.main(); + EphemeronSplay().main(); } -class Splay extends BenchmarkBase { - const Splay() : super("Splay"); - - // Configuration. - static final int kTreeSize = 8000; - static final int kTreeModifications = 80; - static final int kTreePayloadDepth = 5; - - static SplayTree tree; - - static Random rnd = new Random(12345); - - // Insert new node with a unique key. - static num insertNewNode() { - num key; - do { - key = rnd.nextDouble(); - } while (tree.find(key) != null); - Payload payload = Payload.generate(kTreePayloadDepth, key.toString()); - tree.insert(key, payload); - return key; - } - - static void mysetup() { - tree = new SplayTree(); - for (int i = 0; i < kTreeSize; i++) insertNewNode(); - } - - static void tearDown() { - // Allow the garbage collector to reclaim the memory - // used by the splay tree no matter how we exit the - // tear down function. - List keys = tree.exportKeys(); - tree = null; - - // Verify that the splay tree has the right size. - int length = keys.length; - if (length != kTreeSize) throw new Error("Splay tree has wrong size"); - - // Verify that the splay tree has sorted, unique keys. - for (int i = 0; i < length - 1; i++) { - if (keys[i] >= keys[i + 1]) throw new Error("Splay tree not sorted"); - } - } - - void warmup() { - exercise(); - } - - void exercise() { - // Replace a few nodes in the splay tree. - for (int i = 0; i < kTreeModifications; i++) { - num key = insertNewNode(); - Node greatest = tree.findGreatestLessThan(key); - if (greatest == null) tree.remove(key); - else tree.remove(greatest.key); - } - } - - static void main() { - mysetup(); - // Don't use benchmark_harness - exercise runtime is not stable (so - // benchmark_harness measuring approach is meaningless for such benchmark - // anyway). - final sw = Stopwatch()..start(); - final benchmark = new Splay(); - while (sw.elapsedMilliseconds < 2000) { - benchmark.exercise(); - } - tearDown(); - } +class EphemeronSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object value) => new EphemeronNode(key, value); } - -class Leaf { - Leaf(String tag) { - string = "String for key $tag in leaf node"; - array = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ]; - } - String string; - List array; -} - - class Payload { Payload(left, right) { this.left = left; @@ -150,188 +70,8 @@ class Payload { } } -class Error implements Exception { - const Error(this.message); - final String message; -} - - -/** - * A splay tree is a self-balancing binary search tree with the additional - * property that recently accessed elements are quick to access again. - * It performs basic operations such as insertion, look-up and removal - * in O(log(n)) amortized time. - */ -class SplayTree { - SplayTree(); - - /** - * Inserts a node into the tree with the specified [key] and value if - * the tree does not already contain a node with the specified key. If - * the value is inserted, it becomes the root of the tree. - */ - void insert(num key, value) { - if (isEmpty) { - root = new Node(key, value); - return; - } - // Splay on the key to move the last node on the search path for - // the key to the root of the tree. - splay(key); - if (root.key == key) return; - Node node = new Node(key, value); - if (key > root.key) { - node.left = root; - node.right = root.right; - root.right = null; - } else { - node.right = root; - node.left = root.left; - root.left = null; - } - root = node; - } - - /** - * Removes a node with the specified key from the tree if the tree - * contains a node with this key. The removed node is returned. If - * [key] is not found, an exception is thrown. - */ - Node remove(num key) { - if (isEmpty) throw new Error('Key not found: $key'); - splay(key); - if (root.key != key) throw new Error('Key not found: $key'); - Node removed = root; - if (root.left == null) { - root = root.right; - } else { - Node right = root.right; - root = root.left; - // Splay to make sure that the new root has an empty right child. - splay(key); - // Insert the original right child as the right child of the new - // root. - root.right = right; - } - return removed; - } - - /** - * Returns the node having the specified [key] or null if the tree doesn't - * contain a node with the specified [key]. - */ - Node find(num key) { - if (isEmpty) return null; - splay(key); - return root.key == key ? root : null; - } - - /** - * Returns the Node having the maximum key value. - */ - Node findMax([Node start]) { - if (isEmpty) return null; - Node current = null == start ? root : start; - while (current.right != null) current = current.right; - return current; - } - - /** - * Returns the Node having the maximum key value that - * is less than the specified [key]. - */ - Node findGreatestLessThan(num key) { - if (isEmpty) return null; - // Splay on the key to move the node with the given key or the last - // node on the search path to the top of the tree. - splay(key); - // Now the result is either the root node or the greatest node in - // the left subtree. - if (root.key < key) return root; - if (root.left != null) return findMax(root.left); - return null; - } - - /** - * Perform the splay operation for the given key. Moves the node with - * the given key to the top of the tree. If no node has the given - * key, the last node on the search path is moved to the top of the - * tree. This is the simplified top-down splaying algorithm from: - * "Self-adjusting Binary Search Trees" by Sleator and Tarjan - */ - void splay(num key) { - if (isEmpty) return; - // Create a dummy node. The use of the dummy node is a bit - // counter-intuitive: The right child of the dummy node will hold - // the L tree of the algorithm. The left child of the dummy node - // will hold the R tree of the algorithm. Using a dummy node, left - // and right will always be nodes and we avoid special cases. - final Node dummy = new Node(null, null); - Node left = dummy; - Node right = dummy; - Node current = root; - while (true) { - if (key < current.key) { - if (current.left == null) break; - if (key < current.left.key) { - // Rotate right. - Node tmp = current.left; - current.left = tmp.right; - tmp.right = current; - current = tmp; - if (current.left == null) break; - } - // Link right. - right.left = current; - right = current; - current = current.left; - } else if (key > current.key) { - if (current.right == null) break; - if (key > current.right.key) { - // Rotate left. - Node tmp = current.right; - current.right = tmp.left; - tmp.left = current; - current = tmp; - if (current.right == null) break; - } - // Link left. - left.right = current; - left = current; - current = current.right; - } else { - break; - } - } - // Assemble. - left.right = current.left; - right.left = current.right; - current.left = dummy.right; - current.right = dummy.left; - root = current; - } - - /** - * Returns a list with all the keys of the tree. - */ - List exportKeys() { - List result = []; - if (!isEmpty) root.traverse((Node node) => result.add(node.key)); - return result; - } - - // Tells whether the tree is empty. - bool get isEmpty => null == root; - - // Pointer to the root node of the tree. - Node root; -} - - -class Node { - Node(this.key, this.value); - final num key; - final Object value; +class EphemeronNode extends Node { + EphemeronNode(num key, Object value) : super(key, value); // This ordering of fields is delibrate: one key is visited before the expando // and one after. @@ -343,17 +83,4 @@ class Node { set left(Node value) => expando[leftKey] = value; Node get right => expando[rightKey]; set right(Node value) => expando[rightKey] = value; - - /** - * Performs an ordered traversal of the subtree starting here. - */ - void traverse(void f(Node n)) { - Node current = this; - while (current != null) { - Node left = current.left; - if (left != null) left.traverse(f); - f(current); - current = current.right; - } - } } diff --git a/runtime/tests/vm/dart_2/splay_test.dart b/runtime/tests/vm/dart_2/splay_test.dart index 9f44661bdeee..18cfe1080231 100644 --- a/runtime/tests/vm/dart_2/splay_test.dart +++ b/runtime/tests/vm/dart_2/splay_test.dart @@ -2,8 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -// This test is a copy of the Splay benchmark that is run with a variety of -// different GC options. It makes for a good GC stress test because it +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it // continuously makes small changes to a large, long-lived data structure, // stressing lots of combinations of references between new-gen and old-gen // objects, and between marked and unmarked objects. @@ -38,97 +38,17 @@ // VMOptions=--no_load_cse --no_dead_store_elimination // VMOptions=--test_il_serialization -import "dart:math"; -import 'package:benchmark_harness/benchmark_harness.dart'; +import "splay_common.dart"; void main() { - Splay.main(); + StrongSplay().main(); } -class Splay extends BenchmarkBase { - const Splay() : super("Splay"); - - // Configuration. - static final int kTreeSize = 8000; - static final int kTreeModifications = 80; - static final int kTreePayloadDepth = 5; - - static SplayTree tree; - - static Random rnd = new Random(12345); - - // Insert new node with a unique key. - static num insertNewNode() { - num key; - do { - key = rnd.nextDouble(); - } while (tree.find(key) != null); - Payload payload = Payload.generate(kTreePayloadDepth, key.toString()); - tree.insert(key, payload); - return key; - } - - static void mysetup() { - tree = new SplayTree(); - for (int i = 0; i < kTreeSize; i++) insertNewNode(); - } - - static void tearDown() { - // Allow the garbage collector to reclaim the memory - // used by the splay tree no matter how we exit the - // tear down function. - List keys = tree.exportKeys(); - tree = null; - - // Verify that the splay tree has the right size. - int length = keys.length; - if (length != kTreeSize) throw new Error("Splay tree has wrong size"); - - // Verify that the splay tree has sorted, unique keys. - for (int i = 0; i < length - 1; i++) { - if (keys[i] >= keys[i + 1]) throw new Error("Splay tree not sorted"); - } - } - - void warmup() { - exercise(); - } - - void exercise() { - // Replace a few nodes in the splay tree. - for (int i = 0; i < kTreeModifications; i++) { - num key = insertNewNode(); - Node greatest = tree.findGreatestLessThan(key); - if (greatest == null) tree.remove(key); - else tree.remove(greatest.key); - } - } - - static void main() { - mysetup(); - // Don't use benchmark_harness - exercise runtime is not stable (so - // benchmark_harness measuring approach is meaningless for such benchmark - // anyway). - final sw = Stopwatch()..start(); - final benchmark = new Splay(); - while (sw.elapsedMilliseconds < 2000) { - benchmark.exercise(); - } - tearDown(); - } +class StrongSplay extends Splay { + Object newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object value) => new StrongNode(key, value); } - -class Leaf { - Leaf(String tag) { - string = "String for key $tag in leaf node"; - array = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ]; - } - String string; - List array; -} - - class Payload { Payload(this.left, this.right); var left, right; @@ -140,201 +60,8 @@ class Payload { } } -class Error implements Exception { - const Error(this.message); - final String message; -} - - -/** - * A splay tree is a self-balancing binary search tree with the additional - * property that recently accessed elements are quick to access again. - * It performs basic operations such as insertion, look-up and removal - * in O(log(n)) amortized time. - */ -class SplayTree { - SplayTree(); - - /** - * Inserts a node into the tree with the specified [key] and value if - * the tree does not already contain a node with the specified key. If - * the value is inserted, it becomes the root of the tree. - */ - void insert(num key, value) { - if (isEmpty) { - root = new Node(key, value); - return; - } - // Splay on the key to move the last node on the search path for - // the key to the root of the tree. - splay(key); - if (root.key == key) return; - Node node = new Node(key, value); - if (key > root.key) { - node.left = root; - node.right = root.right; - root.right = null; - } else { - node.right = root; - node.left = root.left; - root.left = null; - } - root = node; - } - - /** - * Removes a node with the specified key from the tree if the tree - * contains a node with this key. The removed node is returned. If - * [key] is not found, an exception is thrown. - */ - Node remove(num key) { - if (isEmpty) throw new Error('Key not found: $key'); - splay(key); - if (root.key != key) throw new Error('Key not found: $key'); - Node removed = root; - if (root.left == null) { - root = root.right; - } else { - Node right = root.right; - root = root.left; - // Splay to make sure that the new root has an empty right child. - splay(key); - // Insert the original right child as the right child of the new - // root. - root.right = right; - } - return removed; - } - - /** - * Returns the node having the specified [key] or null if the tree doesn't - * contain a node with the specified [key]. - */ - Node find(num key) { - if (isEmpty) return null; - splay(key); - return root.key == key ? root : null; - } - - /** - * Returns the Node having the maximum key value. - */ - Node findMax([Node start]) { - if (isEmpty) return null; - Node current = null == start ? root : start; - while (current.right != null) current = current.right; - return current; - } - - /** - * Returns the Node having the maximum key value that - * is less than the specified [key]. - */ - Node findGreatestLessThan(num key) { - if (isEmpty) return null; - // Splay on the key to move the node with the given key or the last - // node on the search path to the top of the tree. - splay(key); - // Now the result is either the root node or the greatest node in - // the left subtree. - if (root.key < key) return root; - if (root.left != null) return findMax(root.left); - return null; - } - - /** - * Perform the splay operation for the given key. Moves the node with - * the given key to the top of the tree. If no node has the given - * key, the last node on the search path is moved to the top of the - * tree. This is the simplified top-down splaying algorithm from: - * "Self-adjusting Binary Search Trees" by Sleator and Tarjan - */ - void splay(num key) { - if (isEmpty) return; - // Create a dummy node. The use of the dummy node is a bit - // counter-intuitive: The right child of the dummy node will hold - // the L tree of the algorithm. The left child of the dummy node - // will hold the R tree of the algorithm. Using a dummy node, left - // and right will always be nodes and we avoid special cases. - final Node dummy = new Node(null, null); - Node left = dummy; - Node right = dummy; - Node current = root; - while (true) { - if (key < current.key) { - if (current.left == null) break; - if (key < current.left.key) { - // Rotate right. - Node tmp = current.left; - current.left = tmp.right; - tmp.right = current; - current = tmp; - if (current.left == null) break; - } - // Link right. - right.left = current; - right = current; - current = current.left; - } else if (key > current.key) { - if (current.right == null) break; - if (key > current.right.key) { - // Rotate left. - Node tmp = current.right; - current.right = tmp.left; - tmp.left = current; - current = tmp; - if (current.right == null) break; - } - // Link left. - left.right = current; - left = current; - current = current.right; - } else { - break; - } - } - // Assemble. - left.right = current.left; - right.left = current.right; - current.left = dummy.right; - current.right = dummy.left; - root = current; - } - - /** - * Returns a list with all the keys of the tree. - */ - List exportKeys() { - List result = []; - if (!isEmpty) root.traverse((Node node) => result.add(node.key)); - return result; - } - - // Tells whether the tree is empty. - bool get isEmpty => null == root; - - // Pointer to the root node of the tree. - Node root; -} - - -class Node { - Node(this.key, this.value); - final num key; - final Object value; +class StrongNode extends Node { + StrongNode(num key, Object value) : super(key, value); Node left, right; - - /** - * Performs an ordered traversal of the subtree starting here. - */ - void traverse(void f(Node n)) { - Node current = this; - while (current != null) { - Node left = current.left; - if (left != null) left.traverse(f); - f(current); - current = current.right; - } - } } diff --git a/runtime/tests/vm/dart_2/splay_weak_test.dart b/runtime/tests/vm/dart_2/splay_weak_test.dart new file mode 100644 index 000000000000..ccf551fd2868 --- /dev/null +++ b/runtime/tests/vm/dart_2/splay_weak_test.dart @@ -0,0 +1,107 @@ +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// This test is a derivative of the Splay benchmark that is run with a variety +// of different GC options. It makes for a good GC stress test because it +// continuously makes small changes to a large, long-lived data structure, +// stressing lots of combinations of references between new-gen and old-gen +// objects, and between marked and unmarked objects. + +// This file is copied into another directory and the default opt out scheme of +// CFE using the pattern 'vm/dart_2' doesn't work, so opt it out explicitly. +// @dart=2.9 + +// VMOptions= +// VMOptions=--no_concurrent_mark --no_concurrent_sweep +// VMOptions=--no_concurrent_mark --concurrent_sweep +// VMOptions=--no_concurrent_mark --use_compactor +// VMOptions=--no_concurrent_mark --use_compactor --force_evacuation +// VMOptions=--concurrent_mark --no_concurrent_sweep +// VMOptions=--concurrent_mark --concurrent_sweep +// VMOptions=--concurrent_mark --use_compactor +// VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 +// VMOptions=--verify_before_gc +// VMOptions=--verify_after_gc +// VMOptions=--verify_before_gc --verify_after_gc +// VMOptions=--verify_store_buffer +// VMOptions=--verify_after_marking +// VMOptions=--stress_write_barrier_elimination +// VMOptions=--old_gen_heap_size=150 + +import "splay_common.dart"; + +void main() { + WeakSplay().main(); +} + +class WeakSplay extends Splay { + newPayload(int depth, String tag) => Payload.generate(depth, tag); + Node newNode(num key, Object value) => new WeakNode(key, value); +} + +class Payload { + Payload(left, right) { + this.left = left; + this.right = right; + } + + // This ordering of fields is delibrate: one strong reference visited before + // the weak reference and one after. + var leftWeak; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var leftStrong; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var rightStrong; + var rightWeak; + + get left => leftWeak?.target; + set left(value) { + leftWeak = new WeakReference(value); + // Indirection: chance for WeakRef to be scanned before target is marked. + leftStrong = [[value]]; + } + + get right => rightWeak?.target; + set right(value) { + rightWeak = new WeakReference(value); + // Indirection: chance for WeakRef to be scanned before target is marked. + rightStrong = [[value]]; + } + + static generate(depth, tag) { + if (depth == 0) return new Leaf(tag); + return new Payload(generate(depth - 1, tag), generate(depth - 1, tag)); + } +} + +class WeakNode extends Node { + WeakNode(num key, Object value) : super(key, value); + + // This ordering of fields is delibrate: one strong reference visited before + // the weak reference and one after. + var leftWeak; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var leftStrong; + @pragma("vm:entry-point") // TODO(50571): Remove illegal optimization. + var rightStrong; + var rightWeak; + + Node get left => leftWeak?.target; + set left(Node value) { + leftWeak = value == null ? null : new WeakReference(value); + // Indirection: chance for WeakRef to be scanned before target is marked. + leftStrong = [[value]]; + } + + Node get right => rightWeak?.target; + set right(Node value) { + rightWeak = value == null ? null : new WeakReference(value); + // Indirection: chance for WeakRef to be scanned before target is marked. + rightStrong = [[value]]; + } +} diff --git a/runtime/tests/vm/dart_2/split_aot_kernel_generation_test.dart b/runtime/tests/vm/dart_2/split_aot_kernel_generation_test.dart index 7646544b5fd5..183f506055c3 100644 --- a/runtime/tests/vm/dart_2/split_aot_kernel_generation_test.dart +++ b/runtime/tests/vm/dart_2/split_aot_kernel_generation_test.dart @@ -4,7 +4,7 @@ // @dart = 2.9 -// OtherResources=splay_test.dart +// OtherResources=splay_test.dart splay_common.dart // Tests AOT kernel generation split into 2 steps using '--from-dill' option. diff --git a/runtime/vm/heap/pages.h b/runtime/vm/heap/pages.h index 406cdedde7b8..e4a5477595dc 100644 --- a/runtime/vm/heap/pages.h +++ b/runtime/vm/heap/pages.h @@ -253,18 +253,23 @@ class PageSpace { bool AllocatedExternal(intptr_t size) { ASSERT(size >= 0); intptr_t size_in_words = size >> kWordSizeLog2; - intptr_t next_external_in_words = usage_.external_in_words + size_in_words; - if (next_external_in_words < 0 || - next_external_in_words > kMaxAddrSpaceInWords) { - return false; - } - usage_.external_in_words = next_external_in_words; + intptr_t expected = usage_.external_in_words.load(); + intptr_t desired; + do { + desired = expected + size_in_words; + if (desired < 0 || desired > kMaxAddrSpaceInWords) { + return false; + } + ASSERT(desired >= 0); + } while ( + !usage_.external_in_words.compare_exchange_weak(expected, desired)); return true; } void FreedExternal(intptr_t size) { ASSERT(size >= 0); intptr_t size_in_words = size >> kWordSizeLog2; usage_.external_in_words -= size_in_words; + ASSERT(usage_.external_in_words >= 0); } // Bulk data allocation. diff --git a/runtime/vm/heap/scavenger.cc b/runtime/vm/heap/scavenger.cc index 08ad0df6cf32..5990740ba24e 100644 --- a/runtime/vm/heap/scavenger.cc +++ b/runtime/vm/heap/scavenger.cc @@ -62,18 +62,21 @@ enum { COMPILE_ASSERT(static_cast(kForwarded) == static_cast(kHeapObjectTag)); -static inline bool IsForwarding(uword header) { +DART_FORCE_INLINE +static bool IsForwarding(uword header) { uword bits = header & kForwardingMask; ASSERT((bits == kNotForwarded) || (bits == kForwarded)); return bits == kForwarded; } -static inline ObjectPtr ForwardedObj(uword header) { +DART_FORCE_INLINE +static ObjectPtr ForwardedObj(uword header) { ASSERT(IsForwarding(header)); return static_cast(header); } -static inline uword ForwardingHeader(ObjectPtr target) { +DART_FORCE_INLINE +static uword ForwardingHeader(ObjectPtr target) { uword result = static_cast(target); ASSERT(IsForwarding(result)); return result; @@ -86,7 +89,7 @@ static inline uword ForwardingHeader(ObjectPtr target) { // worker in the current thread will abandon this copy. We do not mark the loads // here as relaxed so the C++ compiler still has the freedom to reorder them. NO_SANITIZE_THREAD -static inline void objcpy(void* dst, const void* src, size_t size) { +static void objcpy(void* dst, const void* src, size_t size) { // A memcopy specialized for objects. We can assume: // - dst and src do not overlap ASSERT( @@ -284,7 +287,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor { } } - inline void ProcessWeakProperties(); + void ProcessWeakProperties(); bool HasWork() { if (scavenger_->abort_) return false; @@ -330,6 +333,8 @@ class ScavengerVisitorBase : public ObjectPointerVisitor { static bool ForwardOrSetNullIfCollected(uword heap_base, CompressedObjectPtr* ptr_address); + void ProcessOldFinalizerEntry(FinalizerEntryPtr entry); + private: void UpdateStoreBuffer(ObjectPtr obj) { ASSERT(obj->IsHeapObject()); @@ -511,7 +516,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor { return TryAllocateCopySlow(size); } - DART_NOINLINE inline uword TryAllocateCopySlow(intptr_t size); + DART_NOINLINE uword TryAllocateCopySlow(intptr_t size); DART_NOINLINE DART_NORETURN void AbortScavenge() { if (FLAG_verbose_gc) { @@ -523,9 +528,9 @@ class ScavengerVisitorBase : public ObjectPointerVisitor { thread_->long_jump_base()->Jump(1); } - inline void ProcessToSpace(); + void ProcessToSpace(); DART_FORCE_INLINE intptr_t ProcessCopied(ObjectPtr raw_obj); - inline void ProcessPromotedList(); + void ProcessPromotedList(); bool IsNotForwarding(ObjectPtr raw) { ASSERT(raw->IsHeapObject()); @@ -533,8 +538,8 @@ class ScavengerVisitorBase : public ObjectPointerVisitor { return !IsForwarding(ReadHeaderRelaxed(raw)); } - inline void MournWeakProperties(); - inline void MournOrUpdateWeakReferences(); + void MournWeakProperties(); + void MournOrUpdateWeakReferences(); Thread* thread_; Scavenger* scavenger_; @@ -1117,7 +1122,6 @@ void Scavenger::IterateStoreBuffers(ScavengerVisitorBase* visitor) { ASSERT(raw_object->untag()->IsRemembered()); raw_object->untag()->ClearRememberedBit(); visitor->VisitingOldObject(raw_object); - intptr_t class_id = raw_object->GetClassId(); // This treats old-space weak references in WeakProperty, WeakReference, // and FinalizerEntry as strong references. This prevents us from having // to enqueue them in `visitor->delayed_`. Enqueuing them in the delayed @@ -1125,37 +1129,9 @@ void Scavenger::IterateStoreBuffers(ScavengerVisitorBase* visitor) { // marking and one for the objects seen in the store buffers + new space. // Treating the weak references as strong here means we can have a single // `next_seen_by_gc` field. - if (UNLIKELY(class_id == kFinalizerEntryCid)) { - FinalizerEntryPtr raw_entry = - static_cast(raw_object); - if (FLAG_trace_finalizers) { - THR_Print("Scavenger::IterateStoreBuffers Processing Entry %p\n", - raw_entry->untag()); - } - // Detect `FinalizerEntry::value` promotion to update external space. - // - // This treats old-space FinalizerEntry fields as strong. Values, detach - // keys, and finalizers in new space won't be reclaimed until after they - // are promoted. - // This will only visit the strong references, end enqueue the entry. - // This enables us to update external space in MournFinalized. - const Heap::Space before_gc_space = SpaceForExternal(raw_entry); - UntaggedFinalizerEntry::VisitFinalizerEntryPointers(raw_entry, visitor); - if (before_gc_space == Heap::kNew) { - const Heap::Space after_gc_space = SpaceForExternal(raw_entry); - if (after_gc_space == Heap::kOld) { - const intptr_t external_size = raw_entry->untag()->external_size_; - if (external_size > 0) { - if (FLAG_trace_finalizers) { - THR_Print( - "Scavenger %p Store buffer, promoting external size %" Pd - " bytes from new to old space\n", - visitor, external_size); - } - visitor->isolate_group()->heap()->PromotedExternal(external_size); - } - } - } + if (UNLIKELY(raw_object->GetClassId() == kFinalizerEntryCid)) { + visitor->ProcessOldFinalizerEntry( + static_cast(raw_object)); } else { // This treats old-space WeakProperties and WeakReferences as strong. A // dead key or target won't be reclaimed until after it is promoted. @@ -1271,7 +1247,11 @@ void ScavengerVisitorBase::ProcessPromotedList() { // objects to be resolved in the to space. ASSERT(!raw_object->untag()->IsRemembered()); VisitingOldObject(raw_object); - raw_object->untag()->VisitPointersNonvirtual(this); + if (UNLIKELY(raw_object->GetClassId() == kFinalizerEntryCid)) { + ProcessOldFinalizerEntry(static_cast(raw_object)); + } else { + raw_object->untag()->VisitPointersNonvirtual(this); + } if (raw_object->untag()->IsMarked()) { // Complete our promise from ScavengePointer. Note that marker cannot // visit this object until it pops a block from the mark stack, which @@ -1401,6 +1381,37 @@ intptr_t ScavengerVisitorBase::ProcessCopied(ObjectPtr raw_obj) { return raw_obj->untag()->VisitPointersNonvirtual(this); } +template +void ScavengerVisitorBase::ProcessOldFinalizerEntry( + FinalizerEntryPtr raw_entry) { + if (FLAG_trace_finalizers) { + THR_Print("Scavenger::ProcessOldFinalizerEntry %p\n", raw_entry->untag()); + } + // Detect `FinalizerEntry::value` promotion to update external space. + // + // This treats old-space FinalizerEntry fields as strong. Values, detach + // keys, and finalizers in new space won't be reclaimed until after they + // are promoted. + // This will only visit the strong references, end enqueue the entry. + // This enables us to update external space in MournFinalized. + const Heap::Space before_gc_space = SpaceForExternal(raw_entry); + UntaggedFinalizerEntry::VisitFinalizerEntryPointers(raw_entry, this); + if (before_gc_space == Heap::kNew) { + const Heap::Space after_gc_space = SpaceForExternal(raw_entry); + if (after_gc_space == Heap::kOld) { + const intptr_t external_size = raw_entry->untag()->external_size_; + if (external_size > 0) { + if (FLAG_trace_finalizers) { + THR_Print("Scavenger %p, promoting external size %" Pd + " bytes from new to old space\n", + this, external_size); + } + isolate_group()->heap()->PromotedExternal(external_size); + } + } + } +} + void Scavenger::MournWeakTables() { TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "MournWeakTables"); diff --git a/runtime/vm/heap/scavenger.h b/runtime/vm/heap/scavenger.h index 4f133caea3d7..9f04c8e21a47 100644 --- a/runtime/vm/heap/scavenger.h +++ b/runtime/vm/heap/scavenger.h @@ -194,14 +194,18 @@ class Scavenger { // this allocation will make it exceed kMaxAddrSpaceInWords. bool AllocatedExternal(intptr_t size) { ASSERT(size >= 0); - intptr_t next_external_size_in_words = - (external_size_ >> kWordSizeLog2) + (size >> kWordSizeLog2); - if (next_external_size_in_words < 0 || - next_external_size_in_words > kMaxAddrSpaceInWords) { - return false; - } - external_size_ += size; - ASSERT(external_size_ >= 0); + intptr_t expected = external_size_.load(); + intptr_t desired; + do { + intptr_t next_external_size_in_words = + (external_size_ >> kWordSizeLog2) + (size >> kWordSizeLog2); + if (next_external_size_in_words < 0 || + next_external_size_in_words > kMaxAddrSpaceInWords) { + return false; + } + desired = expected + size; + ASSERT(desired >= 0); + } while (!external_size_.compare_exchange_weak(expected, desired)); return true; } void FreedExternal(intptr_t size) { diff --git a/tests/ffi/finalizer_external_size_accounting_test.dart b/tests/ffi/finalizer_external_size_accounting_test.dart new file mode 100644 index 000000000000..e29d56aba320 --- /dev/null +++ b/tests/ffi/finalizer_external_size_accounting_test.dart @@ -0,0 +1,48 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// Issue https://github.com/dart-lang/sdk/issues/50537 + +import "dart:async"; +import "dart:ffi"; +import "dart:io"; + +final libc = DynamicLibrary.process(); +typedef MallocForeign = Pointer Function(IntPtr size); +typedef MallocNative = Pointer Function(int size); +final malloc = libc.lookupFunction('malloc'); +typedef FreeForeign = Void Function(Pointer); +final free = libc.lookup>('free'); +final freeFinalizer = NativeFinalizer(free); + +class Resource implements Finalizable { + Pointer _target; + Resource() : _target = malloc(8) { + if (_target == nullptr) { + throw OutOfMemoryError(); + } + freeFinalizer.attach(this, _target, detach: this, externalSize: 8); + } +} + +void main() { + if (Platform.isWindows) { + print("No malloc via self process lookup on Windows"); + return; + } + + // Split across turns so the internal finalizer cleanup can run. + // Cf. https://github.com/dart-lang/sdk/issues/50570 + final sw = Stopwatch()..start(); + step() { + if (sw.elapsedMilliseconds < 2000) { + // VM assertion: external size of each generation should always be + // non-negative. + List.generate(1000, (_) => new Resource()); + Timer.run(step); + } + } + + Timer.run(step); +} diff --git a/tests/ffi_2/finalizer_external_size_accounting_test.dart b/tests/ffi_2/finalizer_external_size_accounting_test.dart new file mode 100644 index 000000000000..dd3755996f5d --- /dev/null +++ b/tests/ffi_2/finalizer_external_size_accounting_test.dart @@ -0,0 +1,50 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// Issue https://github.com/dart-lang/sdk/issues/50537 + +// @dart=2.9 + +import "dart:async"; +import "dart:ffi"; +import "dart:io"; + +final libc = DynamicLibrary.process(); +typedef MallocForeign = Pointer Function(IntPtr size); +typedef MallocNative = Pointer Function(int size); +final malloc = libc.lookupFunction('malloc'); +typedef FreeForeign = Void Function(Pointer); +final free = libc.lookup>('free'); +final freeFinalizer = NativeFinalizer(free); + +class Resource implements Finalizable { + Pointer _target; + Resource() : _target = malloc(8) { + if (_target == nullptr) { + throw OutOfMemoryError(); + } + freeFinalizer.attach(this, _target, detach: this, externalSize: 8); + } +} + +void main() { + if (Platform.isWindows) { + print("No malloc via self process lookup on Windows"); + return; + } + + // Split across turns so the internal finalizer cleanup can run. + // Cf. https://github.com/dart-lang/sdk/issues/50570 + final sw = Stopwatch()..start(); + step() { + if (sw.elapsedMilliseconds < 2000) { + // VM assertion: external size of each generation should always be + // non-negative. + List.generate(1000, (_) => new Resource()); + Timer.run(step); + } + } + + Timer.run(step); +}