Don't visit every uop on every collection

brandtbucher · Sep 19, 2024 · cae9e43 · cae9e43
1 parent f2327f4
commit cae9e43
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 5 deletions.
diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h
@@ -259,6 +259,7 @@ struct _is {
     struct callable_cache callable_cache;
     _PyOptimizerObject *optimizer;
     _PyExecutorObject *executor_list_head;
+    bool executor_compact;
 
     _rare_events rare_events;
     PyDict_WatchCallback builtins_dict_watcher;

diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
@@ -31,7 +31,8 @@ typedef struct {
     uint8_t oparg;
     uint16_t valid:1;
     uint16_t linked:1;
-    uint16_t chain_depth:14;  // Must be big engough for MAX_CHAIN_DEPTH - 1.
+    uint16_t compact:1;
+    uint16_t chain_depth:13;  // Must be big engough for MAX_CHAIN_DEPTH - 1.
     int index;           // Index of ENTER_EXECUTOR (if code isn't NULL, below).
     _PyBloomFilter bloom;
     _PyExecutorLinkListNode links;

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
@@ -4735,6 +4735,10 @@ dummy_func(
                         tstate->previous_executor = (PyObject *)current_executor;
                         GOTO_TIER_ONE(target);
                     }
+                    if (current_executor->vm_data.chain_depth == 0) {
+                        current_executor->vm_data.compact = true;
+                        tstate->interp->executor_compact = true;
+                    }
                 }
                 exit->executor = executor;
             }

diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c
@@ -1286,7 +1286,10 @@ _Py_HandlePending(PyThreadState *tstate)
     /* GC scheduled to run */
     if ((breaker & _PY_GC_SCHEDULED_BIT) != 0) {
         _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT);
-        _Py_Executors_Compact(tstate->interp);
+        if (tstate->interp->executor_compact) {
+            tstate->interp->executor_compact = false;
+            _Py_Executors_Compact(tstate->interp);
+        }
         _Py_RunGC(tstate);
     }
 

diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
diff --git a/Python/optimizer.c b/Python/optimizer.c
@@ -201,6 +201,7 @@ _PyOptimizer_Optimize(
         (*executor_ptr)->vm_data.code = NULL;
     }
     (*executor_ptr)->vm_data.chain_depth = chain_depth;
+    (*executor_ptr)->vm_data.compact = false;
     assert((*executor_ptr)->vm_data.valid);
     return 1;
 }
@@ -257,6 +258,9 @@ uop_dealloc(_PyExecutorObject *self) {
     _PyObject_GC_UNTRACK(self);
     assert(self->vm_data.code == NULL);
     unlink_executor(self);
+    for (uint32_t i = 0; i < self->exit_count; i++) {
+        Py_XDECREF(self->exits[i].executor);
+    }
 #ifdef _Py_JIT
     _PyJIT_Free(self);
 #endif
@@ -1229,6 +1233,8 @@ static _PyExecutorObject *
 compact_executor(_PyExecutorObject *root)
 {
     assert(root->vm_data.chain_depth == 0);
+    assert(root->vm_data.compact);
+    root->vm_data.compact = false;
     _PyBloomFilter dependencies;
     _Py_BloomFilter_Init(&dependencies);
     // Perform a breadth-first search to find all of the executors we want to compact:
@@ -1290,7 +1296,12 @@ compact_executor(_PyExecutorObject *root)
     if (nexecutors == 1) {
         return (_PyExecutorObject *)Py_NewRef(root);
     }
-    return make_executor_from_uops(buffer, length, &dependencies);
+    _PyExecutorObject *new = make_executor_from_uops(buffer, length, &dependencies);
+    if (new) {
+        new->vm_data.chain_depth = 0;
+        new->vm_data.compact = false;
+    }
+    return new;
 }
 
 int
@@ -1307,7 +1318,7 @@ _Py_Executors_Compact(PyInterpreterState *interp)
     for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) {
         assert(exec->vm_data.valid);
         _PyExecutorObject *next = exec->vm_data.links.next;
-        if (exec->vm_data.code && PyList_Append(compact, (PyObject *)exec)) {
+        if (exec->vm_data.compact && PyList_Append(compact, (PyObject *)exec)) {
             Py_DECREF(compact);
             return -1;
         }
@@ -1328,7 +1339,6 @@ _Py_Executors_Compact(PyInterpreterState *interp)
             int index = instruction->op.arg;
             assert(code->co_executors->executors[index] == old);
             insert_executor(code, instruction, index, new);
-            new->vm_data.chain_depth = 0;
             executor_clear(old);
         }
         Py_DECREF(new);

diff --git a/Python/pystate.c b/Python/pystate.c
@@ -660,6 +660,7 @@ init_interpreter(PyInterpreterState *interp,
 #ifdef _Py_TIER2
     (void)_Py_SetOptimizer(interp, NULL);
     interp->executor_list_head = NULL;
+    interp->executor_compact = false;
 #endif
     if (interp != &runtime->_main_interpreter) {
         /* Fix the self-referential, statically initialized fields. */