Add signal support

We check for signals on most calls to runtime functions in the llvm tier and the bjit and inside the interpreter on some additional places,
because checking on every bytecode is a too large slowdown (>10%).

from_cpython/Lib/test/test_signal.py

@@ -1,4 +1,3 @@
-# expected: fail
 import unittest
 from test import test_support
 from contextlib import closing
@@ -80,7 +79,8 @@ def run_test(self):
         # don't worry about re-setting the default handlers.
         signal.signal(signal.SIGHUP, self.handlerA)
         signal.signal(signal.SIGUSR1, self.handlerB)
-        signal.signal(signal.SIGUSR2, signal.SIG_IGN)
+        # Pyston change: pyston uses SIGUSR2 internally
+        # signal.signal(signal.SIGUSR2, signal.SIG_IGN)
         signal.signal(signal.SIGALRM, signal.default_int_handler)
 
         # Variables the signals will modify:
@@ -117,9 +117,11 @@ def run_test(self):
             if test_support.verbose:
                 print "HandlerBCalled exception caught"
 
-        child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)])
-        if child:
-            self.wait(child)  # Nothing should happen.
+
+        # Pyston change: pyston uses SIGUSR2 internally
+        # child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)])
+        # if child:
+        #     self.wait(child)  # Nothing should happen.
 
         try:
             signal.alarm(1)

from_cpython/Modules/signalmodule.c

@@ -626,6 +626,9 @@ initsignal(void)
         old_siginthandler = PyOS_setsig(SIGINT, signal_handler);
     }
 
+    // Pyston change: let the GC scan the handlers
+    PyGC_AddPotentialRoot(Handlers, sizeof(Handlers));
+
 #ifdef SIGHUP
     x = PyInt_FromLong(SIGHUP);
     PyDict_SetItemString(d, "SIGHUP", x);
@@ -895,10 +898,6 @@ PyErr_CheckSignals(void)
     if (!is_tripped)
         return 0;
 
-    // Pyston change:
-    Py_FatalError("TODO");
-
-#if 0
     int i;
     PyObject *f;
 
@@ -943,7 +942,6 @@ PyErr_CheckSignals(void)
             Py_DECREF(result);
         }
     }
-#endif
 
     return 0;
 }

src/codegen/ast_interpreter.cpp

@@ -916,43 +916,66 @@ Value ASTInterpreter::visit_stmt(AST_stmt* node) {
         printf("\n");
     }
 
+    Value rtn;
     switch (node->type) {
         case AST_TYPE::Assert:
-            return visit_assert((AST_Assert*)node);
+            rtn = visit_assert((AST_Assert*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Assign:
-            return visit_assign((AST_Assign*)node);
+            rtn = visit_assign((AST_Assign*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Delete:
-            return visit_delete((AST_Delete*)node);
+            rtn = visit_delete((AST_Delete*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Exec:
-            return visit_exec((AST_Exec*)node);
+            rtn = visit_exec((AST_Exec*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Expr:
             // docstrings are str constant expression statements.
             // ignore those while interpreting.
-            if ((((AST_Expr*)node)->value)->type != AST_TYPE::Str)
-                return visit_expr((AST_Expr*)node);
+            if ((((AST_Expr*)node)->value)->type != AST_TYPE::Str) {
+                rtn = visit_expr((AST_Expr*)node);
+                ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            }
             break;
         case AST_TYPE::Pass:
-            return Value(); // nothing todo
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break; // nothing todo
         case AST_TYPE::Print:
-            return visit_print((AST_Print*)node);
+            rtn = visit_print((AST_Print*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Raise:
-            return visit_raise((AST_Raise*)node);
+            rtn = visit_raise((AST_Raise*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Return:
-            return visit_return((AST_Return*)node);
+            rtn = visit_return((AST_Return*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
         case AST_TYPE::Global:
-            return visit_global((AST_Global*)node);
+            rtn = visit_global((AST_Global*)node);
+            ASTInterpreterJitInterface::pendingCallsCheckHelper();
+            break;
 
         // pseudo
         case AST_TYPE::Branch:
-            return visit_branch((AST_Branch*)node);
+            rtn = visit_branch((AST_Branch*)node);
+            break;
         case AST_TYPE::Jump:
-            return visit_jump((AST_Jump*)node);
+            rtn = visit_jump((AST_Jump*)node);
+            break;
         case AST_TYPE::Invoke:
-            return visit_invoke((AST_Invoke*)node);
+            rtn = visit_invoke((AST_Invoke*)node);
+            break;
         default:
             RELEASE_ASSERT(0, "not implemented");
     };
-    return Value();
+    return rtn;
 }
 
 Value ASTInterpreter::visit_return(AST_Return* node) {
@@ -1646,6 +1669,11 @@ Box* ASTInterpreterJitInterface::landingpadHelper(void* _interpreter) {
     return rtn;
 }
 
+void ASTInterpreterJitInterface::pendingCallsCheckHelper() {
+    if (unlikely(_pendingcalls_to_do))
+        makePendingCalls();
+}
+
 Box* ASTInterpreterJitInterface::setExcInfoHelper(void* _interpreter, Box* type, Box* value, Box* traceback) {
     ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
     interpreter->getFrameInfo()->exc = ExcInfo(type, value, traceback);

src/codegen/ast_interpreter.h

@@ -45,6 +45,7 @@ struct ASTInterpreterJitInterface {
     static Box* derefHelper(void* interp, InternedString s);
     static Box* doOSRHelper(void* interp, AST_Jump* node);
     static Box* landingpadHelper(void* interp);
+    static void pendingCallsCheckHelper();
     static Box* setExcInfoHelper(void* interp, Box* type, Box* value, Box* traceback);
     static void setLocalClosureHelper(void* interp, long vreg, InternedString id, Box* v);
     static Box* uncacheExcInfoHelper(void* interp);

src/codegen/baseline_jit.cpp

@@ -471,6 +471,10 @@ void JitFragmentWriter::emitOSRPoint(AST_Jump* node) {
     addAction([=]() { _emitOSRPoint(result, node_var); }, { result, node_var, getInterp() }, ActionType::NORMAL);
 }
 
+void JitFragmentWriter::emitPendingCallsCheck() {
+    call(false, (void*)ASTInterpreterJitInterface::pendingCallsCheckHelper);
+}
+
 void JitFragmentWriter::emitPrint(RewriterVar* dest, RewriterVar* var, bool nl) {
     if (!dest)
         dest = call(false, (void*)getSysStdout);
@@ -693,12 +697,17 @@ RewriterVar* JitFragmentWriter::emitPPCall(void* func_addr, llvm::ArrayRef<Rewri
         RewriterVar* obj_cls_var = result->getAttr(offsetof(Box, cls));
         addAction([=]() { _emitRecordType(type_recorder_var, obj_cls_var); }, { type_recorder_var, obj_cls_var },
                   ActionType::NORMAL);
+
+        emitPendingCallsCheck();
         return result;
     }
+    emitPendingCallsCheck();
     return result;
 #else
     assert(args_vec.size() < 7);
-    return call(false, func_addr, args_vec);
+    RewriterVar* result = call(false, func_addr, args_vec);
+    emitPendingCallsCheck();
+    return result;
 #endif
 }
 

src/codegen/baseline_jit.h

@@ -246,6 +246,7 @@ class JitFragmentWriter : public Rewriter {
     void emitExec(RewriterVar* code, RewriterVar* globals, RewriterVar* locals, FutureFlags flags);
     void emitJump(CFGBlock* b);
     void emitOSRPoint(AST_Jump* node);
+    void emitPendingCallsCheck();
     void emitPrint(RewriterVar* dest, RewriterVar* var, bool nl);
     void emitRaise0();
     void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);

src/codegen/entry.cpp

@@ -378,16 +378,6 @@ static void handle_sigprof_investigate_stattimer(int signum) {
 }
 #endif
 
-static void handle_sigint(int signum) {
-    assert(signum == SIGINT);
-    // TODO: this should set a flag saying a KeyboardInterrupt is pending.
-    // For now, just call abort(), so that we get a traceback at least.
-    fprintf(stderr, "SIGINT!\n");
-    joinRuntime();
-    Stats::dump(false);
-    abort();
-}
-
 void initCodegen() {
     llvm::InitializeNativeTarget();
     llvm::InitializeNativeTargetAsmPrinter();
@@ -481,9 +471,8 @@ void initCodegen() {
 
     setupRuntime();
 
-    // signal(SIGFPE, &handle_sigfpe);
-    signal(SIGUSR1, &handle_sigusr1);
-    signal(SIGINT, &handle_sigint);
+// signal(SIGFPE, &handle_sigfpe);
+// signal(SIGUSR1, &handle_sigusr1);
 
 #if ENABLE_SAMPLING_PROFILER
     struct itimerval prof_timer;

src/codegen/irgen/irgenerator.cpp

@@ -339,8 +339,54 @@ class IREmitterImpl : public IREmitter {
     llvm::BasicBlock*& curblock;
     IRGenerator* irgenerator;
 
+    void emitPendingCallsCheck(AST_stmt* stmt, llvm::BasicBlock* exc_dest) {
+        auto&& builder = *getBuilder();
+
+        llvm::GlobalVariable* pendingcalls_to_do_gv = g.cur_module->getGlobalVariable("_pendingcalls_to_do");
+        if (!pendingcalls_to_do_gv) {
+            static_assert(sizeof(_pendingcalls_to_do) == 4, "");
+            pendingcalls_to_do_gv = new llvm::GlobalVariable(
+                *g.cur_module, g.i32, false, llvm::GlobalValue::ExternalLinkage, 0, "_pendingcalls_to_do");
+            pendingcalls_to_do_gv->setAlignment(4);
+        }
+
+        llvm::BasicBlock* cur_block = builder.GetInsertBlock();
+
+        llvm::BasicBlock* pendingcalls_set = createBasicBlock("_pendingcalls_set");
+        pendingcalls_set->moveAfter(cur_block);
+        llvm::BasicBlock* join_block = createBasicBlock("continue_after_pendingcalls_check");
+        join_block->moveAfter(pendingcalls_set);
+
+        llvm::Value* pendingcalls_to_do_val = builder.CreateLoad(pendingcalls_to_do_gv, true /* volatile */);
+        llvm::Value* is_zero
+            = builder.CreateICmpEQ(pendingcalls_to_do_val, getConstantInt(0, pendingcalls_to_do_val->getType()));
+
+        llvm::Metadata* md_vals[]
+            = { llvm::MDString::get(g.context, "branch_weights"), llvm::ConstantAsMetadata::get(getConstantInt(1000)),
+                llvm::ConstantAsMetadata::get(getConstantInt(1)) };
+        llvm::MDNode* branch_weights = llvm::MDNode::get(g.context, llvm::ArrayRef<llvm::Metadata*>(md_vals));
+
+
+        builder.CreateCondBr(is_zero, join_block, pendingcalls_set, branch_weights);
+        {
+            setCurrentBasicBlock(pendingcalls_set);
+
+            emitSetCurrentStmt(stmt);
+            if (exc_dest) {
+                builder.CreateInvoke(g.funcs.makePendingCalls, join_block, exc_dest);
+            } else {
+                builder.CreateCall(g.funcs.makePendingCalls);
+                builder.CreateBr(join_block);
+            }
+        }
+
+        cur_block = join_block;
+        setCurrentBasicBlock(join_block);
+    }
+
     llvm::CallSite emitCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
                             ExceptionStyle target_exception_style) {
+
         if (target_exception_style == CXX && (unw_info.hasHandler() || irstate->getExceptionStyle() == CAPI)) {
             // Create the invoke:
             llvm::BasicBlock* normal_dest
@@ -363,9 +409,12 @@ class IREmitterImpl : public IREmitter {
             // Normal case:
             getBuilder()->SetInsertPoint(normal_dest);
             curblock = normal_dest;
+            emitPendingCallsCheck(unw_info.current_stmt, exc_dest);
             return rtn;
         } else {
             llvm::CallInst* cs = getBuilder()->CreateCall(callee, args);
+            if (target_exception_style == CXX)
+                emitPendingCallsCheck(unw_info.current_stmt, NULL);
             return cs;
         }
     }
@@ -468,6 +517,15 @@ class IREmitterImpl : public IREmitter {
         return llvm::BasicBlock::Create(g.context, name, irstate->getLLVMFunction());
     }
 
+    // Our current frame introspection approach requires that we update the currently executed stmt before doing a call
+    // to a function which could throw an exception, inspect the python call frame,...
+    // Only patchpoint don't need to set the current statement because the stmt will be inluded in the stackmap args.
+    void emitSetCurrentStmt(AST_stmt* stmt) {
+        getBuilder()->CreateStore(stmt ? embedRelocatablePtr(stmt, g.llvm_aststmt_type_ptr)
+                                       : getNullPtr(g.llvm_aststmt_type_ptr),
+                                  irstate->getStmtVar());
+    }
+
     llvm::Value* createCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
                             ExceptionStyle target_exception_style = CXX) override {
 #ifndef NDEBUG
@@ -487,10 +545,7 @@ class IREmitterImpl : public IREmitter {
             }
         }
 #endif
-
-        llvm::Value* stmt = unw_info.current_stmt ? embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr)
-                                                  : getNullPtr(g.llvm_aststmt_type_ptr);
-        getBuilder()->CreateStore(stmt, irstate->getStmtVar());
+        emitSetCurrentStmt(unw_info.current_stmt);
         return emitCall(unw_info, callee, args, target_exception_style).getInstruction();
     }
 
@@ -2881,9 +2936,10 @@ class IRGeneratorImpl : public IRGenerator {
     }
 
     void doSafePoint(AST_stmt* next_statement) override {
-        // Unwind info is always needed in allowGLReadPreemption if it has any chance of
-        // running arbitrary code like finalizers.
-        emitter.createCall(UnwindInfo(next_statement, NULL), g.funcs.allowGLReadPreemption);
+        // We need to setup frame introspection by updating the current stmt because we can run can run arbitrary code
+        // like finalizers inside allowGLReadPreemption.
+        emitter.emitSetCurrentStmt(next_statement);
+        emitter.getBuilder()->CreateCall(g.funcs.allowGLReadPreemption);
     }
 
     // Create a (or reuse an existing) block that will catch a CAPI exception, and then forward

src/codegen/runtime_hooks.cpp

@@ -198,6 +198,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(createClosure);
     GET(createGenerator);
     GET(createSet);
+    GET(makePendingCalls);
 
     GET(getattr);
     GET(getattr_capi);

src/codegen/runtime_hooks.h

@@ -34,7 +34,7 @@ struct GlobalFuncs {
 
     llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *createFunctionFromMetadata, *getFunctionMetadata,
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
-        *createUserClass, *createClosure, *createGenerator, *createSet;
+        *createUserClass, *createClosure, *createGenerator, *createSet, *makePendingCalls;
     llvm::Value* getattr, *getattr_capi, *setattr, *delattr, *delitem, *delGlobal, *nonzero, *binop, *compare,
         *augbinop, *unboxedLen, *getitem, *getitem_capi, *getclsattr, *getGlobal, *setitem, *unaryop, *import,
         *importFrom, *importStar, *repr, *exceptionMatches, *yield, *getiterHelper, *hasnext, *setGlobal, *apply_slice;

src/core/threading.cpp

@@ -497,9 +497,13 @@ static void* find_stack() {
     return NULL; /* not found =^P */
 }
 
+static long main_thread_id;
+
 void registerMainThread() {
     LOCK_REGION(&threading_lock);
 
+    main_thread_id = pthread_self();
+
     assert(!current_internal_thread_state);
     current_internal_thread_state = new ThreadStateInternal(find_stack(), pthread_self(), &cur_thread_state);
     current_threads[pthread_self()] = current_internal_thread_state;
@@ -524,6 +528,10 @@ void finishMainThread() {
     // TODO maybe this is the place to wait for non-daemon threads?
 }
 
+bool isMainThread() {
+    return pthread_self() == main_thread_id;
+}
+
 
 // For the "AllowThreads" regions, let's save the thread state at the beginning of the region.
 // This means that the thread won't get interrupted by the signals we would otherwise need to

src/core/threading.h

@@ -51,6 +51,8 @@ intptr_t start_thread(void* (*start_func)(Box*, Box*, Box*), Box* arg1, Box* arg
 void registerMainThread();
 void finishMainThread();
 
+bool isMainThread();
+
 // Hook for the GC; will visit all the threads (including the current one), visiting their
 // stacks and thread-local PyThreadState objects
 void visitAllStacks(gc::GCVisitor* v);