Skip to content

Commit

Permalink
enable CPython's _ast module and use cpython's parser in builtin
Browse files Browse the repository at this point in the history 
function compile
  • Loading branch information
@aisk
aisk committed Jan 12, 2016
1 parent 2526c94 commit 096af7e
Show file tree
Hide file tree
Showing 13 changed files with 283 additions and 366 deletions.
74 changes: 35 additions & 39 deletions from_cpython/Python/Python-ast.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,7 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){sOss}",
type, base, "_fields", fnames, "__module__", "_ast");
Py_DECREF(fnames);
PyGC_AddRoot(result);
return (PyTypeObject*)result;
}

Expand Down Expand Up @@ -789,27 +790,27 @@ static int init_types(void)
if (!add_attributes(expr_context_type, NULL, 0)) return 0;
Load_type = make_type("Load", expr_context_type, NULL, 0);
if (!Load_type) return 0;
Load_singleton = PyType_GenericNew(Load_type, NULL, NULL);
Load_singleton = PyGC_AddRoot(PyType_GenericNew(Load_type, NULL, NULL));
if (!Load_singleton) return 0;
Store_type = make_type("Store", expr_context_type, NULL, 0);
if (!Store_type) return 0;
Store_singleton = PyType_GenericNew(Store_type, NULL, NULL);
Store_singleton = PyGC_AddRoot(PyType_GenericNew(Store_type, NULL, NULL));
if (!Store_singleton) return 0;
Del_type = make_type("Del", expr_context_type, NULL, 0);
if (!Del_type) return 0;
Del_singleton = PyType_GenericNew(Del_type, NULL, NULL);
Del_singleton = PyGC_AddRoot(PyType_GenericNew(Del_type, NULL, NULL));
if (!Del_singleton) return 0;
AugLoad_type = make_type("AugLoad", expr_context_type, NULL, 0);
if (!AugLoad_type) return 0;
AugLoad_singleton = PyType_GenericNew(AugLoad_type, NULL, NULL);
AugLoad_singleton = PyGC_AddRoot(PyType_GenericNew(AugLoad_type, NULL, NULL));
if (!AugLoad_singleton) return 0;
AugStore_type = make_type("AugStore", expr_context_type, NULL, 0);
if (!AugStore_type) return 0;
AugStore_singleton = PyType_GenericNew(AugStore_type, NULL, NULL);
AugStore_singleton = PyGC_AddRoot(PyType_GenericNew(AugStore_type, NULL, NULL));
if (!AugStore_singleton) return 0;
Param_type = make_type("Param", expr_context_type, NULL, 0);
if (!Param_type) return 0;
Param_singleton = PyType_GenericNew(Param_type, NULL, NULL);
Param_singleton = PyGC_AddRoot(PyType_GenericNew(Param_type, NULL, NULL));
if (!Param_singleton) return 0;
slice_type = make_type("slice", &AST_type, NULL, 0);
if (!slice_type) return 0;
Expand All @@ -827,124 +828,124 @@ static int init_types(void)
if (!add_attributes(boolop_type, NULL, 0)) return 0;
And_type = make_type("And", boolop_type, NULL, 0);
if (!And_type) return 0;
And_singleton = PyType_GenericNew(And_type, NULL, NULL);
And_singleton = PyGC_AddRoot(PyType_GenericNew(And_type, NULL, NULL));
if (!And_singleton) return 0;
Or_type = make_type("Or", boolop_type, NULL, 0);
if (!Or_type) return 0;
Or_singleton = PyType_GenericNew(Or_type, NULL, NULL);
Or_singleton = PyGC_AddRoot(PyType_GenericNew(Or_type, NULL, NULL));
if (!Or_singleton) return 0;
operator_type = make_type("operator", &AST_type, NULL, 0);
if (!operator_type) return 0;
if (!add_attributes(operator_type, NULL, 0)) return 0;
Add_type = make_type("Add", operator_type, NULL, 0);
if (!Add_type) return 0;
Add_singleton = PyType_GenericNew(Add_type, NULL, NULL);
Add_singleton = PyGC_AddRoot(PyType_GenericNew(Add_type, NULL, NULL));
if (!Add_singleton) return 0;
Sub_type = make_type("Sub", operator_type, NULL, 0);
if (!Sub_type) return 0;
Sub_singleton = PyType_GenericNew(Sub_type, NULL, NULL);
Sub_singleton = PyGC_AddRoot(PyType_GenericNew(Sub_type, NULL, NULL));
if (!Sub_singleton) return 0;
Mult_type = make_type("Mult", operator_type, NULL, 0);
if (!Mult_type) return 0;
Mult_singleton = PyType_GenericNew(Mult_type, NULL, NULL);
Mult_singleton = PyGC_AddRoot(PyType_GenericNew(Mult_type, NULL, NULL));
if (!Mult_singleton) return 0;
Div_type = make_type("Div", operator_type, NULL, 0);
if (!Div_type) return 0;
Div_singleton = PyType_GenericNew(Div_type, NULL, NULL);
Div_singleton = PyGC_AddRoot(PyType_GenericNew(Div_type, NULL, NULL));
if (!Div_singleton) return 0;
Mod_type = make_type("Mod", operator_type, NULL, 0);
if (!Mod_type) return 0;
Mod_singleton = PyType_GenericNew(Mod_type, NULL, NULL);
Mod_singleton = PyGC_AddRoot(PyType_GenericNew(Mod_type, NULL, NULL));
if (!Mod_singleton) return 0;
Pow_type = make_type("Pow", operator_type, NULL, 0);
if (!Pow_type) return 0;
Pow_singleton = PyType_GenericNew(Pow_type, NULL, NULL);
Pow_singleton = PyGC_AddRoot(PyType_GenericNew(Pow_type, NULL, NULL));
if (!Pow_singleton) return 0;
LShift_type = make_type("LShift", operator_type, NULL, 0);
if (!LShift_type) return 0;
LShift_singleton = PyType_GenericNew(LShift_type, NULL, NULL);
LShift_singleton = PyGC_AddRoot(PyType_GenericNew(LShift_type, NULL, NULL));
if (!LShift_singleton) return 0;
RShift_type = make_type("RShift", operator_type, NULL, 0);
if (!RShift_type) return 0;
RShift_singleton = PyType_GenericNew(RShift_type, NULL, NULL);
RShift_singleton = PyGC_AddRoot(PyType_GenericNew(RShift_type, NULL, NULL));
if (!RShift_singleton) return 0;
BitOr_type = make_type("BitOr", operator_type, NULL, 0);
if (!BitOr_type) return 0;
BitOr_singleton = PyType_GenericNew(BitOr_type, NULL, NULL);
BitOr_singleton = PyGC_AddRoot(PyType_GenericNew(BitOr_type, NULL, NULL));
if (!BitOr_singleton) return 0;
BitXor_type = make_type("BitXor", operator_type, NULL, 0);
if (!BitXor_type) return 0;
BitXor_singleton = PyType_GenericNew(BitXor_type, NULL, NULL);
BitXor_singleton = PyGC_AddRoot(PyType_GenericNew(BitXor_type, NULL, NULL));
if (!BitXor_singleton) return 0;
BitAnd_type = make_type("BitAnd", operator_type, NULL, 0);
if (!BitAnd_type) return 0;
BitAnd_singleton = PyType_GenericNew(BitAnd_type, NULL, NULL);
BitAnd_singleton = PyGC_AddRoot(PyType_GenericNew(BitAnd_type, NULL, NULL));
if (!BitAnd_singleton) return 0;
FloorDiv_type = make_type("FloorDiv", operator_type, NULL, 0);
if (!FloorDiv_type) return 0;
FloorDiv_singleton = PyType_GenericNew(FloorDiv_type, NULL, NULL);
FloorDiv_singleton = PyGC_AddRoot(PyType_GenericNew(FloorDiv_type, NULL, NULL));
if (!FloorDiv_singleton) return 0;
unaryop_type = make_type("unaryop", &AST_type, NULL, 0);
if (!unaryop_type) return 0;
if (!add_attributes(unaryop_type, NULL, 0)) return 0;
Invert_type = make_type("Invert", unaryop_type, NULL, 0);
if (!Invert_type) return 0;
Invert_singleton = PyType_GenericNew(Invert_type, NULL, NULL);
Invert_singleton = PyGC_AddRoot(PyType_GenericNew(Invert_type, NULL, NULL));
if (!Invert_singleton) return 0;
Not_type = make_type("Not", unaryop_type, NULL, 0);
if (!Not_type) return 0;
Not_singleton = PyType_GenericNew(Not_type, NULL, NULL);
Not_singleton = PyGC_AddRoot(PyType_GenericNew(Not_type, NULL, NULL));
if (!Not_singleton) return 0;
UAdd_type = make_type("UAdd", unaryop_type, NULL, 0);
if (!UAdd_type) return 0;
UAdd_singleton = PyType_GenericNew(UAdd_type, NULL, NULL);
UAdd_singleton = PyGC_AddRoot(PyType_GenericNew(UAdd_type, NULL, NULL));
if (!UAdd_singleton) return 0;
USub_type = make_type("USub", unaryop_type, NULL, 0);
if (!USub_type) return 0;
USub_singleton = PyType_GenericNew(USub_type, NULL, NULL);
USub_singleton = PyGC_AddRoot(PyType_GenericNew(USub_type, NULL, NULL));
if (!USub_singleton) return 0;
cmpop_type = make_type("cmpop", &AST_type, NULL, 0);
if (!cmpop_type) return 0;
if (!add_attributes(cmpop_type, NULL, 0)) return 0;
Eq_type = make_type("Eq", cmpop_type, NULL, 0);
if (!Eq_type) return 0;
Eq_singleton = PyType_GenericNew(Eq_type, NULL, NULL);
Eq_singleton = PyGC_AddRoot(PyType_GenericNew(Eq_type, NULL, NULL));
if (!Eq_singleton) return 0;
NotEq_type = make_type("NotEq", cmpop_type, NULL, 0);
if (!NotEq_type) return 0;
NotEq_singleton = PyType_GenericNew(NotEq_type, NULL, NULL);
NotEq_singleton = PyGC_AddRoot(PyType_GenericNew(NotEq_type, NULL, NULL));
if (!NotEq_singleton) return 0;
Lt_type = make_type("Lt", cmpop_type, NULL, 0);
if (!Lt_type) return 0;
Lt_singleton = PyType_GenericNew(Lt_type, NULL, NULL);
Lt_singleton = PyGC_AddRoot(PyType_GenericNew(Lt_type, NULL, NULL));
if (!Lt_singleton) return 0;
LtE_type = make_type("LtE", cmpop_type, NULL, 0);
if (!LtE_type) return 0;
LtE_singleton = PyType_GenericNew(LtE_type, NULL, NULL);
LtE_singleton = PyGC_AddRoot(PyType_GenericNew(LtE_type, NULL, NULL));
if (!LtE_singleton) return 0;
Gt_type = make_type("Gt", cmpop_type, NULL, 0);
if (!Gt_type) return 0;
Gt_singleton = PyType_GenericNew(Gt_type, NULL, NULL);
Gt_singleton = PyGC_AddRoot(PyType_GenericNew(Gt_type, NULL, NULL));
if (!Gt_singleton) return 0;
GtE_type = make_type("GtE", cmpop_type, NULL, 0);
if (!GtE_type) return 0;
GtE_singleton = PyType_GenericNew(GtE_type, NULL, NULL);
GtE_singleton = PyGC_AddRoot(PyType_GenericNew(GtE_type, NULL, NULL));
if (!GtE_singleton) return 0;
Is_type = make_type("Is", cmpop_type, NULL, 0);
if (!Is_type) return 0;
Is_singleton = PyType_GenericNew(Is_type, NULL, NULL);
Is_singleton = PyGC_AddRoot(PyType_GenericNew(Is_type, NULL, NULL));
if (!Is_singleton) return 0;
IsNot_type = make_type("IsNot", cmpop_type, NULL, 0);
if (!IsNot_type) return 0;
IsNot_singleton = PyType_GenericNew(IsNot_type, NULL, NULL);
IsNot_singleton = PyGC_AddRoot(PyType_GenericNew(IsNot_type, NULL, NULL));
if (!IsNot_singleton) return 0;
In_type = make_type("In", cmpop_type, NULL, 0);
if (!In_type) return 0;
In_singleton = PyType_GenericNew(In_type, NULL, NULL);
In_singleton = PyGC_AddRoot(PyType_GenericNew(In_type, NULL, NULL));
if (!In_singleton) return 0;
NotIn_type = make_type("NotIn", cmpop_type, NULL, 0);
if (!NotIn_type) return 0;
NotIn_singleton = PyType_GenericNew(NotIn_type, NULL, NULL);
NotIn_singleton = PyGC_AddRoot(PyType_GenericNew(NotIn_type, NULL, NULL));
if (!NotIn_singleton) return 0;
comprehension_type = make_type("comprehension", &AST_type,
comprehension_fields, 3);
Expand Down Expand Up @@ -6779,13 +6780,8 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
return res;
}

// Pyston temporary change: we're not using this file for the python module, at least not yet.
#if 0
int PyAST_Check(PyObject* obj)
{
init_types();
return PyObject_IsInstance(obj, (PyObject*)&AST_type);
}
#endif


1 change: 0 additions & 1 deletion src/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
gc/gc_alloc.cpp
gc/heap.cpp
runtime/bool.cpp
runtime/builtin_modules/ast.cpp
runtime/builtin_modules/builtins.cpp
runtime/builtin_modules/gc.cpp
runtime/builtin_modules/pyston.cpp
Expand Down
24 changes: 19 additions & 5 deletions src/codegen/cpython_ast.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -633,8 +633,9 @@ class Converter {
raiseSyntaxError("'break' outside loop", stmt->lineno, stmt->col_offset, fn, "", true);
return new AST_Break();
case Continue_kind:
if (loop_depth == 0)
raiseSyntaxError("'continue' not properly in loop", stmt->lineno, stmt->col_offset, fn, "", true);
if (in_finally)
raiseSyntaxError("'continue' not supported inside 'finally' clause", stmt->lineno, stmt->col_offset,
fn, "", true);
if (in_finally)
raiseSyntaxError("'continue' not supported inside 'finally' clause", stmt->lineno, stmt->col_offset,
fn, "", true);
Expand All @@ -651,14 +652,27 @@ class Converter {
return r;
}

AST_Module* convert(mod_ty mod) {
AST* convert(mod_ty mod) {
switch (mod->kind) {
case Module_kind:
case Module_kind: {
AST_Module* rtn = new AST_Module(llvm::make_unique<InternedStringPool>());
assert(!this->pool);
this->pool = rtn->interned_strings.get();
convertAll<stmt_ty>(mod->v.Module.body, rtn->body);
return rtn;
}
case Interactive_kind: {
AST_Module* rtn = new AST_Module(llvm::make_unique<InternedStringPool>());
assert(!this->pool);
this->pool = rtn->interned_strings.get();
convertAll<stmt_ty>(mod->v.Interactive.body, rtn->body);
makeModuleInteractive(rtn);
return rtn;
}
case Expression_kind: {
AST_Expression* rtn = new AST_Expression(llvm::make_unique<InternedStringPool>());
this->pool = rtn->interned_strings.get();
rtn->body = this->convert(mod->v.Expression.body);
return rtn;
}
default:
Expand All @@ -667,7 +681,7 @@ class Converter {
}
};

AST_Module* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn) {
AST* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn) {
Converter c(fn);
return c.convert(mod);
}
Expand Down
2 changes: 1 addition & 1 deletion src/codegen/cpython_ast.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ namespace pyston {
// Convert a CPython ast object to a Pyston ast object.
// This will also check for certain kinds of "syntax errors" (ex continue not in loop) and will
// throw them as C++ exceptions.
AST_Module* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
AST* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
}

#endif
65 changes: 49 additions & 16 deletions src/codegen/irgen/hooks.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,39 +488,72 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
RELEASE_ASSERT(iflags == 0, "");

AST* parsed;
mod_ty mod;

if (PyAST_Check(source)) {
parsed = unboxAst(source);
int mode;
ArenaWrapper arena;
if (type_str->s() == "exec")
mode = 0;
else if (type_str->s() == "eval")
mode = 1;
else if (type_str->s() == "single")
mode = 2;
else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
}
mod = PyAST_obj2mod(source, arena, mode);
if (PyErr_Occurred())
throwCAPIException();
parsed = cpythonToPystonAST(mod, filename_str->c_str());
} else {
RELEASE_ASSERT(PyString_Check(source), "");
llvm::StringRef source_str = static_cast<BoxedString*>(source)->s();

if (type_str->s() == "exec") {
parsed = parseExec(source_str, future_flags);
} else if (type_str->s() == "eval") {
parsed = parseEval(source_str, future_flags);
} else if (type_str->s() == "single") {
parsed = parseExec(source_str, future_flags, true);
} else {
int mode;
if (type_str->s() == "exec")
mode = Py_file_input;
else if (type_str->s() == "eval")
mode = Py_eval_input;
else if (type_str->s() == "single")
mode = Py_single_input;
else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
}

PyCompilerFlags cf;
cf.cf_flags = future_flags;
ArenaWrapper arena;
const char* code = static_cast<BoxedString*>(source)->s().data();
assert(arena);
const char* fn = filename_str->c_str();
mod = PyParser_ASTFromString(code, fn, mode, &cf, arena);
if (!mod)
throwCAPIException();

parsed = cpythonToPystonAST(mod, filename_str->c_str());
}

if (only_ast)
return boxAst(parsed);
if (only_ast) {
Box* result = PyAST_mod2obj(mod);
if (PyErr_Occurred())
throwCAPIException();

return result;
}

PyCompilerFlags pcf;
pcf.cf_flags = future_flags;

FunctionMetadata* md;
if (type_str->s() == "exec" || type_str->s() == "single") {
// TODO: CPython parses execs as Modules
if (parsed->type != AST_TYPE::Module)
raiseExcHelper(TypeError, "expected Module node, got %s", boxAst(parsed)->cls->tp_name);
if (parsed->type != AST_TYPE::Module) {
raiseExcHelper(TypeError, "expected Module node, got %s", AST_TYPE::stringify(parsed->type));
}
md = compileExec(static_cast<AST_Module*>(parsed), filename_str, &pcf);
} else if (type_str->s() == "eval") {
if (parsed->type != AST_TYPE::Expression)
raiseExcHelper(TypeError, "expected Expression node, got %s", boxAst(parsed)->cls->tp_name);
if (parsed->type != AST_TYPE::Expression) {
raiseExcHelper(TypeError, "expected Expression node, got %s", AST_TYPE::stringify(parsed->type));
}
md = compileEval(static_cast<AST_Expression*>(parsed), filename_str, &pcf);
} else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
Expand Down
9 changes: 6 additions & 3 deletions src/codegen/parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1027,7 +1027,8 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
mod_ty mod = PyParser_ASTFromString(code, fn, Py_file_input, &cf, arena);
if (!mod)
throwCAPIException();
auto rtn = cpythonToPystonAST(mod, fn);
assert(mod->kind != Interactive_kind);
auto rtn = static_cast<AST_Module*>(cpythonToPystonAST(mod, fn));
return rtn;
}

Expand Down Expand Up @@ -1072,7 +1073,8 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
if (!mod)
throwCAPIException();
auto rtn = cpythonToPystonAST(mod, fn);
assert(mod->kind != Interactive_kind);
auto rtn = static_cast<AST_Module*>(cpythonToPystonAST(mod, fn));
return rtn;
}

Expand Down Expand Up @@ -1158,7 +1160,8 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
mod_ty mod = PyParser_ASTFromFile(fp, fn, Py_file_input, 0, 0, &cf, NULL, arena);
if (!mod)
throwCAPIException();
module = cpythonToPystonAST(mod, fn);
assert(mod->kind != Interactive_kind);
module = static_cast<AST_Module*>(cpythonToPystonAST(mod, fn));
} else {
module = pypa_parse(fn, inherited_flags);
RELEASE_ASSERT(module, "unknown parse error");
Expand Down
Loading

0 comments on commit 096af7e

Please sign in to comment.