jl_value_t as parser options for improved flexibility

base/meta.jl

@@ -150,19 +150,24 @@ struct ParseError <: Exception
 end
 
 function _jl_parse(text::AbstractString, filename::AbstractString,
-                   pos::Integer, rule::Integer)
+                   pos::Integer; options...)
     if pos < 1 || pos > ncodeunits(text) + 1
         throw(BoundsError(text, pos))
     end
-    # Technically only need pointers to UTF-8 buffers here. For now converting
-    # to a plain String is the easy way to ensure that.
+    # Technically only need pointers to UTF-8 buffers here. Converting to a
+    # plain String is the easy way to ensure that.
     filename = String(filename)
     text = String(text)
+    options = values(options)
+    if length(options) == 1 && haskey(options, :rule)
+        # Hack: Pass rule as a symbol for benefit of jl_fl_parse
+        options = options.rule
+    end
     # Call into the parser which can be replaced globally during bootstrap with
     # jl_set_parser
     ex,pos = ccall(:jl_parse, Any,
-                   (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t, Csize_t, Cint),
-                   text, sizeof(text), filename, sizeof(filename), pos-1, rule)
+                   (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t, Csize_t, Any),
+                   text, sizeof(text), filename, sizeof(filename), pos-1, options)
     # internally, pos is a zero-based byte offset - convert back.
     ex, pos+1
 end
@@ -190,12 +195,7 @@ julia> Meta.parse("x = 3, y = 5", 5)
 function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
                depwarn::Bool=true)
     filename = "none"
-    JL_PARSE_ATOM = 1
-    JL_PARSE_STATEMENT = 2
-    rule = greedy ? JL_PARSE_STATEMENT : JL_PARSE_ATOM
-    # For now, assume all parser warnings are depwarns
-    # TODO: remove parser-depwarn; parser no longer emits warnings.
-    ex, pos = _jl_parse(str, "none", pos, rule)
+    ex, pos = _jl_parse(str, "none", pos; rule=greedy ? :statement : :atom)
     if raise && isa(ex,Expr) && ex.head === :error
         throw(ParseError(ex.args[1]))
     end
@@ -240,13 +240,11 @@ function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
 end
 
 function parseatom(text::AbstractString, pos::Integer; filename="none")
-    JL_PARSE_ATOM = 1
-    return _jl_parse(text, filename, pos, JL_PARSE_ATOM)
+    return _jl_parse(text, filename, pos; rule=:atom)
 end
 
 function parseall(text::AbstractString; filename="none")
-    JL_PARSE_ALL = 3
-    ex,_ = _jl_parse(text, filename, 1, JL_PARSE_ALL)
+    ex,_ = _jl_parse(text, filename, 1; rule=:all)
     return ex
 end
 

src/ast.c

@@ -67,6 +67,7 @@ jl_sym_t *gc_preserve_begin_sym; jl_sym_t *gc_preserve_end_sym;
 jl_sym_t *coverageeffect_sym; jl_sym_t *escape_sym;
 jl_sym_t *aliasscope_sym; jl_sym_t *popaliasscope_sym;
 jl_sym_t *optlevel_sym;
+jl_sym_t *atom_sym; jl_sym_t *statement_sym; jl_sym_t *all_sym;
 
 static uint8_t flisp_system_image[] = {
 #include <julia_flisp.boot.inc>
@@ -402,6 +403,9 @@ void jl_init_common_symbols(void)
     coverageeffect_sym = jl_symbol("code_coverage_effect");
     aliasscope_sym = jl_symbol("aliasscope");
     popaliasscope_sym = jl_symbol("popaliasscope");
+    atom_sym = jl_symbol("atom");
+    statement_sym = jl_symbol("statement");
+    all_sym = jl_symbol("all");
 }
 
 JL_DLLEXPORT void jl_lisp_prompt(void)
@@ -785,16 +789,20 @@ static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v)
 // content to `filename`. Return an svec of (parse_result, final_pos)
 JL_DLLEXPORT jl_value_t *jl_fl_parse(const char* text, size_t text_len,
                                      const char* filename, size_t filename_len,
-                                     size_t offset, int rule)
+                                     size_t offset, jl_value_t *options)
 {
     JL_TIMING(PARSING);
     if (offset > text_len) {
         jl_value_t *textstr = jl_pchar_to_string(text, text_len);
         JL_GC_PUSH1(&textstr);
         jl_bounds_error(textstr, jl_box_long(offset));
     }
-    else if (offset != 0 && rule == JL_PARSE_ALL) {
-        jl_error("Partial parsing not support by top level grammar rule");
+    jl_sym_t *rule = (jl_sym_t*)options;
+    if (rule != atom_sym && rule != statement_sym && rule != all_sym) {
+        jl_error("jl_fl_parse: unrecognized parse options");
+    }
+    if (offset != 0 && rule == all_sym) {
+        jl_error("Parse `all`: offset not supported");
     }
 
     jl_ast_context_t *ctx = jl_ast_ctx_enter();
@@ -803,23 +811,19 @@ JL_DLLEXPORT jl_value_t *jl_fl_parse(const char* text, size_t text_len,
     value_t fl_filename = cvalue_static_cstrn(fl_ctx, filename, filename_len);
     value_t fl_expr;
     size_t pos1 = 0;
-    if (rule == JL_PARSE_ALL) {
+    if (rule == all_sym) {
         value_t e = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")),
                               fl_text, fl_filename);
         fl_expr = e;
         pos1 = e == fl_ctx->FL_EOF ? text_len : 0;
     }
-    else if (rule == JL_PARSE_STATEMENT || rule == JL_PARSE_ATOM) {
-        value_t greedy = rule == JL_PARSE_STATEMENT ? fl_ctx->T : fl_ctx->F;
+    else {
+        value_t greedy = rule == statement_sym ? fl_ctx->T : fl_ctx->F;
         value_t p = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-parse-one")),
                               fl_text, fl_filename, fixnum(offset), greedy);
         fl_expr = car_(p);
         pos1 = tosize(fl_ctx, cdr_(p), "parse");
     }
-    else {
-        jl_ast_ctx_leave(ctx);
-        jl_errorf("Unknown parse rule %d", (int)rule);
-    }
 
     // Convert to julia values
     jl_value_t *expr=NULL, *end_offset=NULL;

src/frontend.c

@@ -20,9 +20,9 @@ JL_DLLEXPORT void jl_set_parser(jl_parse_func_t parser)
 
 JL_DLLEXPORT jl_value_t *jl_parse(const char* text, size_t text_len,
                                   const char* filename, size_t filename_len,
-                                  size_t offset, int rule)
+                                  size_t offset, jl_value_t* options)
 {
-    return (*jl_current_parser)(text, text_len, filename, filename_len, offset, rule);
+    return (*jl_current_parser)(text, text_len, filename, filename_len, offset, options);
 }
 
 // C API
@@ -31,18 +31,17 @@ JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len,
                                       const char *filename, size_t filename_len)
 {
     jl_value_t *p = jl_parse(text, text_len, filename, filename_len,
-                             0, JL_PARSE_ALL);
+                             0, (jl_value_t*)all_sym);
     return jl_svecref(p, 0);
 }
 
 // this is for parsing one expression out of a string, keeping track of
 // the current position.
-// FIXME: Add filename?
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len,
                                          int pos0, int greedy)
 {
     return jl_parse(text, text_len, "none", 4,
-                    pos0, greedy ? JL_PARSE_STATEMENT : JL_PARSE_ATOM);
+                    pos0, greedy ? (jl_value_t*)statement_sym : (jl_value_t*)atom_sym);
 }
 
 // deprecated

src/julia.h

@@ -1600,18 +1600,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist);
 JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods);
 JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz, jl_array_t *depmods);
 
-// front end interface
-typedef enum {
-    JL_PARSE_ATOM      = 1,
-    JL_PARSE_STATEMENT = 2,
-    JL_PARSE_ALL       = 3,
-} jl_parse_rule_t;
-
 // parsing
-JL_DLLEXPORT jl_value_t *jl_parse(const char* text, size_t text_len,
-                                  const char* filename, size_t filename_len,
-                                  size_t offset, int rule);
-// Convenince functions
 JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len,
                                       const char *filename, size_t filename_len);
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,

src/julia_internal.h

@@ -638,13 +638,16 @@ jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs
 
 int jl_has_meta(jl_array_t *body, jl_sym_t *sym);
 
-// Parser replacement
-typedef jl_value_t* (*jl_parse_func_t)(const char*, size_t, const char*, size_t, size_t, int);
+// Experimental parser replacement API
+JL_DLLEXPORT jl_value_t *jl_parse(const char* text, size_t text_len,
+                                  const char* filename, size_t filename_len,
+                                  size_t offset, jl_value_t *options);
+typedef jl_value_t* (*jl_parse_func_t)(const char*, size_t, const char*, size_t, size_t, jl_value_t*);
 JL_DLLEXPORT void jl_set_parser(jl_parse_func_t parser);
 // Builtin flisp parser
 JL_DLLEXPORT jl_value_t *jl_fl_parse(const char* text, size_t text_len,
                                      const char* filename, size_t filename_len,
-                                     size_t offset, int rule);
+                                     size_t offset, jl_value_t *options);
 
 //--------------------------------------------------
 // Backtraces
@@ -1174,6 +1177,7 @@ extern jl_sym_t *gc_preserve_begin_sym; extern jl_sym_t *gc_preserve_end_sym;
 extern jl_sym_t *failed_sym; extern jl_sym_t *done_sym; extern jl_sym_t *runnable_sym;
 extern jl_sym_t *coverageeffect_sym; extern jl_sym_t *escape_sym;
 extern jl_sym_t *optlevel_sym;
+extern jl_sym_t *atom_sym; extern jl_sym_t *statement_sym; extern jl_sym_t *all_sym;
 
 struct _jl_sysimg_fptrs_t;