Proof-of-concept SyntaxKind as enum

src/bin/gen.rs

@@ -29,8 +29,10 @@ impl Grammar {
 
     fn to_syntax_kinds(&self) -> String {
         let mut acc = String::new();
-        acc.push_str("// Generated from grammar.ron\n");
-        acc.push_str("use tree::{SyntaxKind, SyntaxInfo};\n");
+        acc.push_str("#![allow(bad_style, missing_docs, unreachable_pub)]\n");
+        acc.push_str("#![cfg_attr(rustfmt, rustfmt_skip)]\n");
+        acc.push_str("//! Generated from grammar.ron\n");
+        acc.push_str("use tree::SyntaxInfo;\n");
         acc.push_str("\n");
 
         let syntax_kinds: Vec<String> = self.keywords
@@ -40,41 +42,49 @@ impl Grammar {
             .chain(self.nodes.iter().cloned())
             .collect();
 
-        for (idx, kind) in syntax_kinds.iter().enumerate() {
-            let sname = scream(kind);
-            write!(
-                acc,
-                "pub const {}: SyntaxKind = SyntaxKind({});\n",
-                sname, idx
-            ).unwrap();
+        // enum SyntaxKind
+        acc.push_str("/// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT_DEF`.\n");
+        acc.push_str("#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]\n");
+        acc.push_str("#[repr(u32)]\n");
+        acc.push_str("pub enum SyntaxKind {\n");
+        for kind in syntax_kinds.iter() {
+            write!(acc, "    {},\n", scream(kind)).unwrap();
         }
         acc.push_str("\n");
-        write!(
-            acc,
-            "static INFOS: [SyntaxInfo; {}] = [\n",
-            syntax_kinds.len()
-        ).unwrap();
+        acc.push_str("    TOMBSTONE = !0 - 1,\n");
+        acc.push_str("    EOF = !0,\n");
+        acc.push_str("}\n");
+        acc.push_str("pub(crate) use self::SyntaxKind::*;\n");
+        acc.push_str("\n");
+
+        // fn info
+        acc.push_str("impl SyntaxKind {\n");
+        acc.push_str("    pub(crate) fn info(self) -> &'static SyntaxInfo {\n");
+        acc.push_str("        match self {\n");
         for kind in syntax_kinds.iter() {
             let sname = scream(kind);
             write!(
                 acc,
-                "    SyntaxInfo {{ name: \"{sname}\" }},\n",
+                "            {sname} => &SyntaxInfo {{ name: \"{sname}\" }},\n",
                 sname = sname
             ).unwrap();
         }
-        acc.push_str("];\n");
+        acc.push_str("\n");
+        acc.push_str("            TOMBSTONE => &SyntaxInfo { name: \"TOMBSTONE\" },\n");
+        acc.push_str("            EOF => &SyntaxInfo { name: \"EOF\" },\n");
+        acc.push_str("        }\n");
+        acc.push_str("    }\n");
+        acc.push_str("}\n");
         acc.push_str("\n");
 
-        acc.push_str("pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {\n");
-        acc.push_str("    &INFOS[kind.0 as usize]\n");
-        acc.push_str("}\n\n");
+        // fn ident_to_keyword
         acc.push_str("pub(crate) fn ident_to_keyword(ident: &str) -> Option<SyntaxKind> {\n");
-        acc.push_str("   match ident {\n");
+        acc.push_str("    match ident {\n");
         for kw in self.keywords.iter() {
-            write!(acc, "       {:?} => Some({}),\n", kw, kw_token(kw)).unwrap();
+            write!(acc, "        {:?} => Some({}),\n", kw, kw_token(kw)).unwrap();
         }
-        acc.push_str("       _ => None,\n");
-        acc.push_str("   }\n");
+        acc.push_str("        _ => None,\n");
+        acc.push_str("    }\n");
         acc.push_str("}\n");
         acc
     }

src/lexer/mod.rs

@@ -17,6 +17,7 @@ use self::strings::{is_string_literal_start, scan_byte_char_or_string, scan_char
 mod comments;
 use self::comments::{scan_comment, scan_shebang};
 
+/// Break a string up into its component tokens
 pub fn tokenize(text: &str) -> Vec<Token> {
     let mut text = text;
     let mut acc = Vec::new();
@@ -28,6 +29,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
     }
     acc
 }
+/// Get the next token from a string
 pub fn next_token(text: &str) -> Token {
     assert!(!text.is_empty());
     let mut ptr = Ptr::new(text);

src/lib.rs

@@ -1,22 +1,40 @@
+//! An experimental implementation of [Rust RFC#2256 libsyntax2.0][rfc#2256].
+//!
+//! The intent is to be an IDE-ready parser, i.e. one that offers
+//!
+//! - easy and fast incremental re-parsing,
+//! - graceful handling of errors, and
+//! - maintains all information in the source file.
+//!
+//! For more information, see [the RFC][rfc#2265], or [the working draft][RFC.md].
+//!
+//!   [rfc#2256]: <https://github.com/rust-lang/rfcs/pull/2256>
+//!   [RFC.md]: <https://github.com/matklad/libsyntax2/blob/master/docs/RFC.md>
+
+#![forbid(missing_debug_implementations, unconditional_recursion, future_incompatible)]
+#![deny(bad_style, unsafe_code, missing_docs)]
+//#![warn(unreachable_pub)] // rust-lang/rust#47816
+
 extern crate unicode_xid;
 
 mod text;
 mod tree;
 mod lexer;
 mod parser;
 
-#[cfg_attr(rustfmt, rustfmt_skip)]
 pub mod syntax_kinds;
 pub use text::{TextRange, TextUnit};
 pub use tree::{File, FileBuilder, Node, Sink, SyntaxKind, Token};
 pub use lexer::{next_token, tokenize};
 pub use parser::parse;
 
+/// Utilities for simple uses of the parser.
 pub mod utils {
     use std::fmt::Write;
 
     use {File, Node};
 
+    /// Parse a file and create a string representation of the resulting parse tree.
     pub fn dump_tree(file: &File) -> String {
         let mut result = String::new();
         go(file.root(), &mut result, 0);

src/parser/event_parser/grammar/mod.rs

@@ -1,6 +1,5 @@
 use super::parser::{Parser, TokenSet};
 use SyntaxKind;
-use tree::EOF;
 use syntax_kinds::*;
 
 mod items;

src/parser/event_parser/parser.rs

@@ -1,8 +1,7 @@
 use {SyntaxKind, TextUnit, Token};
 use super::Event;
 use super::super::is_insignificant;
-use syntax_kinds::{ERROR, L_CURLY, R_CURLY};
-use tree::{EOF, TOMBSTONE};
+use SyntaxKind::{EOF, ERROR, L_CURLY, R_CURLY, TOMBSTONE};
 
 pub(crate) struct Marker {
     pos: u32,

src/parser/mod.rs

@@ -1,11 +1,11 @@
 use {File, FileBuilder, Sink, SyntaxKind, Token};
 
 use syntax_kinds::*;
-use tree::TOMBSTONE;
 
 mod event_parser;
 use self::event_parser::Event;
 
+/// Parse a sequence of tokens into the representative node tree
 pub fn parse(text: String, tokens: &[Token]) -> File {
     let events = event_parser::parse(&text, tokens);
     from_events_to_file(text, tokens, events)