Merge #59

59: Moved TokenSet into it's own file. r=matklad a=Plasticcaz

As discussed in Issue #11, the only thing left in that issue that hasn't been fixed appears to be that TokenSet is not in it's own file. This pull request pulls TokenSet, it's macros and it's test into it's own file.

Co-authored-by: Zac Winter <plasticcaz@gmail.com>

crates/libsyntax2/src/grammar/mod.rs

@@ -32,7 +32,8 @@ mod type_params;
 mod types;
 
 use {
-    parser_api::{Marker, CompletedMarker, Parser, TokenSet},
+    token_set::TokenSet,
+    parser_api::{Marker, CompletedMarker, Parser},
     SyntaxKind::{self, *},
 };
 pub(crate) use self::{

crates/libsyntax2/src/lib.rs

@@ -31,6 +31,7 @@ pub mod algo;
 pub mod ast;
 mod lexer;
 #[macro_use]
+mod token_set;
 mod parser_api;
 mod grammar;
 mod parser_impl;

crates/libsyntax2/src/parser_api.rs

@@ -1,45 +1,10 @@
 use {
+    token_set::TokenSet,
     parser_impl::ParserImpl,
     SyntaxKind::{self, ERROR},
     drop_bomb::DropBomb,
 };
 
-#[derive(Clone, Copy)]
-pub(crate) struct TokenSet(pub(crate) u128);
-
-fn mask(kind: SyntaxKind) -> u128 {
-    1u128 << (kind as usize)
-}
-
-impl TokenSet {
-    pub const EMPTY: TokenSet = TokenSet(0);
-
-    pub fn contains(&self, kind: SyntaxKind) -> bool {
-        self.0 & mask(kind) != 0
-    }
-}
-
-#[macro_export]
-macro_rules! token_set {
-    ($($t:ident),*) => { TokenSet($(1u128 << ($t as usize))|*) };
-    ($($t:ident),* ,) => { token_set!($($t),*) };
-}
-
-#[macro_export]
-macro_rules! token_set_union {
-    ($($ts:expr),*) => { TokenSet($($ts.0)|*) };
-    ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
-}
-
-#[test]
-fn token_set_works_for_tokens() {
-    use SyntaxKind::*;
-    let ts = token_set! { EOF, SHEBANG };
-    assert!(ts.contains(EOF));
-    assert!(ts.contains(SHEBANG));
-    assert!(!ts.contains(PLUS));
-}
-
 /// `Parser` struct provides the low-level API for
 /// navigating through the stream of tokens and
 /// constructing the parse tree. The actual parsing

crates/libsyntax2/src/token_set.rs

@@ -0,0 +1,37 @@
+use SyntaxKind;
+
+#[derive(Clone, Copy)]
+pub(crate) struct TokenSet(pub(crate) u128);
+
+fn mask(kind: SyntaxKind) -> u128 {
+    1u128 << (kind as usize)
+}
+
+impl TokenSet {
+    pub const EMPTY: TokenSet = TokenSet(0);
+
+    pub fn contains(&self, kind: SyntaxKind) -> bool {
+        self.0 & mask(kind) != 0
+    }
+}
+
+#[macro_export]
+macro_rules! token_set {
+    ($($t:ident),*) => { TokenSet($(1u128 << ($t as usize))|*) };
+    ($($t:ident),* ,) => { token_set!($($t),*) };
+}
+
+#[macro_export]
+macro_rules! token_set_union {
+    ($($ts:expr),*) => { TokenSet($($ts.0)|*) };
+    ($($ts:expr),* ,) => { token_set_union!($($ts),*) };
+}
+
+#[test]
+fn token_set_works_for_tokens() {
+    use SyntaxKind::*;
+    let ts = token_set! { EOF, SHEBANG };
+    assert!(ts.contains(EOF));
+    assert!(ts.contains(SHEBANG));
+    assert!(!ts.contains(PLUS));
+}