From afcf259b3ae7f34e913e9b26e7ca28b06c2271c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Tue, 17 Jan 2017 12:33:21 +0100
Subject: [PATCH 1/5] Use less UTF-8 logic when not needed.

---
 src/tokenizer.rs | 153 ++++++++++++++++++++++++-----------------------
 1 file changed, 77 insertions(+), 76 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 3fac405e..faef7e1d 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -380,8 +380,8 @@ impl<'a> Tokenizer<'a> {
     }
 
     #[inline]
-    fn starts_with(&self, needle: &str) -> bool {
-        self.input[self.position..].starts_with(needle)
+    fn starts_with(&self, needle: &[u8]) -> bool {
+        self.input.as_bytes()[self.position..].starts_with(needle)
     }
 }
 
@@ -405,88 +405,88 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
     if tokenizer.is_eof() {
         return None
     }
-    let c = tokenizer.next_char();
+    let c = tokenizer.next_byte_unchecked();
     let token = match c {
-        '\t' | '\n' | ' ' | '\r' | '\x0C' => {
+        b'\t' | b'\n' | b' ' | b'\r' | b'\x0C' => {
             let start_position = tokenizer.position();
             tokenizer.advance(1);
             while !tokenizer.is_eof() {
-                match tokenizer.next_char() {
-                    ' ' | '\t' | '\n' | '\r' | '\x0C' => tokenizer.advance(1),
+                match tokenizer.next_byte_unchecked() {
+                    b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => tokenizer.advance(1),
                     _ => break,
                 }
             }
             WhiteSpace(tokenizer.slice_from(start_position))
         },
-        '"' => consume_string(tokenizer, false),
-        '#' => {
+        b'"' => consume_string(tokenizer, false),
+        b'#' => {
             tokenizer.advance(1);
             if is_ident_start(tokenizer) { IDHash(consume_name(tokenizer)) }
-            else if !tokenizer.is_eof() && match tokenizer.next_char() {
-                'a'...'z' | 'A'...'Z' | '0'...'9' | '-' | '_' => true,
-                '\\' => !tokenizer.has_newline_at(1),
-                _ => c > '\x7F',  // Non-ASCII
+            else if !tokenizer.is_eof() && match tokenizer.next_byte_unchecked() {
+                b'a'...b'z' | b'A'...b'Z' | b'0'...b'9' | b'-' | b'_' => true,
+                b'\\' => !tokenizer.has_newline_at(1),
+                _ => !c.is_ascii(),
             } { Hash(consume_name(tokenizer)) }
-            else { Delim(c) }
+            else { Delim('#') }
         },
-        '$' => {
-            if tokenizer.starts_with("$=") { tokenizer.advance(2); SuffixMatch }
-            else { tokenizer.advance(1); Delim(c) }
+        b'$' => {
+            if tokenizer.starts_with(b"$=") { tokenizer.advance(2); SuffixMatch }
+            else { tokenizer.advance(1); Delim('$') }
         },
-        '\'' => consume_string(tokenizer, true),
-        '(' => { tokenizer.advance(1); ParenthesisBlock },
-        ')' => { tokenizer.advance(1); CloseParenthesis },
-        '*' => {
-            if tokenizer.starts_with("*=") { tokenizer.advance(2); SubstringMatch }
-            else { tokenizer.advance(1); Delim(c) }
+        b'\'' => consume_string(tokenizer, true),
+        b'(' => { tokenizer.advance(1); ParenthesisBlock },
+        b')' => { tokenizer.advance(1); CloseParenthesis },
+        b'*' => {
+            if tokenizer.starts_with(b"*=") { tokenizer.advance(2); SubstringMatch }
+            else { tokenizer.advance(1); Delim('*') }
         },
-        '+' => {
+        b'+' => {
             if (
                 tokenizer.has_at_least(1)
-                && matches!(tokenizer.char_at(1), '0'...'9')
+                && matches!(tokenizer.byte_at(1), b'0'...b'9')
             ) || (
                 tokenizer.has_at_least(2)
-                && tokenizer.char_at(1) == '.'
-                && matches!(tokenizer.char_at(2), '0'...'9')
+                && tokenizer.byte_at(1) == b'.'
+                && matches!(tokenizer.byte_at(2), b'0'...b'9')
             ) {
                 consume_numeric(tokenizer)
             } else {
                 tokenizer.advance(1);
-                Delim(c)
+                Delim('+')
             }
         },
-        ',' => { tokenizer.advance(1); Comma },
-        '-' => {
+        b',' => { tokenizer.advance(1); Comma },
+        b'-' => {
             if (
                 tokenizer.has_at_least(1)
-                && matches!(tokenizer.char_at(1), '0'...'9')
+                && matches!(tokenizer.byte_at(1), b'0'...b'9')
             ) || (
                 tokenizer.has_at_least(2)
-                && tokenizer.char_at(1) == '.'
-                && matches!(tokenizer.char_at(2), '0'...'9')
+                && tokenizer.byte_at(1) == b'.'
+                && matches!(tokenizer.byte_at(2), b'0'...b'9')
             ) {
                 consume_numeric(tokenizer)
-            } else if tokenizer.starts_with("-->") {
+            } else if tokenizer.starts_with(b"-->") {
                 tokenizer.advance(3);
                 CDC
             } else if is_ident_start(tokenizer) {
                 consume_ident_like(tokenizer)
             } else {
                 tokenizer.advance(1);
-                Delim(c)
+                Delim('-')
             }
         },
-        '.' => {
+        b'.' => {
             if tokenizer.has_at_least(1)
-                && matches!(tokenizer.char_at(1), '0'...'9'
+                && matches!(tokenizer.byte_at(1), b'0'...b'9'
             ) {
                 consume_numeric(tokenizer)
             } else {
                 tokenizer.advance(1);
-                Delim(c)
+                Delim('.')
             }
         }
-        '/' if tokenizer.starts_with("/*") => {
+        b'/' if tokenizer.starts_with(b"/*") => {
             tokenizer.advance(2);  // consume "/*"
             let start_position = tokenizer.position();
             let content;
@@ -503,58 +503,59 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
             }
             Comment(content)
         }
-        '0'...'9' => consume_numeric(tokenizer),
-        ':' => { tokenizer.advance(1); Colon },
-        ';' => { tokenizer.advance(1); Semicolon },
-        '<' => {
-            if tokenizer.starts_with("<!--") {
+        b'0'...b'9' => consume_numeric(tokenizer),
+        b':' => { tokenizer.advance(1); Colon },
+        b';' => { tokenizer.advance(1); Semicolon },
+        b'<' => {
+            if tokenizer.starts_with(b"<!--") {
                 tokenizer.advance(4);
                 CDO
             } else {
                 tokenizer.advance(1);
-                Delim(c)
+                Delim('<')
             }
         },
-        '@' => {
+        b'@' => {
             tokenizer.advance(1);
             if is_ident_start(tokenizer) { AtKeyword(consume_name(tokenizer)) }
-            else { Delim(c) }
+            else { Delim('@') }
         },
-        'u' | 'U' => {
+        b'u' | b'U' => {
             if tokenizer.has_at_least(2)
-               && tokenizer.char_at(1) == '+'
-               && matches!(tokenizer.char_at(2), '0'...'9' | 'a'...'f' | 'A'...'F' | '?')
+               && tokenizer.byte_at(1) == b'+'
+               && matches!(tokenizer.byte_at(2), b'0'...b'9' | b'a'...b'f' | b'A'...b'F' | b'?')
             { consume_unicode_range(tokenizer) }
             else { consume_ident_like(tokenizer) }
         },
-        'a'...'z' | 'A'...'Z' | '_' | '\0' => consume_ident_like(tokenizer),
-        '[' => { tokenizer.advance(1); SquareBracketBlock },
-        '\\' => {
+        b'a'...b'z' | b'A'...b'Z' | b'_' | b'\0' => consume_ident_like(tokenizer),
+        b'[' => { tokenizer.advance(1); SquareBracketBlock },
+        b'\\' => {
             if !tokenizer.has_newline_at(1) { consume_ident_like(tokenizer) }
-            else { tokenizer.advance(1); Delim(c) }
+            else { tokenizer.advance(1); Delim('\\') }
         },
-        ']' => { tokenizer.advance(1); CloseSquareBracket },
-        '^' => {
-            if tokenizer.starts_with("^=") { tokenizer.advance(2); PrefixMatch }
-            else { tokenizer.advance(1); Delim(c) }
+        b']' => { tokenizer.advance(1); CloseSquareBracket },
+        b'^' => {
+            if tokenizer.starts_with(b"^=") { tokenizer.advance(2); PrefixMatch }
+            else { tokenizer.advance(1); Delim('^') }
         },
-        '{' => { tokenizer.advance(1); CurlyBracketBlock },
-        '|' => {
-            if tokenizer.starts_with("|=") { tokenizer.advance(2); DashMatch }
-            else if tokenizer.starts_with("||") { tokenizer.advance(2); Column }
-            else { tokenizer.advance(1); Delim(c) }
+        b'{' => { tokenizer.advance(1); CurlyBracketBlock },
+        b'|' => {
+            if tokenizer.starts_with(b"|=") { tokenizer.advance(2); DashMatch }
+            else if tokenizer.starts_with(b"||") { tokenizer.advance(2); Column }
+            else { tokenizer.advance(1); Delim('|') }
         },
-        '}' => { tokenizer.advance(1); CloseCurlyBracket },
-        '~' => {
-            if tokenizer.starts_with("~=") { tokenizer.advance(2); IncludeMatch }
-            else { tokenizer.advance(1); Delim(c) }
+        b'}' => { tokenizer.advance(1); CloseCurlyBracket },
+        b'~' => {
+            if tokenizer.starts_with(b"~=") { tokenizer.advance(2); IncludeMatch }
+            else { tokenizer.advance(1); Delim('~') }
         },
         _ => {
-            if c > '\x7F' {  // Non-ASCII
+            if !c.is_ascii() {  // Non-ASCII
                 consume_ident_like(tokenizer)
             } else {
+                let ret = Delim(tokenizer.next_char());
                 tokenizer.advance(1);
-                Delim(c)
+                ret
             }
         },
     };
@@ -641,15 +642,15 @@ fn consume_quoted_string<'a>(tokenizer: &mut Tokenizer<'a>, single_quote: bool)
 
 #[inline]
 fn is_ident_start(tokenizer: &mut Tokenizer) -> bool {
-    !tokenizer.is_eof() && match tokenizer.next_char() {
-        'a'...'z' | 'A'...'Z' | '_' | '\0' => true,
-        '-' => tokenizer.has_at_least(1) && match tokenizer.char_at(1) {
-            'a'...'z' | 'A'...'Z' | '-' | '_' | '\0' => true,
-            '\\' => !tokenizer.has_newline_at(1),
-            c => c > '\x7F',  // Non-ASCII
+    !tokenizer.is_eof() && match tokenizer.next_byte_unchecked() {
+        b'a'...b'z' | b'A'...b'Z' | b'_' | b'\0' => true,
+        b'-' => tokenizer.has_at_least(1) && match tokenizer.byte_at(1) {
+            b'a'...b'z' | b'A'...b'Z' | b'-' | b'_' | b'\0' => true,
+            b'\\' => !tokenizer.has_newline_at(1),
+            c => !c.is_ascii(),
         },
-        '\\' => !tokenizer.has_newline_at(1),
-        c => c > '\x7F',  // Non-ASCII
+        b'\\' => !tokenizer.has_newline_at(1),
+        c => !c.is_ascii(),
     }
 }
 

From 46762084db55ab6227cf3f2bbfe87bb711d1a845 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Wed, 18 Jan 2017 00:31:14 +0100
Subject: [PATCH 2/5] Use a procedural macro to create jump tables instead of
 doing branching.

This increases the performance of the stylesheet tokenization test about 20%,
and now one of the hottest instructions is the sign extension rust does to index
in the array.
---
 Cargo.toml               |   6 +
 build.rs                 |  40 ++++++
 src/lib.rs               |  18 +++
 src/macros/match_byte.rs | 271 +++++++++++++++++++++++++++++++++++++++
 src/macros/mod.rs        |   5 +
 src/tokenizer.rs         | 206 +++++++++++++++++------------
 6 files changed, 461 insertions(+), 85 deletions(-)
 create mode 100644 build.rs
 create mode 100644 src/macros/match_byte.rs
 create mode 100644 src/macros/mod.rs

diff --git a/Cargo.toml b/Cargo.toml
index 414003dc..f46859b4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ repository = "https://github.com/servo/rust-cssparser"
 readme = "README.md"
 keywords = ["css", "syntax", "parser"]
 license = "MPL-2.0"
+build = "build.rs"
 
 
 [dev-dependencies]
@@ -22,7 +23,12 @@ heapsize = {version = ">=0.1.1, <0.4.0", optional = true}
 matches = "0.1"
 serde = {version = ">=0.6.6, <0.9", optional = true}
 
+[build-dependencies]
+syn = { version = "0.10.6", features = ["full", "visit"]}
+quote = "0.3"
+
 [features]
 serde-serialization = [ "serde" ]
 heap_size = [ "heapsize" ]
 bench = []
+dummy_match_byte = []
diff --git a/build.rs b/build.rs
new file mode 100644
index 00000000..84e36be5
--- /dev/null
+++ b/build.rs
@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#[macro_use] extern crate quote;
+extern crate syn;
+
+use std::env;
+use std::path::Path;
+
+
+#[cfg(feature = "dummy_match_byte")]
+mod codegen {
+    use std::path::Path;
+    pub fn main(_: &Path) {}
+}
+
+#[cfg(not(feature = "dummy_match_byte"))]
+#[path = "src/macros/mod.rs"]
+mod macros;
+
+#[cfg(not(feature = "dummy_match_byte"))]
+mod codegen {
+    use macros;
+    use std::env;
+    use std::path::Path;
+
+    pub fn main(tokenizer_rs: &Path) {
+        macros::match_byte::expand(tokenizer_rs,
+                                   &Path::new(&env::var("OUT_DIR").unwrap()).join("tokenizer.rs"));
+
+    }
+}
+
+fn main() {
+    let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+    let tokenizer_rs = Path::new(&manifest_dir).join("src/tokenizer.rs");
+    codegen::main(&tokenizer_rs);
+    println!("cargo:rerun-if-changed={}", tokenizer_rs.display());
+}
diff --git a/src/lib.rs b/src/lib.rs
index 0681bcde..58859e88 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -137,7 +137,25 @@ macro_rules! match_ignore_ascii_case {
 }
 
 mod rules_and_declarations;
+
+#[cfg(feature = "dummy_match_byte")]
+macro_rules! match_byte {
+    ($value:expr, $($rest:tt)* ) => {
+        match $value {
+            $(
+                $rest
+            )+
+        }
+    };
+}
+
+#[cfg(feature = "dummy_match_byte")]
 mod tokenizer;
+
+#[cfg(not(feature = "dummy_match_byte"))]
+mod tokenizer {
+    include!(concat!(env!("OUT_DIR"), "/tokenizer.rs"));
+}
 mod parser;
 mod from_bytes;
 mod color;
diff --git a/src/macros/match_byte.rs b/src/macros/match_byte.rs
new file mode 100644
index 00000000..79519d01
--- /dev/null
+++ b/src/macros/match_byte.rs
@@ -0,0 +1,271 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use quote::{ToTokens, Tokens};
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::Path;
+use std::vec;
+use std::iter;
+use syn;
+
+pub fn expand(from: &Path, to: &Path) {
+    let mut source = String::new();
+    File::open(from).unwrap().read_to_string(&mut source).unwrap();
+    let tts = syn::parse_token_trees(&source).expect("Parsing rules.rs module");
+    let mut tokens = Tokens::new();
+    tokens.append_all(expand_tts(tts));
+
+    let code = tokens.to_string().replace("{ ", "{\n").replace(" }", "\n}");
+    File::create(to).unwrap().write_all(code.as_bytes()).unwrap();
+}
+
+fn expand_tts(tts: Vec<syn::TokenTree>) -> Vec<syn::TokenTree> {
+    use syn::*;
+    let mut expanded = Vec::new();
+    let mut tts = tts.into_iter();
+    while let Some(tt) = tts.next() {
+        match tt {
+            TokenTree::Token(Token::Ident(ident)) => {
+                if ident != "match_byte" {
+                    expanded.push(TokenTree::Token(Token::Ident(ident)));
+                    continue;
+                }
+
+                match tts.next() {
+                    Some(TokenTree::Token(Token::Not)) => {},
+                    other => {
+                        expanded.push(TokenTree::Token(Token::Ident(ident)));
+                        if let Some(other) = other {
+                            expanded.push(other);
+                        }
+                        continue;
+                    }
+                }
+
+                let tts = match tts.next() {
+                    Some(TokenTree::Delimited(Delimited { tts, .. })) => tts,
+                    other => {
+                        expanded.push(TokenTree::Token(Token::Ident(ident)));
+                        expanded.push(TokenTree::Token(Token::Not));
+                        if let Some(other) = other {
+                            expanded.push(other);
+                        }
+                        continue;
+                    }
+                };
+
+                let (to_be_matched, table, cases, wildcard_binding) = parse_match_bytes_macro(tts);
+                let expr = expand_match_bytes_macro(to_be_matched,
+                                                    &table,
+                                                    cases,
+                                                    wildcard_binding);
+
+                let tts = syn::parse_token_trees(&expr)
+                    .expect("parsing macro expansion as token trees");
+                expanded.extend(expand_tts(tts));
+            }
+            TokenTree::Delimited(Delimited { delim, tts }) => {
+                expanded.push(TokenTree::Delimited(Delimited {
+                    delim: delim,
+                    tts: expand_tts(tts),
+                }))
+            }
+            other => expanded.push(other),
+        }
+    }
+    expanded
+}
+
+/// Parses a token tree corresponding to the `match_byte` macro.
+///
+/// ## Example
+///
+/// ```rust
+/// match_byte! { tokenizer.next_byte_unchecked(),
+///     b'a'..b'z' => { ... }
+///     b'0'..b'9' => { ... }
+///     b'\n' | b'\\' => { ... }
+///     foo => { ... }
+/// }
+///
+/// Returns:
+///  * The token tree that contains the expression to be matched (in this case
+///    `tokenizer.next_byte_unchecked()`.
+///
+///  * The table with the different cases per byte, each entry in the table
+///    contains a non-zero integer representing a different arm of the
+///    match expression.
+///
+///  * The list of cases containing the expansion of the arms of the match
+///    expression.
+///
+///  * An optional identifier to which the wildcard pattern is matched (`foo` in
+///    this case).
+///
+fn parse_match_bytes_macro(tts: Vec<syn::TokenTree>) -> (Vec<syn::TokenTree>, [u8; 256], Vec<Case>, Option<syn::Ident>) {
+    let mut tts = tts.into_iter();
+
+    // Grab the thing we're matching, until we find a comma.
+    let mut left_hand_side = vec![];
+    loop {
+        match tts.next() {
+            Some(syn::TokenTree::Token(syn::Token::Comma)) => break,
+            Some(other) => left_hand_side.push(other),
+            None => panic!("Expected not to run out of tokens looking for a comma"),
+        }
+    }
+
+    let mut cases = vec![];
+    let mut table = [0u8; 256];
+
+    let mut tts = tts.peekable();
+    let mut case_id: u8 = 1;
+    let mut binding = None;
+    while tts.len() > 0 {
+        cases.push(parse_case(&mut tts, &mut table, &mut binding, case_id));
+
+        // Allow an optional comma between cases.
+        match tts.peek() {
+            Some(&syn::TokenTree::Token(syn::Token::Comma)) => {
+                tts.next();
+            },
+            _ => {},
+        }
+
+        case_id += 1;
+    }
+
+    (left_hand_side, table, cases, binding)
+}
+
+#[derive(Debug)]
+struct Case(Vec<syn::TokenTree>);
+
+/// Parses a single pattern => expression, and returns the case, filling in the
+/// table with the case id for every byte that matched.
+///
+/// The `binding` parameter is the identifier that is used by the wildcard
+/// pattern.
+fn parse_case(tts: &mut iter::Peekable<vec::IntoIter<syn::TokenTree>>,
+              table: &mut [u8; 256],
+              binding: &mut Option<syn::Ident>,
+              case_id: u8)
+              -> Case {
+    // The last byte checked, as part of this pattern, to properly detect
+    // ranges.
+    let mut last_byte: Option<u8> = None;
+
+    // Loop through the pattern filling with bytes the table.
+    loop {
+        match tts.next() {
+            Some(syn::TokenTree::Token(syn::Token::Literal(syn::Lit::Byte(byte)))) => {
+                table[byte as usize] = case_id;
+                last_byte = Some(byte);
+            }
+            Some(syn::TokenTree::Token(syn::Token::BinOp(syn::BinOpToken::Or))) => {
+                last_byte = None; // This pattern is over.
+            },
+            Some(syn::TokenTree::Token(syn::Token::DotDotDot)) => {
+                assert!(last_byte.is_some(), "Expected closed range!");
+                match tts.next() {
+                    Some(syn::TokenTree::Token(syn::Token::Literal(syn::Lit::Byte(byte)))) => {
+                        for b in last_byte.take().unwrap()..byte {
+                            if table[b as usize] == 0 {
+                                table[b as usize] = case_id;
+                            }
+                        }
+                        if table[byte as usize] == 0 {
+                            table[byte as usize] = case_id;
+                        }
+                    }
+                    other => panic!("Expected closed range, got: {:?}", other),
+                }
+            },
+            Some(syn::TokenTree::Token(syn::Token::FatArrow)) => break,
+            Some(syn::TokenTree::Token(syn::Token::Ident(ident))) => {
+                assert_eq!(last_byte, None, "I don't support ranges with identifiers!");
+                assert_eq!(*binding, None);
+                for mut byte in table.iter_mut() {
+                    if *byte == 0 {
+                        *byte = case_id;
+                    }
+                }
+                *binding = Some(ident)
+            }
+            Some(syn::TokenTree::Token(syn::Token::Underscore)) => {
+                assert_eq!(last_byte, None);
+                for mut byte in table.iter_mut() {
+                    if *byte == 0 {
+                        *byte = case_id;
+                    }
+                }
+            },
+            other => panic!("Expected literal byte, got: {:?}", other),
+        }
+    }
+
+    match tts.next() {
+        Some(syn::TokenTree::Delimited(syn::Delimited { delim: syn::DelimToken::Brace, tts })) => {
+            Case(tts)
+        }
+        other => panic!("Expected case with braces after fat arrow, got: {:?}", other),
+    }
+}
+
+fn expand_match_bytes_macro(to_be_matched: Vec<syn::TokenTree>,
+                            table: &[u8; 256],
+                            cases: Vec<Case>,
+                            binding: Option<syn::Ident>)
+                            -> String {
+    use std::fmt::Write;
+
+    assert!(!to_be_matched.is_empty());
+    assert!(table.iter().all(|b| *b != 0), "Incomplete pattern? Bogus code!");
+
+    // We build the expression with text since it's easier.
+    let mut expr = "{\n".to_owned();
+    expr.push_str("enum Case {\n");
+    for (i, _) in cases.iter().enumerate() {
+        write!(&mut expr, "Case{} = {},", i + 1, i + 1).unwrap();
+    }
+    expr.push_str("}\n"); // enum Case
+
+    expr.push_str("static __CASES: [Case; 256] = [");
+    for byte in table.iter() {
+        write!(&mut expr, "Case::Case{}, ", *byte).unwrap();
+    }
+    expr.push_str("];\n");
+
+    let mut tokens = Tokens::new();
+    let to_be_matched = syn::Delimited {
+        delim: if binding.is_some() { syn::DelimToken::Brace } else { syn::DelimToken::Paren },
+        tts: to_be_matched
+    };
+    to_be_matched.to_tokens(&mut tokens);
+
+    if let Some(ref binding) = binding {
+        write!(&mut expr, "let {} = {};\n", binding.to_string(), tokens.as_str()).unwrap();
+    }
+
+    write!(&mut expr, "match __CASES[{} as usize] {{", match binding {
+        Some(binding) => binding.to_string(),
+        None => tokens.to_string(),
+    }).unwrap();
+
+    for (i, case) in cases.into_iter().enumerate() {
+        let mut case_tokens = Tokens::new();
+        let case = syn::Delimited {
+            delim: syn::DelimToken::Brace,
+            tts: case.0
+        };
+        case.to_tokens(&mut case_tokens);
+        write!(&mut expr, "Case::Case{} => {},\n", i + 1, case_tokens.as_str()).unwrap();
+    }
+    expr.push_str("}\n"); // match
+
+    expr.push_str("}\n"); // top
+
+    expr
+}
diff --git a/src/macros/mod.rs b/src/macros/mod.rs
new file mode 100644
index 00000000..6799e549
--- /dev/null
+++ b/src/macros/mod.rs
@@ -0,0 +1,5 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+pub mod match_byte;
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index faef7e1d..118a69dd 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -406,7 +406,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
         return None
     }
     let c = tokenizer.next_byte_unchecked();
-    let token = match c {
+    let token = match_byte! { c,
         b'\t' | b'\n' | b' ' | b'\r' | b'\x0C' => {
             let start_position = tokenizer.position();
             tokenizer.advance(1);
@@ -418,7 +418,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
             }
             WhiteSpace(tokenizer.slice_from(start_position))
         },
-        b'"' => consume_string(tokenizer, false),
+        b'"' => { consume_string(tokenizer, false) },
         b'#' => {
             tokenizer.advance(1);
             if is_ident_start(tokenizer) { IDHash(consume_name(tokenizer)) }
@@ -433,7 +433,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
             if tokenizer.starts_with(b"$=") { tokenizer.advance(2); SuffixMatch }
             else { tokenizer.advance(1); Delim('$') }
         },
-        b'\'' => consume_string(tokenizer, true),
+        b'\'' => { consume_string(tokenizer, true) },
         b'(' => { tokenizer.advance(1); ParenthesisBlock },
         b')' => { tokenizer.advance(1); CloseParenthesis },
         b'*' => {
@@ -486,24 +486,29 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
                 Delim('.')
             }
         }
-        b'/' if tokenizer.starts_with(b"/*") => {
-            tokenizer.advance(2);  // consume "/*"
-            let start_position = tokenizer.position();
-            let content;
-            match tokenizer.input[tokenizer.position..].find("*/") {
-                Some(offset) => {
-                    tokenizer.advance(offset);
-                    content = tokenizer.slice_from(start_position);
-                    tokenizer.advance(2);
-                }
-                None => {
-                    tokenizer.position = tokenizer.input.len();
-                    content = tokenizer.slice_from(start_position);
+        b'/' => {
+            if tokenizer.starts_with(b"/*") {
+                tokenizer.advance(2);  // consume "/*"
+                let start_position = tokenizer.position();
+                let content;
+                match tokenizer.input[tokenizer.position..].find("*/") {
+                    Some(offset) => {
+                        tokenizer.advance(offset);
+                        content = tokenizer.slice_from(start_position);
+                        tokenizer.advance(2);
+                    }
+                    None => {
+                        tokenizer.position = tokenizer.input.len();
+                        content = tokenizer.slice_from(start_position);
+                    }
                 }
+                Comment(content)
+            } else {
+                tokenizer.advance(1);
+                Delim('/')
             }
-            Comment(content)
         }
-        b'0'...b'9' => consume_numeric(tokenizer),
+        b'0'...b'9' => { consume_numeric(tokenizer) },
         b':' => { tokenizer.advance(1); Colon },
         b';' => { tokenizer.advance(1); Semicolon },
         b'<' => {
@@ -527,7 +532,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
             { consume_unicode_range(tokenizer) }
             else { consume_ident_like(tokenizer) }
         },
-        b'a'...b'z' | b'A'...b'Z' | b'_' | b'\0' => consume_ident_like(tokenizer),
+        b'a'...b'z' | b'A'...b'Z' | b'_' | b'\0' => { consume_ident_like(tokenizer) },
         b'[' => { tokenizer.advance(1); SquareBracketBlock },
         b'\\' => {
             if !tokenizer.has_newline_at(1) { consume_ident_like(tokenizer) }
@@ -550,7 +555,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
             else { tokenizer.advance(1); Delim('~') }
         },
         _ => {
-            if !c.is_ascii() {  // Non-ASCII
+            if !c.is_ascii() {
                 consume_ident_like(tokenizer)
             } else {
                 let ret = Delim(tokenizer.next_char());
@@ -581,35 +586,47 @@ fn consume_quoted_string<'a>(tokenizer: &mut Tokenizer<'a>, single_quote: bool)
         if tokenizer.is_eof() {
             return Ok(Borrowed(tokenizer.slice_from(start_pos)))
         }
-        match tokenizer.next_byte_unchecked() {
-            b'"' if !single_quote => {
-                let value = tokenizer.slice_from(start_pos);
-                tokenizer.advance(1);
-                return Ok(Borrowed(value))
+        match_byte! { tokenizer.next_byte_unchecked(),
+            b'"' => {
+                if !single_quote {
+                    let value = tokenizer.slice_from(start_pos);
+                    tokenizer.advance(1);
+                    return Ok(Borrowed(value))
+                }
             }
-            b'\'' if single_quote => {
-                let value = tokenizer.slice_from(start_pos);
-                tokenizer.advance(1);
-                return Ok(Borrowed(value))
+            b'\'' => {
+                if single_quote {
+                    let value = tokenizer.slice_from(start_pos);
+                    tokenizer.advance(1);
+                    return Ok(Borrowed(value))
+                }
             }
             b'\\' | b'\0' => {
                 string_bytes = tokenizer.slice_from(start_pos).as_bytes().to_owned();
                 break
             }
-            b'\n' | b'\r' | b'\x0C' => return Err(()),
-            _ => {
-                tokenizer.consume_byte();
-            }
+            b'\n' | b'\r' | b'\x0C' => { return Err(()) },
+            _ => {}
         }
+        tokenizer.consume_byte();
     }
 
     while !tokenizer.is_eof() {
         if matches!(tokenizer.next_byte_unchecked(), b'\n' | b'\r' | b'\x0C') {
             return Err(());
         }
-        match tokenizer.consume_byte() {
-            b'"' if !single_quote => break,
-            b'\'' if single_quote => break,
+        let c = tokenizer.consume_byte();
+        match_byte! { c,
+            b'"' => {
+                if !single_quote {
+                    break;
+                }
+            }
+            b'\'' => {
+                if single_quote {
+                    break;
+                }
+            }
             b'\\' => {
                 if !tokenizer.is_eof() {
                     match tokenizer.next_byte_unchecked() {
@@ -625,15 +642,19 @@ fn consume_quoted_string<'a>(tokenizer: &mut Tokenizer<'a>, single_quote: bool)
                     }
                 }
                 // else: escaped EOF, do nothing.
+                continue;
             }
             b'\0' => {
                 // string.push('\u{FFFD}'),
                 string_bytes.push(0xef);
                 string_bytes.push(0xbf);
                 string_bytes.push(0xbd);
+                continue;
             }
-            c => string_bytes.push(c),
+            _ => {},
         }
+
+        string_bytes.push(c);
     }
 
     Ok(Owned(to_utf8(string_bytes)))
@@ -642,15 +663,19 @@ fn consume_quoted_string<'a>(tokenizer: &mut Tokenizer<'a>, single_quote: bool)
 
 #[inline]
 fn is_ident_start(tokenizer: &mut Tokenizer) -> bool {
-    !tokenizer.is_eof() && match tokenizer.next_byte_unchecked() {
-        b'a'...b'z' | b'A'...b'Z' | b'_' | b'\0' => true,
-        b'-' => tokenizer.has_at_least(1) && match tokenizer.byte_at(1) {
-            b'a'...b'z' | b'A'...b'Z' | b'-' | b'_' | b'\0' => true,
-            b'\\' => !tokenizer.has_newline_at(1),
-            c => !c.is_ascii(),
+    !tokenizer.is_eof() && match_byte! { tokenizer.next_byte_unchecked(),
+        b'a'...b'z' | b'A'...b'Z' | b'_' | b'\0' => { true },
+        b'-' => {
+            tokenizer.has_at_least(1) && match_byte! { tokenizer.byte_at(1),
+                b'a'...b'z' | b'A'...b'Z' | b'-' | b'_' | b'\0' => {
+                    true
+                }
+                b'\\' => { !tokenizer.has_newline_at(1) }
+                c => { !c.is_ascii() },
+            }
         },
-        b'\\' => !tokenizer.has_newline_at(1),
-        c => !c.is_ascii(),
+        b'\\' => { !tokenizer.has_newline_at(1) },
+        c => { !c.is_ascii() },
     }
 }
 
@@ -680,14 +705,16 @@ fn consume_name<'a>(tokenizer: &mut Tokenizer<'a>) -> Cow<'a, str> {
         if tokenizer.is_eof() {
             return Borrowed(tokenizer.slice_from(start_pos))
         }
-        match tokenizer.next_byte_unchecked() {
-            b'a'...b'z' | b'A'...b'Z' | b'0'...b'9' | b'_' | b'-'  => tokenizer.advance(1),
+        match_byte! { tokenizer.next_byte_unchecked(),
+            b'a'...b'z' | b'A'...b'Z' | b'0'...b'9' | b'_' | b'-' => { tokenizer.advance(1) },
             b'\\' | b'\0' => {
                 value_bytes = tokenizer.slice_from(start_pos).as_bytes().to_owned();
                 break
             }
-            c if c.is_ascii() => return Borrowed(tokenizer.slice_from(start_pos)),
-            _ => {
+            c => {
+                if c.is_ascii() {
+                    return Borrowed(tokenizer.slice_from(start_pos));
+                }
                 tokenizer.advance(1);
             }
         }
@@ -695,7 +722,7 @@ fn consume_name<'a>(tokenizer: &mut Tokenizer<'a>) -> Cow<'a, str> {
 
     while !tokenizer.is_eof() {
         let c = tokenizer.next_byte_unchecked();
-        match c {
+        match_byte! { c,
             b'a'...b'z' | b'A'...b'Z' | b'0'...b'9' | b'_' | b'-'  => {
                 tokenizer.advance(1);
                 value_bytes.push(c)
@@ -712,10 +739,12 @@ fn consume_name<'a>(tokenizer: &mut Tokenizer<'a>) -> Cow<'a, str> {
                 value_bytes.push(0xbf);
                 value_bytes.push(0xbd);
             },
-            c if c.is_ascii() => break,
-            other => {
+            _ => {
+                if c.is_ascii() {
+                    break;
+                }
                 tokenizer.advance(1);
-                value_bytes.push(other)
+                value_bytes.push(c)
             }
         }
     }
@@ -770,19 +799,19 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
 
     if (
         tokenizer.has_at_least(1)
-        && matches!(tokenizer.next_char(), 'e' | 'E')
-        && matches!(tokenizer.char_at(1), '0'...'9')
+        && matches!(tokenizer.next_byte_unchecked(), b'e' | b'E')
+        && matches!(tokenizer.byte_at(1), b'0'...b'9')
     ) || (
         tokenizer.has_at_least(2)
-        && matches!(tokenizer.next_char(), 'e' | 'E')
-        && matches!(tokenizer.char_at(1), '+' | '-')
-        && matches!(tokenizer.char_at(2), '0'...'9')
+        && matches!(tokenizer.next_byte_unchecked(), b'e' | b'E')
+        && matches!(tokenizer.byte_at(1), b'+' | b'-')
+        && matches!(tokenizer.byte_at(2), b'0'...b'9')
     ) {
         is_integer = false;
         tokenizer.advance(1);
-        let (has_sign, sign) = match tokenizer.next_char() {
-            '-' => (true, -1.),
-            '+' => (true, 1.),
+        let (has_sign, sign) = match tokenizer.next_byte_unchecked() {
+            b'-' => (true, -1.),
+            b'+' => (true, 1.),
             _ => (false, 1.),
         };
         if has_sign {
@@ -811,7 +840,7 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
         None
     };
 
-    if !tokenizer.is_eof() && tokenizer.next_char() == '%' {
+    if !tokenizer.is_eof() && tokenizer.next_byte_unchecked() == b'%' {
         tokenizer.advance(1);
         return Percentage(PercentageValue {
             unit_value: value as f32 / 100.,
@@ -853,32 +882,31 @@ fn to_utf8(string_bytes: Vec<u8>) -> String {
 }
 
 fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
-
     for (offset, c) in tokenizer.input[tokenizer.position..].bytes().enumerate() {
-        match c {
+        match_byte! { c,
             b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => {},
-            b'"' | b'\'' => return Err(()),  // Do not advance
+            b'"' | b'\'' => { return Err(()) },  // Do not advance
             b')' => {
                 tokenizer.advance(offset + 1);
                 return Ok(UnquotedUrl(Borrowed("")));
             }
             _ => {
                 tokenizer.advance(offset);
-                return Ok(consume_unquoted_url(tokenizer))
+                return Ok(consume_unquoted_url_internal(tokenizer))
             }
         }
     }
     tokenizer.position = tokenizer.input.len();
     return Ok(UnquotedUrl(Borrowed("")));
 
-    fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
+    fn consume_unquoted_url_internal<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
         let start_pos = tokenizer.position();
         let mut string_bytes: Vec<u8>;
         loop {
             if tokenizer.is_eof() {
                 return UnquotedUrl(Borrowed(tokenizer.slice_from(start_pos)))
             }
-            match tokenizer.next_byte_unchecked() {
+            match_byte! { tokenizer.next_byte_unchecked(),
                 b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => {
                     let value = tokenizer.slice_from(start_pos);
                     tokenizer.advance(1);
@@ -904,13 +932,17 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
             }
         }
         while !tokenizer.is_eof() {
-            match tokenizer.consume_byte() {
+            match_byte! { tokenizer.consume_byte(),
                 b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => {
                     return consume_url_end(tokenizer, Owned(to_utf8(string_bytes)));
                 }
-                b')' => break,
+                b')' => {
+                    break;
+                }
                 b'\x01'...b'\x08' | b'\x0B' | b'\x0E'...b'\x1F' | b'\x7F'  // non-printable
-                    | b'"' | b'\'' | b'(' => return consume_bad_url(tokenizer),
+                    | b'"' | b'\'' | b'(' => {
+                    return consume_bad_url(tokenizer);
+                }
                 b'\\' => {
                     if tokenizer.has_newline_at(0) {
                         return consume_bad_url(tokenizer)
@@ -924,7 +956,7 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
                     string_bytes.push(0xbf);
                     string_bytes.push(0xbd);
                 }
-                c => string_bytes.push(c)
+                c => { string_bytes.push(c) }
             }
         }
         UnquotedUrl(Owned(to_utf8(string_bytes)))
@@ -932,10 +964,12 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
 
     fn consume_url_end<'a>(tokenizer: &mut Tokenizer<'a>, string: Cow<'a, str>) -> Token<'a> {
         while !tokenizer.is_eof() {
-            match tokenizer.consume_byte() {
-                b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => (),
-                b')' => break,
-                _ => return consume_bad_url(tokenizer)
+            match_byte! { tokenizer.consume_byte(),
+                b' ' | b'\t' | b'\n' | b'\r' | b'\x0C' => {},
+                b')' => { break },
+                _ => {
+                    return consume_bad_url(tokenizer);
+                }
             }
         }
         UnquotedUrl(string)
@@ -944,10 +978,12 @@ fn consume_unquoted_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>,
     fn consume_bad_url<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
         // Consume up to the closing )
         while !tokenizer.is_eof() {
-            match tokenizer.consume_byte() {
-                b')' => break,
-                b'\\' => tokenizer.advance(1), // Skip an escaped ')' or '\'
-                _ => ()
+            match_byte! { tokenizer.consume_byte(),
+                b')' => { break },
+                b'\\' => {
+                    tokenizer.advance(1); // Skip an escaped ')' or '\'
+                }
+                _ => {},
             }
         }
         BadUrl
@@ -962,7 +998,7 @@ fn consume_unicode_range<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     let max_question_marks = 6 - hex_digits;
     let mut question_marks = 0;
     while question_marks < max_question_marks && !tokenizer.is_eof()
-            && tokenizer.next_char() == '?' {
+            && tokenizer.next_byte_unchecked() == b'?' {
         question_marks += 1;
         tokenizer.advance(1)
     }
@@ -974,8 +1010,8 @@ fn consume_unicode_range<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     } else {
         start = hex_value;
         if tokenizer.has_at_least(1) &&
-           tokenizer.next_char() == '-' &&
-           matches!(tokenizer.char_at(1), '0'...'9' | 'A'...'F' | 'a'...'f') {
+           tokenizer.next_byte_unchecked() == b'-' &&
+           matches!(tokenizer.byte_at(1), b'0'...b'9' | b'A'...b'F' | b'a'...b'f') {
             tokenizer.advance(1);
             let (hex_value, _) = consume_hex_digits(tokenizer);
             end = hex_value;
@@ -1019,7 +1055,7 @@ fn consume_escape_and_write(tokenizer: &mut Tokenizer, bytes: &mut Vec<u8>) {
 // to not be a newline.
 fn consume_escape(tokenizer: &mut Tokenizer) -> char {
     if tokenizer.is_eof() { return '\u{FFFD}' }  // Escaped EOF
-    match tokenizer.next_byte_unchecked() {
+    match_byte! { tokenizer.next_byte_unchecked(),
         b'0'...b'9' | b'A'...b'F' | b'a'...b'f' => {
             let (c, _) = consume_hex_digits(tokenizer);
             if !tokenizer.is_eof() {
@@ -1046,6 +1082,6 @@ fn consume_escape(tokenizer: &mut Tokenizer) -> char {
             tokenizer.advance(1);
             '\u{FFFD}'
         }
-        _ => tokenizer.consume_char()
+        _ => { tokenizer.consume_char() }
     }
 }

From b509dc01d9882ad9258cc731e765fce17a02d62d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Wed, 18 Jan 2017 01:23:13 +0100
Subject: [PATCH 3/5] Change the signature of next_token to match
 Tokenizer::next()

This was causing unaligned moves (movups instructions), for some reason.
---
 src/tokenizer.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 118a69dd..b22ca2cb 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -262,7 +262,7 @@ impl<'a> Tokenizer<'a> {
 
     #[inline]
     pub fn next(&mut self) -> Result<Token<'a>, ()> {
-        next_token(self).ok_or(())
+        next_token(self)
     }
 
     #[inline]
@@ -401,9 +401,9 @@ pub struct SourceLocation {
 }
 
 
-fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
+fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
     if tokenizer.is_eof() {
-        return None
+        return Err(())
     }
     let c = tokenizer.next_byte_unchecked();
     let token = match_byte! { c,
@@ -564,7 +564,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Option<Token<'a>> {
             }
         },
     };
-    Some(token)
+    Ok(token)
 }
 
 

From 52e65273c0c69d5659637ef3015a236b0ec8368d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Wed, 18 Jan 2017 01:42:08 +0100
Subject: [PATCH 4/5] Remove most of the unnecessary UTF-8 logic from the
 tokenizer.

---
 src/tokenizer.rs | 34 +++++++++++++++-------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index b22ca2cb..ec9ef803 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -359,17 +359,14 @@ impl<'a> Tokenizer<'a> {
     }
 
     #[inline]
-    fn next_char(&self) -> char { self.char_at(0) }
-
-    #[inline]
-    fn char_at(&self, offset: usize) -> char {
-        self.input[self.position + offset..].chars().next().unwrap()
+    fn next_char(&self) -> char {
+        self.input[self.position..].chars().next().unwrap()
     }
 
     #[inline]
     fn has_newline_at(&self, offset: usize) -> bool {
         self.position + offset < self.input.len() &&
-        matches!(self.char_at(offset), '\n' | '\r' | '\x0C')
+        matches!(self.byte_at(offset), b'\n' | b'\r' | b'\x0C')
     }
 
     #[inline]
@@ -405,8 +402,8 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
     if tokenizer.is_eof() {
         return Err(())
     }
-    let c = tokenizer.next_byte_unchecked();
-    let token = match_byte! { c,
+    let b = tokenizer.next_byte_unchecked();
+    let token = match_byte! { b,
         b'\t' | b'\n' | b' ' | b'\r' | b'\x0C' => {
             let start_position = tokenizer.position();
             tokenizer.advance(1);
@@ -425,7 +422,7 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
             else if !tokenizer.is_eof() && match tokenizer.next_byte_unchecked() {
                 b'a'...b'z' | b'A'...b'Z' | b'0'...b'9' | b'-' | b'_' => true,
                 b'\\' => !tokenizer.has_newline_at(1),
-                _ => !c.is_ascii(),
+                _ => !b.is_ascii(),
             } { Hash(consume_name(tokenizer)) }
             else { Delim('#') }
         },
@@ -555,12 +552,11 @@ fn next_token<'a>(tokenizer: &mut Tokenizer<'a>) -> Result<Token<'a>, ()> {
             else { tokenizer.advance(1); Delim('~') }
         },
         _ => {
-            if !c.is_ascii() {
+            if !b.is_ascii() {
                 consume_ident_like(tokenizer)
             } else {
-                let ret = Delim(tokenizer.next_char());
                 tokenizer.advance(1);
-                ret
+                Delim(b as char)
             }
         },
     };
@@ -671,11 +667,11 @@ fn is_ident_start(tokenizer: &mut Tokenizer) -> bool {
                     true
                 }
                 b'\\' => { !tokenizer.has_newline_at(1) }
-                c => { !c.is_ascii() },
+                b => { !b.is_ascii() },
             }
         },
         b'\\' => { !tokenizer.has_newline_at(1) },
-        c => { !c.is_ascii() },
+        b => { !b.is_ascii() },
     }
 }
 
@@ -759,9 +755,9 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     // Do all the math in f64 so that large numbers overflow to +/-inf
     // and i32::{MIN, MAX} are within range.
 
-    let (has_sign, sign) = match tokenizer.next_char() {
-        '-' => (true, -1.),
-        '+' => (true, 1.),
+    let (has_sign, sign) = match tokenizer.next_byte_unchecked() {
+        b'-' => (true, -1.),
+        b'+' => (true, 1.),
         _ => (false, 1.),
     };
     if has_sign {
@@ -780,8 +776,8 @@ fn consume_numeric<'a>(tokenizer: &mut Tokenizer<'a>) -> Token<'a> {
     let mut is_integer = true;
 
     let mut fractional_part: f64 = 0.;
-    if tokenizer.has_at_least(1) && tokenizer.next_char() == '.'
-            && matches!(tokenizer.char_at(1), '0'...'9') {
+    if tokenizer.has_at_least(1) && tokenizer.next_byte_unchecked() == b'.'
+            && matches!(tokenizer.byte_at(1), b'0'...b'9') {
         is_integer = false;
         tokenizer.advance(1);  // Consume '.'
         let mut factor = 0.1;

From 21f85731aa14ccc0a1f75da9ae31c66b2a88654f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= <emilio@crisal.io>
Date: Wed, 18 Jan 2017 02:08:06 +0100
Subject: [PATCH 5/5] travis: Test with and without procedural macro.

---
 .travis.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 94f98322..2f0758b3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,9 @@ script:
   - cargo test --verbose
   - cargo doc --verbose
   - cargo test --features heapsize
+  - cargo test --features dummy_match_byte
+  - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then cargo test --features bench; fi
+  - if [ "$TRAVIS_RUST_VERSION" == "nightly" ]; then cargo test --features "bench dummy_match_byte"; fi
 
 notifications:
   webhooks: http://build.servo.org:54856/travis