Skip to content

Commit

Permalink
fix: cleanup goto (#8)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexanderBrevig
AlexanderBrevig committed Jun 27, 2023
1 parent c0bf41d commit 6fceb45
Show file tree
Hide file tree
Showing 11 changed files with 273 additions and 143 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ anyhow = "1.0.71"
thiserror = "1.0.40"

[dependencies.forth-lexer]
version = "0.1.0"
version = "0.1.1"
path = "lib/forth-lexer"
features = ["ropey"]
113 changes: 55 additions & 58 deletions lib/forth-lexer/src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ pub struct Lexer<'a> {
position: usize,
read_position: usize,
ch: char,
raw: &'a str,
input: Peekable<Chars<'a>>,
}

Expand All @@ -20,6 +21,7 @@ impl<'a> Lexer<'a> {
read_position: 0,
ch: '0',
input: input.chars().peekable(),
raw: input,
};
lex.read_char();

Expand All @@ -32,34 +34,29 @@ impl<'a> Lexer<'a> {
self.ch = '\0';
}

pub fn here<T>(&self) -> Data<T>
where
T: Default,
{
pub fn here(&self) -> Data<'a> {
Data {
start: self.position,
end: self.position,
value: T::default(),
value: "",
}
}

pub fn next_token(&mut self) -> Result<Token, LexError> {
pub fn next_token(&mut self) -> Result<Token<'a>, LexError> {
self.skip_whitespace();

let tok = match self.ch {
':' => {
let mut dat = self.here::<char>();
dat.value = ':';
let mut dat = self.here();
dat.value = &self.raw[self.position..self.read_position];
Token::Colon(dat)
}
';' => {
let mut dat = self.here::<char>();
dat.value = ';';
let mut dat = self.here();
dat.value = &self.raw[self.position..self.read_position];
dat.end = dat.start + 1;
Token::Semicolon(dat)
}
//TODO: comments
//TODO: strings
'%' => {
if self.peek_char().is_digit(2) {
let ident = self.read_number();
Expand Down Expand Up @@ -88,21 +85,21 @@ impl<'a> Lexer<'a> {
}
}
'\'' => {
let begin = self.position;
if !self.peek_char().is_whitespace() {
self.read_char();
if self.peek_char() == '\'' {
let num = self.ch;
self.read_char();
let number = Data::<String> {
let number = Data {
start: self.position - 2,
end: self.position + 1,
value: format!("'{}'", num),
value: &self.raw[begin..(self.position + 1)],
};
Token::Number(number)
} else {
let mut ident = self.read_ident();
ident.start -= 1;
ident.value = format!("{}{}", "'", ident.value);
ident.value = &self.raw[begin..self.position];
Token::Word(ident)
}
} else {
Expand Down Expand Up @@ -142,8 +139,8 @@ impl<'a> Lexer<'a> {
}
}
'\0' => {
let mut dat = self.here::<char>();
dat.value = '\0';
let mut dat = self.here();
dat.value = "\0";
Token::Eof(dat)
}
_ => {
Expand Down Expand Up @@ -181,7 +178,7 @@ impl<'a> Lexer<'a> {
}
}

fn read_comment_to(&mut self, to: char) -> Data<String> {
fn read_comment_to(&mut self, to: char) -> Data<'a> {
let start = self.position;
let mut value = String::new();
while self.ch != to {
Expand All @@ -193,28 +190,28 @@ impl<'a> Lexer<'a> {
self.read_char();
}

Data::<String> {
Data {
start,
end: self.position,
value,
value: &self.raw[start..self.position],
}
}

fn read_ident(&mut self) -> Data<String> {
fn read_ident(&mut self) -> Data<'a> {
let start = self.position;
let mut value = String::new();
while !self.ch.is_whitespace() && self.ch != '\0' {
value.push(self.ch);
self.read_char();
}
Data::<String> {
Data {
start,
end: self.position,
value,
value: &self.raw[start..self.position],
}
}

fn read_number(&mut self) -> Data<String> {
fn read_number(&mut self) -> Data<'a> {
let start = self.position;
let mut value = String::new();
//TODO: parse legal forth numbers
Expand All @@ -228,22 +225,22 @@ impl<'a> Lexer<'a> {
value.push(self.ch);
self.read_char();
}
Data::<String> {
Data {
start,
end: self.position,
value,
value: &self.raw[start..self.position],
}
}

pub fn parse(&mut self) -> Vec<Token> {
pub fn parse(&mut self) -> Vec<Token<'a>> {
let mut tokens = vec![];
while let Ok(tok) = self.next_token() {
match tok {
Token::Eof(_) => {
break;
}
_ => {
tokens.push(tok);
tokens.push(tok.clone());
}
}
}
Expand All @@ -261,13 +258,13 @@ mod tests {
let mut lexer = Lexer::new(": add1 ( n -- n )\n 1 + \\ adds one\n;");
let tokens = lexer.parse();
let expected = vec![
Colon(Data::new(0, 0, ':')),
Word(Data::new(2, 6, "add1".into())),
Comment(Data::new(7, 17, "( n -- n )".into())),
Number(Data::new(20, 21, "1".into())),
Word(Data::new(22, 23, "+".into())),
Comment(Data::new(24, 34, "\\ adds one".into())),
Semicolon(Data::new(35, 36, ';')),
Colon(Data::new(0, 0, ":")),
Word(Data::new(2, 6, "add1")),
Comment(Data::new(7, 17, "( n -- n )")),
Number(Data::new(20, 21, "1")),
Word(Data::new(22, 23, "+")),
Comment(Data::new(24, 34, "\\ adds one")),
Semicolon(Data::new(35, 36, ";")),
];
assert_eq!(tokens, expected)
}
Expand All @@ -277,11 +274,11 @@ mod tests {
let mut lexer = Lexer::new(": add1 1 + ;");
let tokens = lexer.parse();
let expected = vec![
Colon(Data::new(0, 0, ':')),
Word(Data::new(2, 6, "add1".into())),
Number(Data::new(7, 8, "1".into())),
Word(Data::new(9, 10, "+".into())),
Semicolon(Data::new(11, 12, ';')),
Colon(Data::new(0, 0, ":")),
Word(Data::new(2, 6, "add1")),
Number(Data::new(7, 8, "1")),
Word(Data::new(9, 10, "+")),
Semicolon(Data::new(11, 12, ";")),
];
assert_eq!(tokens, expected)
}
Expand All @@ -291,11 +288,11 @@ mod tests {
let mut lexer = Lexer::new("word \\ this is a comment\nword2 ( and this ) word3");
let tokens = lexer.parse();
let expected = vec![
Word(Data::new(0, 4, "word".into())),
Comment(Data::new(5, 24, "\\ this is a comment".into())),
Word(Data::new(25, 30, "word2".into())),
Comment(Data::new(31, 43, "( and this )".into())),
Word(Data::new(44, 49, "word3".into())),
Word(Data::new(0, 4, "word")),
Comment(Data::new(5, 24, "\\ this is a comment")),
Word(Data::new(25, 30, "word2")),
Comment(Data::new(31, 43, "( and this )")),
Word(Data::new(44, 49, "word3")),
];
assert_eq!(tokens, expected)
}
Expand All @@ -305,9 +302,9 @@ mod tests {
let mut lexer = Lexer::new("some\nwords here\0");
let tokens = lexer.parse();
let expected = vec![
Word(Data::new(0, 4, "some".into())),
Word(Data::new(5, 10, "words".into())),
Word(Data::new(11, 15, "here".into())),
Word(Data::new(0, 4, "some")),
Word(Data::new(5, 10, "words")),
Word(Data::new(11, 15, "here")),
];
assert_eq!(tokens, expected)
}
Expand All @@ -316,23 +313,23 @@ mod tests {
fn test_parse_number_literal() {
let mut lexer = Lexer::new("12");
let tokens = lexer.parse();
let expected = vec![Number(Data::new(0, 2, "12".into()))];
let expected = vec![Number(Data::new(0, 2, "12"))];
assert_eq!(tokens, expected)
}

#[test]
fn test_parse_number_oct() {
let mut lexer = Lexer::new("&12");
let tokens = lexer.parse();
let expected = vec![Number(Data::new(0, 3, "&12".into()))];
let expected = vec![Number(Data::new(0, 3, "&12"))];
assert_eq!(tokens, expected)
}

#[test]
fn test_parse_number_bin() {
let mut lexer = Lexer::new("%0100101");
let tokens = lexer.parse();
let expected = vec![Number(Data::new(0, 8, "%0100101".into()))];
let expected = vec![Number(Data::new(0, 8, "%0100101"))];
assert_eq!(tokens, expected);
}

Expand All @@ -343,39 +340,39 @@ mod tests {
// %12345 is not a binary number
let mut lexer = Lexer::new("%12345");
let tokens = lexer.parse();
let expected = vec![Word(Data::new(0, 6, "%12345".into()))];
let expected = vec![Word(Data::new(0, 6, "%12345"))];
assert_eq!(tokens, expected);
}

#[test]
fn test_parse_number_hex() {
let mut lexer = Lexer::new("$FfAaDd");
let tokens = lexer.parse();
let expected = vec![Number(Data::new(0, 7, "$FfAaDd".into()))];
let expected = vec![Number(Data::new(0, 7, "$FfAaDd"))];
assert_eq!(tokens, expected)
}

#[test]
fn test_parse_number_0xhex() {
let mut lexer = Lexer::new("0xFE");
let tokens = lexer.parse();
let expected = vec![Number(Data::new(0, 4, "0xFE".into()))];
let expected = vec![Number(Data::new(0, 4, "0xFE"))];
assert_eq!(tokens, expected)
}

#[test]
fn test_parse_number_char() {
let mut lexer = Lexer::new("'c'");
let tokens = lexer.parse();
let expected = vec![Number(Data::new(0, 3, "'c'".into()))];
let expected = vec![Number(Data::new(0, 3, "'c'"))];
assert_eq!(tokens, expected)
}

#[test]
fn test_parse_number_word() {
let mut lexer = Lexer::new("word");
let tokens = lexer.parse();
let expected = vec![Word(Data::new(0, 4, "word".into()))];
let expected = vec![Word(Data::new(0, 4, "word"))];
assert_eq!(tokens, expected)
}

Expand All @@ -389,7 +386,7 @@ mod tests {
let word2 = if let Some(Token::Word(word)) = tokens.get(1) {
word.to_owned()
} else {
Data::<String>::default()
Data::default()
};
let x = rope.slice(&word2);
assert_eq!("word2", word2.value);
Expand Down
Loading

0 comments on commit 6fceb45

Please sign in to comment.