diff options
Diffstat (limited to 'src/luax/lexer.rs')
| -rw-r--r-- | src/luax/lexer.rs | 426 |
1 files changed, 426 insertions, 0 deletions
diff --git a/src/luax/lexer.rs b/src/luax/lexer.rs new file mode 100644 index 0000000..f1021ec --- /dev/null +++ b/src/luax/lexer.rs @@ -0,0 +1,426 @@ +use crate::error::LuaXError; +use color_eyre::Result; + +use super::tokens::{StringType, Token}; + +// Macro for trying to match with multiple functions +// If a function returns a token, the token is returned +// If a function returns an error, the error is returned +// If a function returns None, the next function is tried +macro_rules! try_all_paths { + () => { + TokenizeResult::None + }; + ($e:expr) => { + $e + }; + ($head:expr $(,$tail:expr)* $(,)?) => { + match $head { + TokenizeResult::Some(t) => TokenizeResult::Some(t), + TokenizeResult::Error(e) => TokenizeResult::Error(e), + TokenizeResult::None => try_all_paths!($($tail,)*), + } + }; +} + +pub trait TokenProducer { + fn next(&mut self) -> Option<Token>; +} + +#[derive(Debug)] +pub struct Lexer<'s> { + src: &'s str, + chars: std::str::Chars<'s>, + // Current character + current: Option<char>, + // Positioning + current_pos_in_bytes: usize, + // EOF + emitted_eof: bool, +} + +#[derive(Debug)] +enum TokenizeResult<'s> { + // A token was produced + Some(Token<'s>), + // No valid token found + None, + // An error occurred + Error(LuaXError), +} + +impl<'s> Lexer<'s> { + pub fn new(src: &'s str) -> Self { + let mut chars = src.chars(); + let current = chars.next(); + Self { + src, + chars, + current, + current_pos_in_bytes: 0, + emitted_eof: false, + } + } + + pub fn next_token(&mut self) -> Result<Option<Token<'s>>> { + let token = self.lex(); + + match token { + TokenizeResult::Some(token) => Ok(Some(token)), + TokenizeResult::Error(error) => Err(error.into()), + TokenizeResult::None => match self.current { + Some(c) => Err(LuaXError::UnexpectedCharacter(c).into()), + None => { + if self.emitted_eof { + Ok(None) + } else { + self.emitted_eof = true; + Ok(Some(Token::Eof)) + } + } + }, + } + } + + fn lex(&mut self) -> TokenizeResult<'s> { + self.skip_whitespace(); + + try_all_paths!( + self.single_char_token(), + self.double_char_token(), + self.triple_char_token(), + self.string(), + self.number(), + self.comment(), + self.identifier_or_keyword(), + ) + } + + fn skip_whitespace(&mut self) { + while let Some(c) = self.current { + match c { + ' ' | '\t' | '\n' => { + self.advance(); + } + _ => break, + } + } + } + + fn comment(&mut self) -> TokenizeResult<'s> { + if self.match_char('-') { + if self.match_char('-') { + while let Some(c) = self.current { + if c == '\n' { + break; + } + self.advance(); + } + self.lex() + } else { + TokenizeResult::Some(Token::Minus) + } + } else { + TokenizeResult::None + } + } + + fn single_char_token_case(&mut self, c: char, kind: Token<'s>) -> TokenizeResult<'s> { + if self.current == Some(c) { + self.advance(); + TokenizeResult::Some(kind) + } else { + TokenizeResult::None + } + } + + fn single_char_token(&mut self) -> TokenizeResult<'s> { + try_all_paths!( + self.single_char_token_case('+', Token::Plus), + self.single_char_token_case('*', Token::Star), + self.single_char_token_case('/', Token::Slash), + self.single_char_token_case('^', Token::Hat), + self.single_char_token_case('%', Token::Percent), + self.single_char_token_case('&', Token::Amp), + self.single_char_token_case('|', Token::Pipe), + self.single_char_token_case('#', Token::Hash), + self.single_char_token_case('(', Token::OpenParen), + self.single_char_token_case(')', Token::CloseParen), + self.single_char_token_case('{', Token::OpenBrace), + self.single_char_token_case('}', Token::CloseBrace), + self.single_char_token_case(']', Token::CloseBracket), + self.single_char_token_case(';', Token::Semicolon), + self.single_char_token_case(',', Token::Comma), + self.single_char_token_case('$', Token::Dollar), + ) + } + + fn double_char_token_case( + &mut self, + first: char, + second: char, + short: Option<Token<'s>>, + long: Token<'s>, + ) -> TokenizeResult<'s> { + if self.match_char(first) { + if self.match_char(second) { + TokenizeResult::Some(long) + } else if let Some(short) = short { + TokenizeResult::Some(short) + } else { + TokenizeResult::None + } + } else { + TokenizeResult::None + } + } + + fn double_char_token_case_with_alts<const N: usize>( + &mut self, + first: char, + second: [char; N], + short: Option<Token<'s>>, + long: [Token<'s>; N], + ) -> TokenizeResult<'s> { + if self.match_char(first) { + for i in 0..N { + if self.match_char(second[i]) { + return TokenizeResult::Some(long[i]); + } + } + if let Some(short) = short { + TokenizeResult::Some(short) + } else { + TokenizeResult::None + } + } else { + TokenizeResult::None + } + } + + fn double_char_token(&mut self) -> TokenizeResult<'s> { + try_all_paths!( + self.double_char_token_case_with_alts( + '<', + ['=', '<'], + Some(Token::Lt), + [Token::Le, Token::LtLt] + ), + self.double_char_token_case_with_alts( + '>', + ['=', '>'], + Some(Token::Gt), + [Token::Ge, Token::GtGt] + ), + self.double_char_token_case('=', '=', Some(Token::Eq), Token::EqEq), + self.double_char_token_case('~', '=', Some(Token::Tilde), Token::TildeEq), + self.double_char_token_case('/', '/', Some(Token::Slash), Token::SlashSlash), + self.double_char_token_case(':', ':', Some(Token::Colon), Token::ColonColon), + ) + } + + fn triple_char_token_case( + &mut self, + first: char, + second: char, + third: char, + short: Token<'s>, + mid: Token<'s>, + long: Token<'s>, + ) -> TokenizeResult<'s> { + if self.match_char(first) { + if self.match_char(second) { + if self.match_char(third) { + TokenizeResult::Some(long) + } else { + TokenizeResult::Some(mid) + } + } else { + TokenizeResult::Some(short) + } + } else { + TokenizeResult::None + } + } + + fn triple_char_token(&mut self) -> TokenizeResult<'s> { + try_all_paths!(self.triple_char_token_case( + '.', + '.', + '.', + Token::Dot, + Token::DotDot, + Token::DotDotDot + ),) + } + + fn number(&mut self) -> TokenizeResult<'s> { + if !self.current.map_or(false, |c| c.is_numeric()) { + return TokenizeResult::None; + } + + let start = self.current_pos_in_bytes; + + while self.current.map_or(false, |c| c.is_numeric()) { + self.advance(); + } + + if self.match_char('.') { + while self.current.map_or(false, |c| c.is_numeric()) { + self.advance(); + } + } + + if self.match_char('e') || self.match_char('E') { + if self.match_char('-') || self.match_char('+') { + self.advance(); + } + while self.current.map_or(false, |c| c.is_numeric()) { + self.advance(); + } + } + + let end = self.current_pos_in_bytes; + + TokenizeResult::Some(Token::Number(self.src[start..end].parse().unwrap())) + } + + fn string(&mut self) -> TokenizeResult<'s> { + let ty = if self.match_char('"') { + StringType::Double + } else if self.match_char('\'') { + StringType::Single + } else if self.match_char('[') { + if self.match_char('[') { + StringType::DoubleBracket + } else { + return TokenizeResult::Some(Token::OpenBracket); + } + } else { + return TokenizeResult::None; + }; + + let start = self.current_pos_in_bytes; + + match ty { + StringType::Single => { + let mut escaped = false; + while self.current.is_some() { + if self.match_char('\\') { + escaped = true; + } else if self.match_char('\'') && !escaped { + break; + } else { + escaped = false; + } + self.advance(); + } + } + StringType::Double => { + let mut escaped = false; + while self.current.is_some() { + if self.match_char('\\') { + escaped = true; + } else if self.match_char('"') && !escaped { + break; + } else { + escaped = false; + } + self.advance(); + } + } + StringType::DoubleBracket => { + let mut almost_close = false; + while self.current.is_some() { + if self.current == Some(']') { + if almost_close { + break; + } else { + almost_close = true; + } + } else { + almost_close = false; + } + self.advance(); + } + } + } + + if self.current.is_none() { + TokenizeResult::Error(LuaXError::UnterminatedStringLiteral) + } else { + let end = self.current_pos_in_bytes - 1; + if ty == StringType::DoubleBracket { + self.advance(); + } + + TokenizeResult::Some(Token::String(&self.src[start..end], ty)) + } + } + + fn identifier_or_keyword(&mut self) -> TokenizeResult<'s> { + if self.current.is_none() || !Self::is_valid_identifier_start(self.current.unwrap()) { + return TokenizeResult::None; + } + + let start = self.current_pos_in_bytes; + self.advance(); + + while self.current.is_some() && Self::is_valid_in_identifier(self.current.unwrap()) { + self.advance(); + } + + let end = self.current_pos_in_bytes; + + TokenizeResult::Some(match &self.src[start..end] { + "and" => Token::And, + "break" => Token::Break, + "do" => Token::Do, + "else" => Token::Else, + "elseif" => Token::ElseIf, + "end" => Token::End, + "false" => Token::False, + "for" => Token::For, + "function" => Token::Function, + "goto" => Token::Goto, + "if" => Token::If, + "in" => Token::In, + "local" => Token::Local, + "nil" => Token::Nil, + "not" => Token::Not, + "or" => Token::Or, + "repeat" => Token::Repeat, + "return" => Token::Return, + "then" => Token::Then, + "true" => Token::True, + "until" => Token::Until, + "while" => Token::While, + _ => Token::Identifier(&self.src[start..end]), + }) + } + + fn advance(&mut self) -> Option<char> { + self.current = self.chars.next(); + self.current_pos_in_bytes += self.current.map_or(0, |c| c.len_utf8()); + + self.current + } + + fn match_char(&mut self, c: char) -> bool { + if self.current == Some(c) { + self.advance(); + true + } else { + false + } + } + + fn is_valid_in_identifier(c: char) -> bool { + Self::is_valid_identifier_start(c) || c.is_numeric() + } + + fn is_valid_identifier_start(c: char) -> bool { + c.is_alphabetic() || c == '_' // TODO: Support some more unicode characters + // like emojis (or even emoji modifier sequences?) + } +} |