huh whuh

4 files changed, 1467 insertions, 0 deletions

diff --git a/src/luax/lexer.rs b/src/luax/lexer.rs
new file mode 100644
index 0000000..f1021ec
--- /dev/null
+++ b/src/luax/lexer.rs
@@ -0,0 +1,426 @@
+use crate::error::LuaXError;
+use color_eyre::Result;
+
+use super::tokens::{StringType, Token};
+
+// Macro for trying to match with multiple functions
+// If a function returns a token, the token is returned
+// If a function returns an error, the error is returned
+// If a function returns None, the next function is tried
+macro_rules! try_all_paths {
+    () => {
+        TokenizeResult::None
+    };
+    ($e:expr) => {
+        $e
+    };
+    ($head:expr $(,$tail:expr)* $(,)?) => {
+        match $head {
+            TokenizeResult::Some(t) => TokenizeResult::Some(t),
+            TokenizeResult::Error(e) => TokenizeResult::Error(e),
+            TokenizeResult::None => try_all_paths!($($tail,)*),
+        }
+    };
+}
+
+pub trait TokenProducer {
+    fn next(&mut self) -> Option<Token>;
+}
+
+#[derive(Debug)]
+pub struct Lexer<'s> {
+    src: &'s str,
+    chars: std::str::Chars<'s>,
+    // Current character
+    current: Option<char>,
+    // Positioning
+    current_pos_in_bytes: usize,
+    // EOF
+    emitted_eof: bool,
+}
+
+#[derive(Debug)]
+enum TokenizeResult<'s> {
+    // A token was produced
+    Some(Token<'s>),
+    // No valid token found
+    None,
+    // An error occurred
+    Error(LuaXError),
+}
+
+impl<'s> Lexer<'s> {
+    pub fn new(src: &'s str) -> Self {
+        let mut chars = src.chars();
+        let current = chars.next();
+        Self {
+            src,
+            chars,
+            current,
+            current_pos_in_bytes: 0,
+            emitted_eof: false,
+        }
+    }
+
+    pub fn next_token(&mut self) -> Result<Option<Token<'s>>> {
+        let token = self.lex();
+
+        match token {
+            TokenizeResult::Some(token) => Ok(Some(token)),
+            TokenizeResult::Error(error) => Err(error.into()),
+            TokenizeResult::None => match self.current {
+                Some(c) => Err(LuaXError::UnexpectedCharacter(c).into()),
+                None => {
+                    if self.emitted_eof {
+                        Ok(None)
+                    } else {
+                        self.emitted_eof = true;
+                        Ok(Some(Token::Eof))
+                    }
+                }
+            },
+        }
+    }
+    
+    fn lex(&mut self) -> TokenizeResult<'s> {
+        self.skip_whitespace();
+
+        try_all_paths!(
+            self.single_char_token(),
+            self.double_char_token(),
+            self.triple_char_token(),
+            self.string(),
+            self.number(),
+            self.comment(),
+            self.identifier_or_keyword(),
+        )
+    }
+
+    fn skip_whitespace(&mut self) {
+        while let Some(c) = self.current {
+            match c {
+                ' ' | '\t' | '\n' => {
+                    self.advance();
+                }
+                _ => break,
+            }
+        }
+    }
+
+    fn comment(&mut self) -> TokenizeResult<'s> {
+        if self.match_char('-') {
+            if self.match_char('-') {
+                while let Some(c) = self.current {
+                    if c == '\n' {
+                        break;
+                    }
+                    self.advance();
+                }
+                self.lex()
+            } else {
+                TokenizeResult::Some(Token::Minus)
+            }
+        } else {
+            TokenizeResult::None
+        }
+    }
+
+    fn single_char_token_case(&mut self, c: char, kind: Token<'s>) -> TokenizeResult<'s> {
+        if self.current == Some(c) {
+            self.advance();
+            TokenizeResult::Some(kind)
+        } else {
+            TokenizeResult::None
+        }
+    }
+
+    fn single_char_token(&mut self) -> TokenizeResult<'s> {
+        try_all_paths!(
+            self.single_char_token_case('+', Token::Plus),
+            self.single_char_token_case('*', Token::Star),
+            self.single_char_token_case('/', Token::Slash),
+            self.single_char_token_case('^', Token::Hat),
+            self.single_char_token_case('%', Token::Percent),
+            self.single_char_token_case('&', Token::Amp),
+            self.single_char_token_case('|', Token::Pipe),
+            self.single_char_token_case('#', Token::Hash),
+            self.single_char_token_case('(', Token::OpenParen),
+            self.single_char_token_case(')', Token::CloseParen),
+            self.single_char_token_case('{', Token::OpenBrace),
+            self.single_char_token_case('}', Token::CloseBrace),
+            self.single_char_token_case(']', Token::CloseBracket),
+            self.single_char_token_case(';', Token::Semicolon),
+            self.single_char_token_case(',', Token::Comma),
+            self.single_char_token_case('$', Token::Dollar),
+        )
+    }
+
+    fn double_char_token_case(
+        &mut self,
+        first: char,
+        second: char,
+        short: Option<Token<'s>>,
+        long: Token<'s>,
+    ) -> TokenizeResult<'s> {
+        if self.match_char(first) {
+            if self.match_char(second) {
+                TokenizeResult::Some(long)
+            } else if let Some(short) = short {
+                TokenizeResult::Some(short)
+            } else {
+                TokenizeResult::None
+            }
+        } else {
+            TokenizeResult::None
+        }
+    }
+
+    fn double_char_token_case_with_alts<const N: usize>(
+        &mut self,
+        first: char,
+        second: [char; N],
+        short: Option<Token<'s>>,
+        long: [Token<'s>; N],
+    ) -> TokenizeResult<'s> {
+        if self.match_char(first) {
+            for i in 0..N {
+                if self.match_char(second[i]) {
+                    return TokenizeResult::Some(long[i]);
+                }
+            }
+            if let Some(short) = short {
+                TokenizeResult::Some(short)
+            } else {
+                TokenizeResult::None
+            }
+        } else {
+            TokenizeResult::None
+        }
+    }
+
+    fn double_char_token(&mut self) -> TokenizeResult<'s> {
+        try_all_paths!(
+            self.double_char_token_case_with_alts(
+                '<',
+                ['=', '<'],
+                Some(Token::Lt),
+                [Token::Le, Token::LtLt]
+            ),
+            self.double_char_token_case_with_alts(
+                '>',
+                ['=', '>'],
+                Some(Token::Gt),
+                [Token::Ge, Token::GtGt]
+            ),
+            self.double_char_token_case('=', '=', Some(Token::Eq), Token::EqEq),
+            self.double_char_token_case('~', '=', Some(Token::Tilde), Token::TildeEq),
+            self.double_char_token_case('/', '/', Some(Token::Slash), Token::SlashSlash),
+            self.double_char_token_case(':', ':', Some(Token::Colon), Token::ColonColon),
+        )
+    }
+
+    fn triple_char_token_case(
+        &mut self,
+        first: char,
+        second: char,
+        third: char,
+        short: Token<'s>,
+        mid: Token<'s>,
+        long: Token<'s>,
+    ) -> TokenizeResult<'s> {
+        if self.match_char(first) {
+            if self.match_char(second) {
+                if self.match_char(third) {
+                    TokenizeResult::Some(long)
+                } else {
+                    TokenizeResult::Some(mid)
+                }
+            } else {
+                TokenizeResult::Some(short)
+            }
+        } else {
+            TokenizeResult::None
+        }
+    }
+
+    fn triple_char_token(&mut self) -> TokenizeResult<'s> {
+        try_all_paths!(self.triple_char_token_case(
+            '.',
+            '.',
+            '.',
+            Token::Dot,
+            Token::DotDot,
+            Token::DotDotDot
+        ),)
+    }
+
+    fn number(&mut self) -> TokenizeResult<'s> {
+        if !self.current.map_or(false, |c| c.is_numeric()) {
+            return TokenizeResult::None;
+        }
+
+        let start = self.current_pos_in_bytes;
+
+        while self.current.map_or(false, |c| c.is_numeric()) {
+            self.advance();
+        }
+
+        if self.match_char('.') {
+            while self.current.map_or(false, |c| c.is_numeric()) {
+                self.advance();
+            }
+        }
+
+        if self.match_char('e') || self.match_char('E') {
+            if self.match_char('-') || self.match_char('+') {
+                self.advance();
+            }
+            while self.current.map_or(false, |c| c.is_numeric()) {
+                self.advance();
+            }
+        }
+
+        let end = self.current_pos_in_bytes;
+
+        TokenizeResult::Some(Token::Number(self.src[start..end].parse().unwrap()))
+    }
+
+    fn string(&mut self) -> TokenizeResult<'s> {
+        let ty = if self.match_char('"') {
+            StringType::Double
+        } else if self.match_char('\'') {
+            StringType::Single
+        } else if self.match_char('[') {
+            if self.match_char('[') {
+                StringType::DoubleBracket
+            } else {
+                return TokenizeResult::Some(Token::OpenBracket);
+            }
+        } else {
+            return TokenizeResult::None;
+        };
+
+        let start = self.current_pos_in_bytes;
+
+        match ty {
+            StringType::Single => {
+                let mut escaped = false;
+                while self.current.is_some() {
+                    if self.match_char('\\') {
+                        escaped = true;
+                    } else if self.match_char('\'') && !escaped {
+                        break;
+                    } else {
+                        escaped = false;
+                    }
+                    self.advance();
+                }
+            }
+            StringType::Double => {
+                let mut escaped = false;
+                while self.current.is_some() {
+                    if self.match_char('\\') {
+                        escaped = true;
+                    } else if self.match_char('"') && !escaped {
+                        break;
+                    } else {
+                        escaped = false;
+                    }
+                    self.advance();
+                }
+            }
+            StringType::DoubleBracket => {
+                let mut almost_close = false;
+                while self.current.is_some() {
+                    if self.current == Some(']') {
+                        if almost_close {
+                            break;
+                        } else {
+                            almost_close = true;
+                        }
+                    } else {
+                        almost_close = false;
+                    }
+                    self.advance();
+                }
+            }
+        }
+
+        if self.current.is_none() {
+            TokenizeResult::Error(LuaXError::UnterminatedStringLiteral)
+        } else {
+            let end = self.current_pos_in_bytes - 1;
+            if ty == StringType::DoubleBracket {
+                self.advance();
+            }
+
+            TokenizeResult::Some(Token::String(&self.src[start..end], ty))
+        }
+    }
+
+    fn identifier_or_keyword(&mut self) -> TokenizeResult<'s> {
+        if self.current.is_none() || !Self::is_valid_identifier_start(self.current.unwrap()) {
+            return TokenizeResult::None;
+        }
+
+        let start = self.current_pos_in_bytes;
+        self.advance();
+
+        while self.current.is_some() && Self::is_valid_in_identifier(self.current.unwrap()) {
+            self.advance();
+        }
+
+        let end = self.current_pos_in_bytes;
+
+        TokenizeResult::Some(match &self.src[start..end] {
+            "and" => Token::And,
+            "break" => Token::Break,
+            "do" => Token::Do,
+            "else" => Token::Else,
+            "elseif" => Token::ElseIf,
+            "end" => Token::End,
+            "false" => Token::False,
+            "for" => Token::For,
+            "function" => Token::Function,
+            "goto" => Token::Goto,
+            "if" => Token::If,
+            "in" => Token::In,
+            "local" => Token::Local,
+            "nil" => Token::Nil,
+            "not" => Token::Not,
+            "or" => Token::Or,
+            "repeat" => Token::Repeat,
+            "return" => Token::Return,
+            "then" => Token::Then,
+            "true" => Token::True,
+            "until" => Token::Until,
+            "while" => Token::While,
+            _ => Token::Identifier(&self.src[start..end]),
+        })
+    }
+
+    fn advance(&mut self) -> Option<char> {
+        self.current = self.chars.next();
+        self.current_pos_in_bytes += self.current.map_or(0, |c| c.len_utf8());
+
+        self.current
+    }
+
+    fn match_char(&mut self, c: char) -> bool {
+        if self.current == Some(c) {
+            self.advance();
+            true
+        } else {
+            false
+        }
+    }
+
+    fn is_valid_in_identifier(c: char) -> bool {
+        Self::is_valid_identifier_start(c) || c.is_numeric()
+    }
+
+    fn is_valid_identifier_start(c: char) -> bool {
+        c.is_alphabetic() || c == '_' // TODO: Support some more unicode characters
+                                      //       like emojis (or even emoji modifier sequences?)
+    }
+}
diff --git a/src/luax/mod.rs b/src/luax/mod.rs
new file mode 100644
index 0000000..a7eb38f
--- /dev/null
+++ b/src/luax/mod.rs
@@ -0,0 +1,101 @@
+use crate::Result;
+
+mod lexer;
+mod preprocessor;
+mod tokens;
+
+#[derive(Debug)]
+pub enum Node<'s> {
+    Code(&'s str),
+    Text(&'s str),
+    Element(Element<'s>),
+}
+
+impl<'s> Node<'s> {
+    pub fn to_table<W: std::io::Write>(&self, f: &mut W) -> Result<()> {
+        match &self {
+            Node::Code(s) => write!(f, "{}", s)?,
+            Node::Text(s) => write!(f, "\"{}\"", s)?,
+            Node::Element(element) => {
+                write!(f, "{{")?;
+                write!(f, "tag=\"{}\", ", element.tag)?;
+                for attribute in &element.attributes {
+                    match &attribute.value {
+                        Value::Literal(s) => write!(f, "{}=\"{}\", ", attribute.key, s)?,
+                        Value::Expression(s) => write!(f, "{}={}, ", attribute.key, s)?,
+                    }
+                }
+                write!(f, "children={{")?;
+                for child in &element.children {
+                    child.to_table(f)?;
+                }
+                write!(f, "}}")?;
+                write!(f, "}}")?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub struct Element<'s> {
+    pub tag: &'s str,
+    pub attributes: Vec<Attribute<'s>>,
+    pub children: Vec<Node<'s>>,
+}
+
+#[derive(Debug)]
+pub struct Attribute<'s> {
+    pub key: &'s str,
+    pub value: Value<'s>,
+}
+
+#[derive(Debug)]
+pub enum Value<'s> {
+    Literal(&'s str),
+    Expression(&'s str),
+}
+
+#[derive(Debug)]
+pub struct Template<'s> {
+    pub root: Node<'s>,
+}
+
+impl<'s> Template<'s> {
+    pub fn to_table<W: std::io::Write>(&self, f: &mut W) -> Result<()> {
+        self.root.to_table(f)
+    }
+}
+
+pub fn preprocess(s: &str) -> Result<String> {
+    let mut buf = Vec::new();
+    let preprocessor = preprocessor::Preprocessor::new(s, &mut buf)?;
+
+    preprocessor.preprocess()?;
+
+    let s = String::from_utf8(buf).unwrap();
+
+    Ok(s)
+}
+
+pub fn preprocess_dir(path: &std::path::Path) -> Result<()> {
+    for entry in std::fs::read_dir(path)? {
+        let entry = entry?;
+        let path = entry.path();
+        if path.is_dir() {
+            preprocess_dir(&path)?;
+        } else {
+            if path.extension().unwrap_or_default() != "luax" {
+                continue;
+            }
+            let s = std::fs::read_to_string(&path)?;
+            let s = preprocess(&s)?;
+
+            let out_path = path.with_extension("lua");
+
+            std::fs::write(out_path, s)?;
+        }
+    }
+
+    Ok(())
+}
diff --git a/src/luax/preprocessor.rs b/src/luax/preprocessor.rs
new file mode 100644
index 0000000..4934fe4
--- /dev/null
+++ b/src/luax/preprocessor.rs
@@ -0,0 +1,793 @@
+use std::io::Write;
+
+use crate::error::LuaXError;
+
+use crate::luax::lexer::Lexer;
+use crate::luax::tokens::Token;
+
+use color_eyre::Result;
+
+/// Macro for trying to get a match from a list of functions
+/// If a function returns a value, the token is returned
+/// If a function returns an error, the error is returned
+/// If a function returns the `InvalidStart` error, the next function is tried
+macro_rules! alternatives {
+    () => {
+        Err(LuaXError::InvalidStart)
+    };
+    ($e:expr) => {
+        $e
+    };
+    ($head:expr $(,$tail:expr)* $(,)?) => {
+        match $head {
+            Ok(t) => Ok(t),
+            Err(e) => {
+                match e.downcast_ref::<LuaXError>() {
+                    Some(LuaXError::InvalidStart) => alternatives!($($tail),*),
+                    _ => Err(e),
+                }
+            }
+        }
+    };
+}
+
+macro_rules! repeat_until_not {
+    ($e:expr) => {
+        loop {
+            match $e {
+                Ok(()) => {}
+                Err(e) => match e.downcast_ref::<LuaXError>() {
+                    Some(LuaXError::InvalidStart) => break,
+                    _ => return Err(e),
+                },
+            }
+        }
+    };
+}
+
+macro_rules! optionally {
+    ($e:expr) => {
+        match $e {
+            Ok(t) => Some(t),
+            Err(e) => match e.downcast_ref::<LuaXError>() {
+                Some(LuaXError::InvalidStart) => None,
+                _ => return Err(e),
+            },
+        }
+    };
+}
+
+macro_rules! require {
+    ($e:expr, $err:expr $(,)?) => {
+        match $e {
+            Ok(t) => t,
+            Err(e) => match e.downcast_ref::<LuaXError>() {
+                Some(LuaXError::InvalidStart) => return Err($err.into()),
+                _ => return Err(e),
+            },
+        }
+    };
+}
+
+pub struct Preprocessor<'s, W: Write> {
+    lexer: Lexer<'s>,
+    current: Token<'s>,
+    out_stream: W,
+}
+
+impl<'s, W: Write> Preprocessor<'s, W> {
+    pub fn new(template: &'s str, mut out_stream: W) -> Result<Self> {
+        let mut lexer = Lexer::new(template);
+        let current = lexer.next_token()?.unwrap();
+        Ok(Preprocessor {
+            lexer,
+            current,
+            out_stream,
+        })
+    }
+
+    fn next_token(&mut self) -> Result<()> {
+        if self.current != Token::Eof {
+            write!(self.out_stream, " {}", self.current)?;
+        }
+        println!("Current: {}", self.current);
+        match self.lexer.next_token()? {
+            Some(token) => {
+                self.current = token;
+            }
+            None => return Err(LuaXError::InvalidStart.into()),
+        }
+        Ok(())
+    }
+
+    fn match_token(&mut self, token: Token<'s>) -> Result<bool> {
+        if self.current == token {
+            self.next_token()?;
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    fn consume_token(&mut self, token: Token<'s>, if_not: LuaXError) -> Result<()> {
+        if self.current == token {
+            self.next_token()?;
+            Ok(())
+        } else {
+            Err(if_not.into())
+        }
+    }
+
+    fn next_token_silent(&mut self) -> Result<()> {
+        match self.lexer.next_token()? {
+            Some(token) => {
+                self.current = token;
+                println!("Current: {}", token);
+            }
+            None => return Err(LuaXError::InvalidStart.into()),
+        }
+        Ok(())
+    }
+
+    fn match_token_silent(&mut self, token: Token<'s>) -> Result<bool> {
+        if self.current == token {
+            self.next_token_silent()?;
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+
+    fn consume_token_silent(&mut self, token: Token<'s>, if_not: LuaXError) -> Result<()> {
+        if self.current == token {
+            self.next_token_silent()?;
+            Ok(())
+        } else {
+            Err(if_not.into())
+        }
+    }
+
+    pub fn preprocess(mut self) -> Result<()> {
+        self.chunk()
+    }
+
+    fn chunk(&mut self) -> Result<()> {
+        self.block()
+    }
+
+    fn block(&mut self) -> Result<()> {
+        repeat_until_not!(self.statement());
+
+        optionally!(self.return_statement());
+
+        Ok(())
+    }
+
+    fn statement(&mut self) -> Result<()> {
+        if self.match_token(Token::Semicolon)? {
+            return Ok(());
+        }
+        alternatives!(
+            self.label(),
+            self.break_statement(),
+            self.goto_statement(),
+            self.do_statement(),
+            self.while_statement(),
+            self.repeat_statement(),
+            self.if_statement(),
+            self.for_statement(),
+            self.function_statement(),
+            self.local_starting_statement(),
+            self.call_or_assignment(),
+        )
+    }
+
+    fn break_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::Break)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        Ok(())
+    }
+
+    fn goto_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::Goto)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(
+            self.identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        Ok(())
+    }
+
+    fn do_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::Do)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(self.block(), LuaXError::ExpectedExpression);
+
+        self.consume_token(Token::End, LuaXError::NeededToken(Token::End.to_string()))
+    }
+
+    fn while_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::While)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(self.expression(), LuaXError::ExpectedExpression);
+
+        self.consume_token(Token::Do, LuaXError::NeededToken(Token::Do.to_string()))?;
+
+        require!(self.block(), LuaXError::ExpectedExpression);
+
+        self.consume_token(Token::End, LuaXError::NeededToken(Token::End.to_string()))
+    }
+
+    fn repeat_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::Repeat)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(self.block(), LuaXError::ExpectedExpression);
+
+        self.consume_token(
+            Token::Until,
+            LuaXError::NeededToken(Token::Until.to_string()),
+        )?;
+
+        require!(self.expression(), LuaXError::ExpectedExpression);
+
+        Ok(())
+    }
+
+    fn if_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::If)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(self.expression(), LuaXError::ExpectedExpression);
+
+        self.consume_token(Token::Then, LuaXError::NeededToken(Token::Then.to_string()))?;
+
+        require!(self.block(), LuaXError::ExpectedExpression);
+
+        loop {
+            if !self.match_token(Token::ElseIf)? {
+                break;
+            }
+
+            require!(self.expression(), LuaXError::ExpectedExpression);
+
+            self.consume_token(Token::Then, LuaXError::NeededToken(Token::Then.to_string()))?;
+
+            require!(self.block(), LuaXError::ExpectedExpression);
+        }
+
+        if self.match_token(Token::Else)? {
+            require!(self.block(), LuaXError::ExpectedExpression);
+        }
+
+        self.consume_token(Token::End, LuaXError::NeededToken(Token::End.to_string()))
+    }
+
+    fn for_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::For)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(
+            self.identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        if self.match_token(Token::Eq)? {
+            require!(self.expression(), LuaXError::ExpectedExpression);
+
+            self.consume_token(
+                Token::Comma,
+                LuaXError::NeededToken(Token::Comma.to_string()),
+            )?;
+
+            require!(self.expression(), LuaXError::ExpectedExpression);
+
+            if self.match_token(Token::Comma)? {
+                require!(self.expression(), LuaXError::ExpectedExpression);
+            }
+
+            require!(self.expression(), LuaXError::ExpectedExpression);
+
+            if self.match_token(Token::Comma)? {
+                require!(self.expression(), LuaXError::ExpectedExpression);
+            }
+        } else {
+            loop {
+                if !self.match_token(Token::Comma)? {
+                    break;
+                }
+
+                require!(
+                    self.identifier(),
+                    LuaXError::NeededToken("identifier".to_string())
+                );
+            }
+
+            self.consume_token(Token::In, LuaXError::NeededToken(Token::In.to_string()))?;
+
+            require!(self.explist(), LuaXError::ExpectedExpression);
+        }
+
+        self.consume_token(Token::Do, LuaXError::NeededToken(Token::Do.to_string()))?;
+
+        require!(self.block(), LuaXError::ExpectedExpression);
+
+        self.consume_token(Token::End, LuaXError::NeededToken(Token::End.to_string()))
+    }
+
+    fn function_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::Function)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(
+            self.funcname(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        require!(self.funcbody(), LuaXError::ExpectedExpression);
+
+        Ok(())
+    }
+
+    fn local_starting_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::Local)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        if self.match_token(Token::Function)? {
+            require!(
+                self.identifier(),
+                LuaXError::NeededToken("identifier".to_string())
+            );
+
+            require!(self.funcbody(), LuaXError::ExpectedExpression);
+
+            return Ok(());
+        }
+
+        require!(
+            self.attribute_name_list(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        if self.match_token(Token::Eq)? {
+            require!(self.explist(), LuaXError::ExpectedExpression);
+        }
+
+        Ok(())
+    }
+
+    fn return_statement(&mut self) -> Result<()> {
+        if !self.match_token(Token::Return)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        optionally!(self.expression());
+
+        loop {
+            if !self.match_token(Token::Comma)? {
+                break;
+            }
+
+            require!(self.expression(), LuaXError::ExpectedExpression);
+        }
+
+        self.match_token(Token::Semicolon)?;
+
+        Ok(())
+    }
+
+    fn attribute(&mut self) -> Result<()> {
+        if !self.match_token(Token::Lt)? {
+            return Ok(());
+        }
+
+        require!(
+            self.identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        self.consume_token(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))
+    }
+
+    fn attribute_name_list(&mut self) -> Result<()> {
+        require!(
+            self.identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        self.attribute()?;
+
+        loop {
+            if !self.match_token(Token::Comma)? {
+                break;
+            }
+
+            require!(
+                self.identifier(),
+                LuaXError::NeededToken("identifier".to_string())
+            );
+
+            self.attribute()?;
+        }
+
+        Ok(())
+    }
+
+    fn label(&mut self) -> Result<()> {
+        if !self.match_token(Token::ColonColon)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(
+            self.identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        self.consume_token(
+            Token::ColonColon,
+            LuaXError::NeededToken(Token::ColonColon.to_string()),
+        )
+    }
+
+    fn funcname(&mut self) -> Result<()> {
+        require!(
+            self.identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        loop {
+            if !self.match_token(Token::Dot)? {
+                break;
+            }
+
+            require!(
+                self.identifier(),
+                LuaXError::NeededToken("identifier".to_string())
+            );
+        }
+
+        if self.match_token(Token::Colon)? {
+            require!(
+                self.identifier(),
+                LuaXError::NeededToken("identifier".to_string())
+            );
+        }
+
+        Ok(())
+    }
+
+    fn explist(&mut self) -> Result<()> {
+        require!(self.expression(), LuaXError::ExpectedVar,);
+
+        loop {
+            if !self.match_token(Token::Comma)? {
+                break;
+            }
+
+            require!(self.expression(), LuaXError::ExpectedVar,);
+        }
+
+        Ok(())
+    }
+
+    fn expression(&mut self) -> Result<()> {
+        alternatives!(
+            self.html_template(),
+            self.literal(),
+            self.function_def(),
+            self.access_or_call(),
+            self.table_constructor(),
+            self.unary_expression(),
+        )?;
+
+        optionally!(self.binary_expression_followup());
+
+        Ok(())
+    }
+
+    fn literal(&mut self) -> Result<()> {
+        alternatives!(
+            self.number(),
+            self.string(),
+            self.boolean(),
+            self.nil_literal(),
+        )
+    }
+
+    fn call_or_assignment(&mut self) -> Result<()> {
+        if optionally!(self.access_or_call()).is_none() {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        if self.match_token(Token::Eq)? {
+            require!(self.explist(), LuaXError::ExpectedExpression,);
+        } else if self.match_token(Token::Comma)? {
+            require!(self.access_or_call(), LuaXError::ExpectedExpression,);
+            while self.match_token(Token::Comma)? {
+                require!(self.access_or_call(), LuaXError::ExpectedExpression,);
+            }
+            if self.match_token(Token::Eq)? {
+                require!(self.explist(), LuaXError::ExpectedExpression,);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn access_or_call(&mut self) -> Result<()> {
+        if self.match_token(Token::OpenParen)? {
+            require!(self.expression(), LuaXError::ExpectedVar,);
+            self.consume_token(
+                Token::CloseParen,
+                LuaXError::NeededToken(Token::CloseParen.to_string()),
+            )?;
+        } else if optionally!(self.identifier()).is_some() {
+        } else {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        // } else if let Some(()) = optionally!(self.html_template()) {
+        //     return Ok(());
+
+        loop {
+            if self.match_token(Token::OpenBracket)? {
+                require!(self.expression(), LuaXError::ExpectedVar,);
+                self.consume_token(
+                    Token::CloseBracket,
+                    LuaXError::NeededToken(Token::CloseBracket.to_string()),
+                )?;
+            } else if self.match_token(Token::Dot)? {
+                require!(
+                    self.identifier(),
+                    LuaXError::NeededToken("identifier".to_string())
+                );
+            } else if optionally!(self.args()).is_some() {
+            } else if self.match_token(Token::Colon)? {
+                require!(
+                    self.identifier(),
+                    LuaXError::NeededToken("identifier".to_string())
+                );
+                require!(self.args(), LuaXError::ExpectedExpression,);
+            } else {
+                break;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn args(&mut self) -> Result<()> {
+        if self.match_token(Token::OpenParen)? {
+            if !self.match_token(Token::CloseParen)? {
+                require!(self.explist(), LuaXError::ExpectedExpression,);
+                self.consume_token(
+                    Token::CloseParen,
+                    LuaXError::NeededToken(Token::CloseParen.to_string()),
+                )
+            } else {
+                Ok(())
+            }
+        } else if optionally!(self.table_constructor()).is_some()
+            || optionally!(self.string()).is_some()
+        {
+            Ok(())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn function_def(&mut self) -> Result<()> {
+        if !self.match_token(Token::Function)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(self.funcbody(), LuaXError::ExpectedExpression);
+
+        Ok(())
+    }
+
+    fn funcbody(&mut self) -> Result<()> {
+        self.consume_token(
+            Token::OpenParen,
+            LuaXError::NeededToken(Token::OpenParen.to_string()),
+        )?;
+
+        optionally!(self.parlist());
+
+        self.consume_token(
+            Token::CloseParen,
+            LuaXError::NeededToken(Token::CloseParen.to_string()),
+        )?;
+
+        require!(self.block(), LuaXError::ExpectedExpression);
+
+        self.consume_token(Token::End, LuaXError::NeededToken(Token::End.to_string()))
+    }
+
+    fn parlist(&mut self) -> Result<()> {
+        if self.match_token(Token::DotDotDot)? {
+            return Ok(());
+        }
+
+        if optionally!(self.identifier()).is_some() {
+            while self.match_token(Token::Comma)? {
+                if self.match_token(Token::DotDotDot)? {
+                    return Ok(());
+                }
+
+                if optionally!(self.identifier()).is_some() {
+                    continue;
+                } else {
+                    return Err(LuaXError::ExpectedExpression.into());
+                }
+            }
+
+            Ok(())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn table_constructor(&mut self) -> Result<()> {
+        if !self.match_token(Token::OpenBrace)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        optionally!(self.fieldlist());
+
+        self.consume_token(
+            Token::CloseBrace,
+            LuaXError::NeededToken(Token::CloseBrace.to_string()),
+        )
+    }
+
+    fn fieldlist(&mut self) -> Result<()> {
+        require!(self.field(), LuaXError::ExpectedExpression);
+
+        loop {
+            if !self.match_token(Token::Comma)? && !self.match_token(Token::Semicolon)? {
+                break;
+            }
+
+            if self.match_token(Token::CloseBrace)? {
+                break;
+            }
+
+            require!(self.field(), LuaXError::ExpectedExpression);
+        }
+
+        Ok(())
+    }
+
+    fn field(&mut self) -> Result<()> {
+        if optionally!(self.identifier()).is_some() {
+            self.consume_token(Token::Eq, LuaXError::NeededToken(Token::Eq.to_string()))?;
+            require!(self.expression(), LuaXError::ExpectedExpression);
+        } else if self.match_token(Token::OpenBracket)? {
+            require!(self.expression(), LuaXError::ExpectedExpression);
+            self.consume_token(
+                Token::CloseBracket,
+                LuaXError::NeededToken(Token::CloseBracket.to_string()),
+            )?;
+            self.consume_token(Token::Eq, LuaXError::NeededToken(Token::Eq.to_string()))?;
+            require!(self.expression(), LuaXError::ExpectedExpression);
+        } else {
+            require!(self.expression(), LuaXError::ExpectedExpression);
+        }
+
+        Ok(())
+    }
+
+    fn binary_expression_followup(&mut self) -> Result<()> {
+        loop {
+            if !self.match_token(Token::Plus)?
+                && !self.match_token(Token::Minus)?
+                && !self.match_token(Token::Star)?
+                && !self.match_token(Token::Slash)?
+                && !self.match_token(Token::SlashSlash)?
+                && !self.match_token(Token::Hat)?
+                && !self.match_token(Token::Percent)?
+                && !self.match_token(Token::Amp)?
+                && !self.match_token(Token::Tilde)?
+                && !self.match_token(Token::Pipe)?
+                && !self.match_token(Token::LtLt)?
+                && !self.match_token(Token::GtGt)?
+                && !self.match_token(Token::DotDot)?
+                && !self.match_token(Token::Lt)?
+                && !self.match_token(Token::Le)?
+                && !self.match_token(Token::Gt)?
+                && !self.match_token(Token::Ge)?
+                && !self.match_token(Token::EqEq)?
+                && !self.match_token(Token::TildeEq)?
+                && !self.match_token(Token::And)?
+                && !self.match_token(Token::Or)?
+            {
+                break;
+            }
+
+            require!(self.expression(), LuaXError::ExpectedExpression);
+        }
+
+        Ok(())
+    }
+
+    fn unary_expression(&mut self) -> Result<()> {
+        if !self.match_token(Token::Not)?
+            && !self.match_token(Token::Hash)?
+            && !self.match_token(Token::Minus)?
+            && !self.match_token(Token::Tilde)?
+        {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        require!(self.expression(), LuaXError::ExpectedExpression);
+
+        Ok(())
+    }
+
+    fn boolean(&mut self) -> Result<()> {
+        if self.match_token(Token::True)? || self.match_token(Token::False)? {
+            Ok(())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn nil_literal(&mut self) -> Result<()> {
+        if self.match_token(Token::Nil)? {
+            Ok(())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn number(&mut self) -> Result<()> {
+        if let Token::Number(_) = self.current {
+            self.next_token()?;
+            Ok(())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn string(&mut self) -> Result<()> {
+        if let Token::String(..) = self.current {
+            self.next_token()?;
+            Ok(())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn identifier(&mut self) -> Result<String> {
+        if let Token::Identifier(s) = self.current {
+            self.next_token()?;
+            Ok(s.to_string())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn html_template(&mut self) -> Result<()> {
+        if self.match_token_silent(Token::Dollar)? {
+            write!(self.out_stream, " {{ 'dollar' }}")?;
+            Ok(())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+}
diff --git a/src/luax/tokens.rs b/src/luax/tokens.rs
new file mode 100644
index 0000000..9f1058a
--- /dev/null
+++ b/src/luax/tokens.rs
@@ -0,0 +1,147 @@
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum StringType {
+    Single,
+    Double,
+    DoubleBracket,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum Token<'s> {
+    /// Keywords
+    Break,
+    Goto,
+    Do,
+    End,
+    While,
+    Repeat,
+    Until,
+    If,
+    Then,
+    ElseIf,
+    Else,
+    For,
+    In,
+    Function,
+    Local,
+    Return,
+    True,
+    False,
+    Nil,
+    /// Symbols
+    Semicolon,
+    Eq,
+    Comma,
+    ColonColon,
+    Dot,
+    Colon,
+    OpenBracket,
+    CloseBracket,
+    DotDotDot,
+    OpenParen,
+    CloseParen,
+    OpenBrace,
+    CloseBrace,
+    /// Operators
+    Plus,
+    Minus,
+    Star,
+    Slash,
+    SlashSlash,
+    Hat,
+    Percent,
+    Amp,
+    Tilde,
+    Pipe,
+    LtLt,
+    GtGt,
+    DotDot,
+    Lt,
+    Le,
+    Gt,
+    Ge,
+    EqEq,
+    TildeEq,
+    And,
+    Or,
+    Hash,
+    Not,
+    /// Containers
+    Number(f64),
+    String(&'s str, StringType),
+    Identifier(&'s str),
+
+    Dollar,
+    Eof,
+}
+
+impl std::fmt::Display for Token<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            Token::Break => write!(f, "break"),
+            Token::Goto => write!(f, "goto"),
+            Token::Do => write!(f, "do"),
+            Token::End => write!(f, "end"),
+            Token::While => write!(f, "while"),
+            Token::Repeat => write!(f, "repeat"),
+            Token::Until => write!(f, "until"),
+            Token::If => write!(f, "if"),
+            Token::Then => write!(f, "then"),
+            Token::ElseIf => write!(f, "elseif"),
+            Token::Else => write!(f, "else"),
+            Token::For => write!(f, "for"),
+            Token::In => write!(f, "in"),
+            Token::Function => write!(f, "function"),
+            Token::Local => write!(f, "local"),
+            Token::Return => write!(f, "return"),
+            Token::True => write!(f, "true"),
+            Token::False => write!(f, "false"),
+            Token::Nil => write!(f, "nil"),
+            Token::Semicolon => write!(f, ";"),
+            Token::Eq => write!(f, "="),
+            Token::Comma => write!(f, ","),
+            Token::ColonColon => write!(f, "::"),
+            Token::Dot => write!(f, "."),
+            Token::Colon => write!(f, ":"),
+            Token::OpenBracket => write!(f, "["),
+            Token::CloseBracket => write!(f, "]"),
+            Token::DotDotDot => write!(f, "..."),
+            Token::OpenParen => write!(f, "("),
+            Token::CloseParen => write!(f, ")"),
+            Token::OpenBrace => write!(f, "{{"),
+            Token::CloseBrace => write!(f, "}}"),
+            Token::Plus => write!(f, "+"),
+            Token::Minus => write!(f, "-"),
+            Token::Star => write!(f, "*"),
+            Token::Slash => write!(f, "/"),
+            Token::SlashSlash => write!(f, "//"),
+            Token::Hat => write!(f, "^"),
+            Token::Percent => write!(f, "%"),
+            Token::Amp => write!(f, "&"),
+            Token::Tilde => write!(f, "~"),
+            Token::Pipe => write!(f, "|"),
+            Token::LtLt => write!(f, "<<"),
+            Token::GtGt => write!(f, ">>"),
+            Token::DotDot => write!(f, ".."),
+            Token::Lt => write!(f, "<"),
+            Token::Le => write!(f, "<="),
+            Token::Gt => write!(f, ">"),
+            Token::Ge => write!(f, ">="),
+            Token::EqEq => write!(f, "=="),
+            Token::TildeEq => write!(f, "~="),
+            Token::And => write!(f, "and"),
+            Token::Or => write!(f, "or"),
+            Token::Hash => write!(f, "#"),
+            Token::Not => write!(f, "not"),
+            Token::Number(n) => write!(f, "{}", n),
+            Token::String(s, st) => match st {
+                StringType::Single => write!(f, "'{}'", s),
+                StringType::Double => write!(f, "\"{}\"", s),
+                StringType::DoubleBracket => write!(f, "[[{}]]", s),
+            },
+            Token::Identifier(s) => write!(f, "{}", s),
+            Token::Dollar => write!(f, "$"),
+
+            Token::Eof => write!(f, "EOF"),
+        }
+    }
+}