luax

9 files changed, 649 insertions, 113 deletions

diff --git a/src/error.rs b/src/error.rs
index 39c8cce..51803c7 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -4,16 +4,13 @@ use std::fmt::{self, Display, Formatter};
 pub enum LuaXError {
     InvalidStart, // used internally
     MissingField(String),
-    WrongFieldType(String),
     NonTableChildren,
+    NonTableAttrs,
     UnexpectedCharacter(char),
-    UnexpectedToken(String),
     NeededToken(String),
     ExpectedVar,
     ExpectedExpression,
     UnterminatedStringLiteral,
-    InvalidEscapeSequence(char),
-    InvalidAssignmentTarget,
 }
 
 impl std::error::Error for LuaXError {}
@@ -23,16 +20,13 @@ impl Display for LuaXError {
         match self {
             LuaXError::InvalidStart => write!(f, "Invalid start"),
             LuaXError::MissingField(field) => write!(f, "Missing required field: {}", field),
-            LuaXError::WrongFieldType(field) => write!(f, "Wrong field type: field {}", field),
             LuaXError::NonTableChildren => write!(f, "Children must be tables"),
+            LuaXError::NonTableAttrs => write!(f, "Attrs must be tables"),
             LuaXError::UnexpectedCharacter(c) => write!(f, "Unexpected character: '{}'", c),
-            LuaXError::UnexpectedToken(token) => write!(f, "Unexpected token: {}", token),
             LuaXError::NeededToken(token) => write!(f, "Needed token: {}", token),
             LuaXError::ExpectedVar => write!(f, "Expected variable"),
             LuaXError::ExpectedExpression => write!(f, "Expected expression"),
             LuaXError::UnterminatedStringLiteral => write!(f, "Unterminated string literal"),
-            LuaXError::InvalidEscapeSequence(c) => write!(f, "Invalid escape sequence: {}", c),
-            LuaXError::InvalidAssignmentTarget => write!(f, "Invalid assignment target"),
         }
     }
 }
diff --git a/src/luax/lexer.rs b/src/luax/lexer.rs
index f1021ec..7d4f20a 100644
--- a/src/luax/lexer.rs
+++ b/src/luax/lexer.rs
@@ -37,6 +37,8 @@ pub struct Lexer<'s> {
     current_pos_in_bytes: usize,
     // EOF
     emitted_eof: bool,
+    allow_unknowns: bool,
+    emit_whitespace: bool,
 }
 
 #[derive(Debug)]
@@ -59,6 +61,8 @@ impl<'s> Lexer<'s> {
             current,
             current_pos_in_bytes: 0,
             emitted_eof: false,
+            allow_unknowns: false,
+            emit_whitespace: false,
         }
     }
 
@@ -69,7 +73,17 @@ impl<'s> Lexer<'s> {
             TokenizeResult::Some(token) => Ok(Some(token)),
             TokenizeResult::Error(error) => Err(error.into()),
             TokenizeResult::None => match self.current {
-                Some(c) => Err(LuaXError::UnexpectedCharacter(c).into()),
+                Some(c) => {
+                    if self.allow_unknowns {
+                        self.advance();
+                        match c {
+                            ' ' | '\t' | '\n' => Ok(Some(Token::Whitespace)),
+                            _ => Ok(Some(Token::Unknown(c))),
+                        }
+                    } else {
+                        Err(LuaXError::UnexpectedCharacter(c).into())
+                    }
+                }
                 None => {
                     if self.emitted_eof {
                         Ok(None)
@@ -81,9 +95,27 @@ impl<'s> Lexer<'s> {
             },
         }
     }
-    
+
+    pub fn allow_unknowns(&mut self) {
+        self.allow_unknowns = true;
+    }
+
+    pub fn disallow_unknowns(&mut self) {
+        self.allow_unknowns = false;
+    }
+
+    pub fn emit_whitespace(&mut self) {
+        self.emit_whitespace = true;
+    }
+
+    pub fn hide_whitespace(&mut self) {
+        self.emit_whitespace = false;
+    }
+
     fn lex(&mut self) -> TokenizeResult<'s> {
-        self.skip_whitespace();
+        if !self.emit_whitespace {
+            self.skip_whitespace();
+        }
 
         try_all_paths!(
             self.single_char_token(),
@@ -138,7 +170,6 @@ impl<'s> Lexer<'s> {
         try_all_paths!(
             self.single_char_token_case('+', Token::Plus),
             self.single_char_token_case('*', Token::Star),
-            self.single_char_token_case('/', Token::Slash),
             self.single_char_token_case('^', Token::Hat),
             self.single_char_token_case('%', Token::Percent),
             self.single_char_token_case('&', Token::Amp),
@@ -146,12 +177,10 @@ impl<'s> Lexer<'s> {
             self.single_char_token_case('#', Token::Hash),
             self.single_char_token_case('(', Token::OpenParen),
             self.single_char_token_case(')', Token::CloseParen),
-            self.single_char_token_case('{', Token::OpenBrace),
             self.single_char_token_case('}', Token::CloseBrace),
             self.single_char_token_case(']', Token::CloseBracket),
             self.single_char_token_case(';', Token::Semicolon),
             self.single_char_token_case(',', Token::Comma),
-            self.single_char_token_case('$', Token::Dollar),
         )
     }
 
@@ -202,9 +231,9 @@ impl<'s> Lexer<'s> {
         try_all_paths!(
             self.double_char_token_case_with_alts(
                 '<',
-                ['=', '<'],
+                ['=', '<', '/'],
                 Some(Token::Lt),
-                [Token::Le, Token::LtLt]
+                [Token::Le, Token::LtLt, Token::OpenClosingTag]
             ),
             self.double_char_token_case_with_alts(
                 '>',
@@ -212,6 +241,8 @@ impl<'s> Lexer<'s> {
                 Some(Token::Gt),
                 [Token::Ge, Token::GtGt]
             ),
+            self.double_char_token_case('{', '$', Some(Token::OpenBrace), Token::LuaStart),
+            self.double_char_token_case('$', '}', None, Token::LuaEnd),
             self.double_char_token_case('=', '=', Some(Token::Eq), Token::EqEq),
             self.double_char_token_case('~', '=', Some(Token::Tilde), Token::TildeEq),
             self.double_char_token_case('/', '/', Some(Token::Slash), Token::SlashSlash),
@@ -279,10 +310,9 @@ impl<'s> Lexer<'s> {
                 self.advance();
             }
         }
-
         let end = self.current_pos_in_bytes;
 
-        TokenizeResult::Some(Token::Number(self.src[start..end].parse().unwrap()))
+        TokenizeResult::Some(Token::Number(&self.src[start..end]))
     }
 
     fn string(&mut self) -> TokenizeResult<'s> {
@@ -301,6 +331,7 @@ impl<'s> Lexer<'s> {
         };
 
         let start = self.current_pos_in_bytes;
+        let mut finished = false;
 
         match ty {
             StringType::Single => {
@@ -309,6 +340,7 @@ impl<'s> Lexer<'s> {
                     if self.match_char('\\') {
                         escaped = true;
                     } else if self.match_char('\'') && !escaped {
+                        finished = true;
                         break;
                     } else {
                         escaped = false;
@@ -322,6 +354,7 @@ impl<'s> Lexer<'s> {
                     if self.match_char('\\') {
                         escaped = true;
                     } else if self.match_char('"') && !escaped {
+                        finished = true;
                         break;
                     } else {
                         escaped = false;
@@ -334,6 +367,7 @@ impl<'s> Lexer<'s> {
                 while self.current.is_some() {
                     if self.current == Some(']') {
                         if almost_close {
+                            finished = true;
                             break;
                         } else {
                             almost_close = true;
@@ -346,7 +380,7 @@ impl<'s> Lexer<'s> {
             }
         }
 
-        if self.current.is_none() {
+        if !finished {
             TokenizeResult::Error(LuaXError::UnterminatedStringLiteral)
         } else {
             let end = self.current_pos_in_bytes - 1;
@@ -400,8 +434,8 @@ impl<'s> Lexer<'s> {
     }
 
     fn advance(&mut self) -> Option<char> {
-        self.current = self.chars.next();
         self.current_pos_in_bytes += self.current.map_or(0, |c| c.len_utf8());
+        self.current = self.chars.next();
 
         self.current
     }
diff --git a/src/luax/mod.rs b/src/luax/mod.rs
index a7eb38f..df9f57d 100644
--- a/src/luax/mod.rs
+++ b/src/luax/mod.rs
@@ -2,71 +2,10 @@ use crate::Result;
 
 mod lexer;
 mod preprocessor;
+#[cfg(test)]
+mod tests;
 mod tokens;
 
-#[derive(Debug)]
-pub enum Node<'s> {
-    Code(&'s str),
-    Text(&'s str),
-    Element(Element<'s>),
-}
-
-impl<'s> Node<'s> {
-    pub fn to_table<W: std::io::Write>(&self, f: &mut W) -> Result<()> {
-        match &self {
-            Node::Code(s) => write!(f, "{}", s)?,
-            Node::Text(s) => write!(f, "\"{}\"", s)?,
-            Node::Element(element) => {
-                write!(f, "{{")?;
-                write!(f, "tag=\"{}\", ", element.tag)?;
-                for attribute in &element.attributes {
-                    match &attribute.value {
-                        Value::Literal(s) => write!(f, "{}=\"{}\", ", attribute.key, s)?,
-                        Value::Expression(s) => write!(f, "{}={}, ", attribute.key, s)?,
-                    }
-                }
-                write!(f, "children={{")?;
-                for child in &element.children {
-                    child.to_table(f)?;
-                }
-                write!(f, "}}")?;
-                write!(f, "}}")?;
-            }
-        }
-        Ok(())
-    }
-}
-
-#[derive(Debug)]
-pub struct Element<'s> {
-    pub tag: &'s str,
-    pub attributes: Vec<Attribute<'s>>,
-    pub children: Vec<Node<'s>>,
-}
-
-#[derive(Debug)]
-pub struct Attribute<'s> {
-    pub key: &'s str,
-    pub value: Value<'s>,
-}
-
-#[derive(Debug)]
-pub enum Value<'s> {
-    Literal(&'s str),
-    Expression(&'s str),
-}
-
-#[derive(Debug)]
-pub struct Template<'s> {
-    pub root: Node<'s>,
-}
-
-impl<'s> Template<'s> {
-    pub fn to_table<W: std::io::Write>(&self, f: &mut W) -> Result<()> {
-        self.root.to_table(f)
-    }
-}
-
 pub fn preprocess(s: &str) -> Result<String> {
     let mut buf = Vec::new();
     let preprocessor = preprocessor::Preprocessor::new(s, &mut buf)?;
diff --git a/src/luax/preprocessor.rs b/src/luax/preprocessor.rs
index 4934fe4..ba51a44 100644
--- a/src/luax/preprocessor.rs
+++ b/src/luax/preprocessor.rs
@@ -73,24 +73,29 @@ pub struct Preprocessor<'s, W: Write> {
     lexer: Lexer<'s>,
     current: Token<'s>,
     out_stream: W,
+    first_token: bool,
 }
 
 impl<'s, W: Write> Preprocessor<'s, W> {
-    pub fn new(template: &'s str, mut out_stream: W) -> Result<Self> {
+    pub fn new(template: &'s str, out_stream: W) -> Result<Self> {
         let mut lexer = Lexer::new(template);
         let current = lexer.next_token()?.unwrap();
         Ok(Preprocessor {
             lexer,
             current,
             out_stream,
+            first_token: true,
         })
     }
 
     fn next_token(&mut self) -> Result<()> {
         if self.current != Token::Eof {
-            write!(self.out_stream, " {}", self.current)?;
+            if !self.first_token {
+                write!(self.out_stream, " ")?;
+            }
+            write!(self.out_stream, "{}", self.current)?;
+            self.first_token = false;
         }
-        println!("Current: {}", self.current);
         match self.lexer.next_token()? {
             Some(token) => {
                 self.current = token;
@@ -122,7 +127,6 @@ impl<'s, W: Write> Preprocessor<'s, W> {
         match self.lexer.next_token()? {
             Some(token) => {
                 self.current = token;
-                println!("Current: {}", token);
             }
             None => return Err(LuaXError::InvalidStart.into()),
         }
@@ -297,12 +301,6 @@ impl<'s, W: Write> Preprocessor<'s, W> {
             if self.match_token(Token::Comma)? {
                 require!(self.expression(), LuaXError::ExpectedExpression);
             }
-
-            require!(self.expression(), LuaXError::ExpectedExpression);
-
-            if self.match_token(Token::Comma)? {
-                require!(self.expression(), LuaXError::ExpectedExpression);
-            }
         } else {
             loop {
                 if !self.match_token(Token::Comma)? {
@@ -543,6 +541,10 @@ impl<'s, W: Write> Preprocessor<'s, W> {
         // } else if let Some(()) = optionally!(self.html_template()) {
         //     return Ok(());
 
+        self.continue_access_or_call()
+    }
+
+    fn continue_access_or_call(&mut self) -> Result<()> {
         loop {
             if self.match_token(Token::OpenBracket)? {
                 require!(self.expression(), LuaXError::ExpectedVar,);
@@ -656,7 +658,9 @@ impl<'s, W: Write> Preprocessor<'s, W> {
     }
 
     fn fieldlist(&mut self) -> Result<()> {
-        require!(self.field(), LuaXError::ExpectedExpression);
+        if optionally!(self.field()).is_none() {
+            return Ok(());
+        }
 
         loop {
             if !self.match_token(Token::Comma)? && !self.match_token(Token::Semicolon)? {
@@ -675,8 +679,11 @@ impl<'s, W: Write> Preprocessor<'s, W> {
 
     fn field(&mut self) -> Result<()> {
         if optionally!(self.identifier()).is_some() {
-            self.consume_token(Token::Eq, LuaXError::NeededToken(Token::Eq.to_string()))?;
-            require!(self.expression(), LuaXError::ExpectedExpression);
+            if self.match_token(Token::Eq)? {
+                require!(self.expression(), LuaXError::ExpectedExpression);
+            } else {
+                self.continue_access_or_call()?;
+            }
         } else if self.match_token(Token::OpenBracket)? {
             require!(self.expression(), LuaXError::ExpectedExpression);
             self.consume_token(
@@ -685,8 +692,8 @@ impl<'s, W: Write> Preprocessor<'s, W> {
             )?;
             self.consume_token(Token::Eq, LuaXError::NeededToken(Token::Eq.to_string()))?;
             require!(self.expression(), LuaXError::ExpectedExpression);
-        } else {
-            require!(self.expression(), LuaXError::ExpectedExpression);
+        } else if optionally!(self.expression()).is_none() {
+            return Err(LuaXError::InvalidStart.into());
         }
 
         Ok(())
@@ -782,12 +789,160 @@ impl<'s, W: Write> Preprocessor<'s, W> {
         }
     }
 
-    fn html_template(&mut self) -> Result<()> {
-        if self.match_token_silent(Token::Dollar)? {
-            write!(self.out_stream, " {{ 'dollar' }}")?;
-            Ok(())
+    fn html_string(&mut self) -> Result<String> {
+        if let Token::String(s, _) = self.current {
+            self.next_token_silent()?;
+            Ok(s.to_string())
+        } else {
+            Err(LuaXError::InvalidStart.into())
+        }
+    }
+
+    fn html_identifier(&mut self) -> Result<String> {
+        if let Token::Identifier(s) = self.current {
+            self.next_token_silent()?;
+            Ok(s.to_string())
         } else {
             Err(LuaXError::InvalidStart.into())
         }
     }
+
+    fn html_template(&mut self) -> Result<()> {
+        if !self.match_token_silent(Token::Lt)? {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        self.lexer.allow_unknowns();
+
+        let tag = require!(
+            self.html_identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        write!(self.out_stream, " {{ tag=\"{}\", ", tag)?;
+
+        self.html_attributes()?;
+
+        if self.match_token_silent(Token::Slash)? {
+            self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))?;
+            write!(self.out_stream, "children={{}} }}")?;
+            return Ok(());
+        }
+
+        self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))?;
+
+        self.html_children()?;
+
+        self.consume_token_silent(
+            Token::OpenClosingTag,
+            LuaXError::NeededToken(Token::OpenClosingTag.to_string()),
+        )?;
+
+        let closing_tag = require!(
+            self.html_identifier(),
+            LuaXError::NeededToken("identifier".to_string())
+        );
+
+        if closing_tag != tag {
+            return Err(LuaXError::InvalidStart.into());
+        }
+
+        self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))?;
+
+        write!(self.out_stream, " }}")?;
+
+        self.lexer.disallow_unknowns();
+
+        Ok(())
+    }
+
+    fn html_attributes(&mut self) -> Result<()> {
+        write!(self.out_stream, "attrs={{")?;
+        loop {
+            let key = optionally!(self.html_identifier());
+
+            if key.is_none() {
+                break;
+            }
+
+            let key = key.unwrap();
+
+            self.consume_token_silent(Token::Eq, LuaXError::NeededToken(Token::Eq.to_string()))?;
+
+            write!(self.out_stream, "{}=", key)?;
+
+            if self.match_token_silent(Token::OpenBrace)? {
+                require!(self.expression(), LuaXError::ExpectedExpression);
+                self.consume_token_silent(
+                    Token::CloseBrace,
+                    LuaXError::NeededToken(Token::CloseBrace.to_string()),
+                )?;
+            } else {
+                let value = require!(
+                    self.html_string(),
+                    LuaXError::NeededToken("string".to_string())
+                );
+
+                write!(self.out_stream, "\"{}\"", value)?;
+            }
+
+            write!(self.out_stream, ", ")?;
+        }
+        write!(self.out_stream, "}}, ")?;
+
+        Ok(())
+    }
+
+    fn html_children(&mut self) -> Result<()> {
+        write!(self.out_stream, "children={{")?;
+        loop {
+            if self.current == Token::OpenClosingTag {
+                break;
+            }
+            if self.match_token_silent(Token::LuaStart)? {
+                self.lexer.disallow_unknowns();
+                require!(self.expression(), LuaXError::ExpectedExpression);
+                self.lexer.allow_unknowns();
+                self.consume_token_silent(
+                    Token::LuaEnd,
+                    LuaXError::NeededToken(Token::LuaEnd.to_string()),
+                )?;
+                write!(self.out_stream, ",")?;
+                continue;
+            }
+
+            if optionally!(self.html_template()).is_some() {
+                write!(self.out_stream, ",")?;
+                continue;
+            }
+
+            if self.current == Token::Lt
+                || self.current == Token::OpenClosingTag
+                || self.current == Token::LuaStart
+            {
+                break;
+            }
+
+            // handle plain HTML text, which can really be anything. Needs to become
+            // a string literal
+            write!(self.out_stream, " \"")?;
+            self.lexer.emit_whitespace();
+            loop {
+                if self.current == Token::Lt
+                    || self.current == Token::LuaStart
+                    || self.current == Token::OpenClosingTag
+                {
+                    break;
+                }
+                // all other tokens *should* be fine to just emit
+                write!(self.out_stream, "{}", self.current)?;
+                self.next_token_silent()?;
+            }
+            self.lexer.hide_whitespace();
+            write!(self.out_stream, "\",")?;
+        }
+        write!(self.out_stream, "}}")?;
+
+        Ok(())
+    }
 }
diff --git a/src/luax/tests/lexer.rs b/src/luax/tests/lexer.rs
new file mode 100644
index 0000000..997dc2c
--- /dev/null
+++ b/src/luax/tests/lexer.rs
@@ -0,0 +1,197 @@
+use crate::luax::lexer::*;
+use crate::luax::tokens::*;
+use color_eyre::Result;
+
+fn compare_tokens<'a>(lua: &'a str, expected: Vec<Token<'a>>) -> Result<()> {
+    let mut lexer = Lexer::new(lua);
+    let mut tokens = Vec::new();
+    loop {
+        let token = lexer.next_token()?;
+        if token.is_none() {
+            break;
+        }
+        tokens.push(token.unwrap());
+    }
+
+    assert_eq!(tokens, expected);
+
+    Ok(())
+}
+
+#[test]
+fn empty() -> Result<()> {
+    compare_tokens("", vec![Token::Eof])
+}
+
+#[test]
+fn number() -> Result<()> {
+    compare_tokens("123", vec![Token::Number("123"), Token::Eof])
+}
+
+#[test]
+fn string() -> Result<()> {
+    compare_tokens(
+        "\"hello world\"",
+        vec![Token::String("hello world", StringType::Double), Token::Eof],
+    )
+}
+
+#[test]
+fn empty_string() -> Result<()> {
+    compare_tokens(
+        "\"\"",
+        vec![Token::String("", StringType::Double), Token::Eof],
+    )
+}
+
+#[test]
+fn all_string_types() -> Result<()> {
+    compare_tokens(
+        "\"hello world\"'hello world'[[hello world]]",
+        vec![
+            Token::String("hello world", StringType::Double),
+            Token::String("hello world", StringType::Single),
+            Token::String("hello world", StringType::DoubleBracket),
+            Token::Eof,
+        ],
+    )
+}
+
+#[test]
+fn table() -> Result<()> {
+    compare_tokens(
+        "{1, 2, 3}",
+        vec![
+            Token::OpenBrace,
+            Token::Number("1"),
+            Token::Comma,
+            Token::Number("2"),
+            Token::Comma,
+            Token::Number("3"),
+            Token::CloseBrace,
+            Token::Eof,
+        ],
+    )
+}
+
+#[test]
+fn table_with_string_keys() -> Result<()> {
+    compare_tokens(
+        "{[\"a\"]=1, [\"b\"]=2, [\"c\"]=3}",
+        vec![
+            Token::OpenBrace,
+            Token::OpenBracket,
+            Token::String("a", StringType::Double),
+            Token::CloseBracket,
+            Token::Eq,
+            Token::Number("1"),
+            Token::Comma,
+            Token::OpenBracket,
+            Token::String("b", StringType::Double),
+            Token::CloseBracket,
+            Token::Eq,
+            Token::Number("2"),
+            Token::Comma,
+            Token::OpenBracket,
+            Token::String("c", StringType::Double),
+            Token::CloseBracket,
+            Token::Eq,
+            Token::Number("3"),
+            Token::CloseBrace,
+            Token::Eof,
+        ],
+    )
+}
+
+#[test]
+fn operators() -> Result<()> {
+    compare_tokens(
+        "+ - * / // ^ % & ~ | << >> .. < <= > >= == ~= and or # not",
+        vec![
+            Token::Plus,
+            Token::Minus,
+            Token::Star,
+            Token::Slash,
+            Token::SlashSlash,
+            Token::Hat,
+            Token::Percent,
+            Token::Amp,
+            Token::Tilde,
+            Token::Pipe,
+            Token::LtLt,
+            Token::GtGt,
+            Token::DotDot,
+            Token::Lt,
+            Token::Le,
+            Token::Gt,
+            Token::Ge,
+            Token::EqEq,
+            Token::TildeEq,
+            Token::And,
+            Token::Or,
+            Token::Hash,
+            Token::Not,
+            Token::Eof,
+        ],
+    )
+}
+
+#[test]
+fn keywords() -> Result<()> {
+    compare_tokens(
+        "and break do else elseif end false for function goto if in local nil not or repeat return then true until while",
+        vec![
+            Token::And,
+            Token::Break,
+            Token::Do,
+            Token::Else,
+            Token::ElseIf,
+            Token::End,
+            Token::False,
+            Token::For,
+            Token::Function,
+            Token::Goto,
+            Token::If,
+            Token::In,
+            Token::Local,
+            Token::Nil,
+            Token::Not,
+            Token::Or,
+            Token::Repeat,
+            Token::Return,
+            Token::Then,
+            Token::True,
+            Token::Until,
+            Token::While,
+            Token::Eof,
+        ],
+    )
+}
+
+#[test]
+fn html() -> Result<()> {
+    compare_tokens(
+        "<div class=\"hello\">hello world</div><img src=\"hello.png\"/>",
+        vec![
+            Token::Lt,
+            Token::Identifier("div"),
+            Token::Identifier("class"),
+            Token::Eq,
+            Token::String("hello", StringType::Double),
+            Token::Gt,
+            Token::Identifier("hello"),
+            Token::Identifier("world"),
+            Token::OpenClosingTag,
+            Token::Identifier("div"),
+            Token::Gt,
+            Token::Lt,
+            Token::Identifier("img"),
+            Token::Identifier("src"),
+            Token::Eq,
+            Token::String("hello.png", StringType::Double),
+            Token::Slash,
+            Token::Gt,
+            Token::Eof,
+        ],
+    )
+}
diff --git a/src/luax/tests/mod.rs b/src/luax/tests/mod.rs
new file mode 100644
index 0000000..f2fed46
--- /dev/null
+++ b/src/luax/tests/mod.rs
@@ -0,0 +1,2 @@
+mod lexer;
+mod preprocessor;
diff --git a/src/luax/tests/preprocessor.rs b/src/luax/tests/preprocessor.rs
new file mode 100644
index 0000000..d5c2f42
--- /dev/null
+++ b/src/luax/tests/preprocessor.rs
@@ -0,0 +1,198 @@
+use crate::luax::{lexer::Lexer, tokens::Token, *};
+use color_eyre::Result;
+
+fn tokenize(input: &str) -> Result<Vec<Token>> {
+    let mut lexer = Lexer::new(input);
+    let mut tokens = Vec::new();
+
+    while let Some(token) = lexer.next_token()? {
+        tokens.push(token);
+    }
+
+    Ok(tokens)
+}
+
+fn compare_output(input: &str, expected: &str) -> Result<()> {
+    let output = preprocess(input)?;
+
+    // collect the tokens from each string
+    let expected_tokens = tokenize(expected)?;
+    let output_tokens = tokenize(&output)?;
+
+    if !expected_tokens
+        .iter()
+        .zip(output_tokens.iter())
+        .all(|(a, b)| a == b)
+    {
+        println!("expected: {:?}", expected_tokens);
+        println!("output: {:?}", output_tokens);
+        let first_mismatch = expected_tokens
+            .iter()
+            .zip(output_tokens.iter())
+            .enumerate()
+            .find(|(_, (a, b))| a != b)
+            .unwrap();
+        println!(
+            "first mismatch: index {}: {:?} != {:?}",
+            first_mismatch.0, first_mismatch.1.0, first_mismatch.1.1
+        );
+        println!("expected: {}", expected);
+        println!("output: {}", output);
+        panic!("output did not match expected");
+    }
+
+    Ok(())
+}
+
+#[test]
+fn empty() -> Result<()> {
+    compare_output("", "")
+}
+
+#[test]
+fn no_preprocessor() -> Result<()> {
+    compare_output("hello", "hello")
+}
+
+#[test]
+fn lua_function() -> Result<()> {
+    compare_output("function hello() end", "function hello ( ) end")
+}
+
+#[test]
+fn lua_function_with_args() -> Result<()> {
+    compare_output(
+        "function hello(a, b, c) end",
+        "function hello ( a , b , c ) end",
+    )
+}
+
+#[test]
+fn variable() -> Result<()> {
+    compare_output(
+        "local hello = 123 hello = 456",
+        "local hello = 123 hello = 456",
+    )
+}
+
+#[test]
+fn call() -> Result<()> {
+    compare_output("hello()", "hello ( )")
+}
+
+#[test]
+fn call_with_args() -> Result<()> {
+    compare_output("hello(1, 2, 3)", "hello ( 1 , 2 , 3 )")
+}
+
+#[test]
+fn call_with_args_and_assign() -> Result<()> {
+    compare_output("local a = hello(1, 2, 3)", "local a = hello ( 1 , 2 , 3 )")
+}
+
+#[test]
+fn call_with_string_arg() -> Result<()> {
+    compare_output("hello \"world\"", "hello \"world\"")
+}
+
+#[test]
+fn call_with_table_arg() -> Result<()> {
+    compare_output("hello {1, 2, 3}", "hello { 1 , 2 , 3 }")
+}
+
+#[test]
+fn long_call_and_access() -> Result<()> {
+    compare_output("a.b.c()[1].d()().e", "a . b . c ( ) [ 1 ] . d ( ) ( ) . e")
+}
+
+#[test]
+fn simple_html() -> Result<()> {
+    compare_output(
+        "return <div></div>",
+        "return { tag=\"div\", attrs={}, children={} }",
+    )
+}
+
+#[test]
+fn html_with_text() -> Result<()> {
+    compare_output(
+        "return <div>hello</div>",
+        "return { tag=\"div\", attrs={}, children={ \"hello\",} }",
+    )
+}
+
+#[test]
+fn html_with_weird_text() -> Result<()> {
+    compare_output(
+        "return <div>hello #world!</div>",
+        "return { tag=\"div\", attrs={}, children={ \"hello #world!\",} }",
+    )
+}
+
+#[test]
+fn html_with_attrs() -> Result<()> {
+    compare_output(
+        "return <div class=\"hello\" id=\"world\"></div>",
+        "return { tag=\"div\", attrs={class=\"hello\", id=\"world\", }, children={} }",
+    )
+}
+
+#[test]
+fn nested_html() -> Result<()> {
+    compare_output(
+        "return <div><span></span></div>",
+        "return { tag=\"div\", attrs={}, children={ { tag=\"span\", attrs={}, children={} },} }",
+    )
+}
+
+#[test]
+fn nested_html_with_text() -> Result<()> {
+    compare_output(
+        "return <div><span>hello</span></div>",
+        "return { tag=\"div\", attrs={}, children={ { tag=\"span\", attrs={}, children={ \"hello\",} },} }",
+    )
+}
+
+#[test]
+fn nested_html_with_attrs() -> Result<()> {
+    compare_output(
+        "return <div><span class=\"hello\" id=\"world\"></span></div>",
+        "return { tag=\"div\", attrs={}, children={ { tag=\"span\", attrs={class=\"hello\", id=\"world\", }, children={} },} }",
+    )
+}
+
+#[test]
+fn self_closing_tag() -> Result<()> {
+    compare_output(
+        "return <div><span /><span /></div>",
+        "return { tag=\"div\", attrs={}, children={ { tag=\"span\", attrs={}, children={} }, { tag=\"span\", attrs={}, children={} },} }",
+    )
+}
+
+#[test]
+fn site() -> Result<()> {
+    compare_output(
+        r#"
+        return <html>
+            <head>
+                <title>My Site</title>
+            </head>
+            <body>
+                <h1>My Site</h1>
+                <p>My site is the best site.</p>
+            </body>
+        </html>
+        "#,
+        r#"
+        return { tag="html", attrs={}, children={
+            { tag="head", attrs={}, children={
+                { tag="title", attrs={}, children={ "My Site",} },
+            } },
+            { tag="body", attrs={}, children={
+                { tag="h1", attrs={}, children={ "My Site",} },
+                { tag="p", attrs={}, children={ "My site is the best site.",} },
+            } },
+        } }
+        "#,
+    )
+}
diff --git a/src/luax/tokens.rs b/src/luax/tokens.rs
index 9f1058a..2c9448d 100644
--- a/src/luax/tokens.rs
+++ b/src/luax/tokens.rs
@@ -66,12 +66,17 @@ pub enum Token<'s> {
     Hash,
     Not,
     /// Containers
-    Number(f64),
+    Number(&'s str),
     String(&'s str, StringType),
     Identifier(&'s str),
 
-    Dollar,
+    // HTML extensions
+    OpenClosingTag,
+    LuaStart,
+    LuaEnd,
     Eof,
+    Unknown(char),
+    Whitespace,
 }
 
 impl std::fmt::Display for Token<'_> {
@@ -139,9 +144,13 @@ impl std::fmt::Display for Token<'_> {
                 StringType::DoubleBracket => write!(f, "[[{}]]", s),
             },
             Token::Identifier(s) => write!(f, "{}", s),
-            Token::Dollar => write!(f, "$"),
+            Token::OpenClosingTag => write!(f, "</"),
+            Token::LuaStart => write!(f, "{{$"),
+            Token::LuaEnd => write!(f, "$}}"),
 
             Token::Eof => write!(f, "EOF"),
+            Token::Unknown(c) => write!(f, "{}", c),
+            Token::Whitespace => write!(f, " "),
         }
     }
 }
diff --git a/src/main.rs b/src/main.rs
index f0683ae..466ffbc 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -19,6 +19,7 @@ fn table_to_html<W: std::io::Write>(table: rlua::Table, f: &mut W) -> Result<()>
 
     write!(f, "<{}", type_name)?;
     let mut children = None;
+    let mut attrs = None;
     for pair in table.pairs::<String, rlua::Value>() {
         let (key, value) = pair?;
         if key == "tag" {
@@ -28,13 +29,19 @@ fn table_to_html<W: std::io::Write>(table: rlua::Table, f: &mut W) -> Result<()>
             children = Some(value);
             continue;
         }
-        match value {
-            rlua::Value::Nil => write!(f, " {}=\"\"", key)?,
-            rlua::Value::Boolean(b) => write!(f, " {}=\"{}\"", key, b)?,
-            rlua::Value::Integer(i) => write!(f, " {}=\"{}\"", key, i)?,
-            rlua::Value::Number(n) => write!(f, " {}=\"{}\"", key, n)?,
-            rlua::Value::String(s) => write!(f, " {}=\"{}\"", key, s.to_str()?)?,
-            _ => return Err(ReluaxError::LuaX(LuaXError::WrongFieldType(key)).into()),
+        if key == "attrs" {
+            attrs = Some(value);
+            continue;
+        }
+    }
+    if let Some(attrs) = attrs {
+        if let rlua::Value::Table(attrs) = attrs {
+            for pair in attrs.pairs::<String, rlua::String>() {
+                let (key, value) = pair?;
+                write!(f, " {}=\"{}\"", key, value.to_str()?)?;
+            }
+        } else {
+            return Err(ReluaxError::LuaX(LuaXError::NonTableAttrs).into());
         }
     }
     write!(f, ">")?;
@@ -93,7 +100,8 @@ async fn main() -> color_eyre::Result<()> {
         return Err(std::io::Error::new(
             std::io::ErrorKind::NotFound,
             format!("{} is not a file", entry.display()),
-        ).into());
+        )
+        .into());
     }
 
     server::Server::serve(args.port).await?;