minor fixes to unknown handling

3 files changed, 7 insertions, 13 deletions

diff --git a/src/luax/lexer.rs b/src/luax/lexer.rs
index bae4003..c075b3a 100644
--- a/src/luax/lexer.rs
+++ b/src/luax/lexer.rs
@@ -102,7 +102,7 @@ impl<'s> Lexer<'s> {
                 TokenizeResult::Some(token) => Ok(Some(token)),
                 TokenizeResult::Error(error) => Err(error.into()),
                 TokenizeResult::None => match self.current {
-                    Some(c) => Err(LuaXError::UnexpectedCharacter(c).into()),
+                    Some(c) => Ok(Some(Token::Unknown(c))),
                     None => {
                         if self.emitted_eof {
                             Ok(None)
diff --git a/src/luax/preprocessor.rs b/src/luax/preprocessor.rs
index cd5d735..2e24c27 100644
--- a/src/luax/preprocessor.rs
+++ b/src/luax/preprocessor.rs
@@ -873,17 +873,7 @@ impl<'s, W: Write> Preprocessor<'s, W> {
         }
 
         // there might be an invalid token immediately after the tag name
-        match self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string())) {
-            Ok(()) => {}
-            Err(e) => match e.downcast_ref::<LuaXError>() {
-                Some(LuaXError::UnexpectedCharacter(_)) => {
-                    self.lexer.enable_html_text_mode();
-                    self.next_token_silent()?;
-                    self.lexer.disable_html_text_mode();
-                },
-                _ => return Err(e),
-            },
-        }
+        self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))?;
 
         self.html_children()?;
 
@@ -973,7 +963,6 @@ impl<'s, W: Write> Preprocessor<'s, W> {
                 Ok(t) => Some(t),
                 Err(e) => match e.downcast_ref::<LuaXError>() {
                     Some(LuaXError::InvalidStart) => None,
-                    Some(LuaXError::UnexpectedCharacter(_)) => None,
                     _ => return Err(e),
                 },
             }
@@ -995,6 +984,9 @@ impl<'s, W: Write> Preprocessor<'s, W> {
             write!(self.out_stream, " \"")?;
             self.lexer.enable_html_text_mode();
             loop {
+                if matches!(self.current, Token::Unknown(_)) {
+                    self.next_token_silent()?;
+                }
                 if self.current == Token::Lt
                     || self.current == Token::LuaStart
                     || self.current == Token::OpenClosingTag
diff --git a/src/luax/tokens.rs b/src/luax/tokens.rs
index 9c23acd..fb3691e 100644
--- a/src/luax/tokens.rs
+++ b/src/luax/tokens.rs
@@ -77,6 +77,7 @@ pub enum Token<'s> {
     LuaEnd,
     Eof,
     HtmlTextChar(char),
+    Unknown(char),
     Whitespace,
 }
 
@@ -152,6 +153,7 @@ impl std::fmt::Display for Token<'_> {
             Token::Bang => write!(f, "!"),
             Token::Eof => write!(f, "EOF"),
             Token::HtmlTextChar(c) => write!(f, "{}", c),
+            Token::Unknown(c) => write!(f, "{}", c),
             Token::Whitespace => write!(f, " "),
         }
     }