diff options
Diffstat (limited to 'src/luax')
| -rw-r--r-- | src/luax/default_html_tags.txt | 127 | ||||
| -rw-r--r-- | src/luax/lexer.rs | 77 | ||||
| -rw-r--r-- | src/luax/mod.rs | 36 | ||||
| -rw-r--r-- | src/luax/preprocessor.rs | 97 | ||||
| -rw-r--r-- | src/luax/tests/preprocessor.rs | 79 | ||||
| -rw-r--r-- | src/luax/tokens.rs | 6 |
6 files changed, 359 insertions, 63 deletions
diff --git a/src/luax/default_html_tags.txt b/src/luax/default_html_tags.txt new file mode 100644 index 0000000..5d84cad --- /dev/null +++ b/src/luax/default_html_tags.txt @@ -0,0 +1,127 @@ +!--...-- +!DOCTYPE +a +abbr +acronym +address +applet +area +article +aside +audio +b +base +basefont +bdi +bdo +big +blockquote +body +br +button +canvas +caption +center +cite +code +col +colgroup +data +datalist +dd +del +details +dfn +dialog +dir +div +dl +dt +em +embed +fieldset +figcaption +figure +font +footer +form +frame +frameset +h1 +h2 +h3 +h4 +h5 +h6 +head +header +hgroup +hr +html +i +iframe +img +input +ins +kbd +label +legend +li +link +main +map +mark +menu +meta +meter +nav +noframes +noscript +object +ol +optgroup +option +output +p +param +picture +pre +progress +q +rp +rt +ruby +s +samp +script +search +section +select +small +source +span +strike +strong +style +sub +summary +sup +svg +table +tbody +td +template +textarea +tfoot +th +thead +time +title +tr +track +tt +u +ul +var +video +wbr diff --git a/src/luax/lexer.rs b/src/luax/lexer.rs index 7d4f20a..bae4003 100644 --- a/src/luax/lexer.rs +++ b/src/luax/lexer.rs @@ -37,8 +37,7 @@ pub struct Lexer<'s> { current_pos_in_bytes: usize, // EOF emitted_eof: bool, - allow_unknowns: bool, - emit_whitespace: bool, + html_text_mode: usize, } #[derive(Debug)] @@ -61,29 +60,32 @@ impl<'s> Lexer<'s> { current, current_pos_in_bytes: 0, emitted_eof: false, - allow_unknowns: false, - emit_whitespace: false, + html_text_mode: 0, } } pub fn next_token(&mut self) -> Result<Option<Token<'s>>> { - let token = self.lex(); + if self.html_text_mode > 0 { + let c = self.current; + self.advance(); - match token { - TokenizeResult::Some(token) => Ok(Some(token)), - TokenizeResult::Error(error) => Err(error.into()), - TokenizeResult::None => match self.current { - Some(c) => { - if self.allow_unknowns { - self.advance(); - match c { - ' ' | '\t' | '\n' => Ok(Some(Token::Whitespace)), - _ => Ok(Some(Token::Unknown(c))), - } + match c { + Some('<') => { + if self.match_char('/') { + Ok(Some(Token::OpenClosingTag)) } else { - Err(LuaXError::UnexpectedCharacter(c).into()) + Ok(Some(Token::Lt)) } } + Some(' ' | '\t' | '\n') => Ok(Some(Token::Whitespace)), + Some('{') => { + if self.match_char('$') { + Ok(Some(Token::LuaStart)) + } else { + Ok(Some(Token::HtmlTextChar('{'))) + } + } + Some(c) => Ok(Some(Token::HtmlTextChar(c))), None => { if self.emitted_eof { Ok(None) @@ -92,30 +94,38 @@ impl<'s> Lexer<'s> { Ok(Some(Token::Eof)) } } - }, - } - } - - pub fn allow_unknowns(&mut self) { - self.allow_unknowns = true; - } + } + } else { + let token = self.lex(); - pub fn disallow_unknowns(&mut self) { - self.allow_unknowns = false; + match token { + TokenizeResult::Some(token) => Ok(Some(token)), + TokenizeResult::Error(error) => Err(error.into()), + TokenizeResult::None => match self.current { + Some(c) => Err(LuaXError::UnexpectedCharacter(c).into()), + None => { + if self.emitted_eof { + Ok(None) + } else { + self.emitted_eof = true; + Ok(Some(Token::Eof)) + } + } + }, + } + } } - pub fn emit_whitespace(&mut self) { - self.emit_whitespace = true; + pub fn enable_html_text_mode(&mut self) { + self.html_text_mode += 1; } - pub fn hide_whitespace(&mut self) { - self.emit_whitespace = false; + pub fn disable_html_text_mode(&mut self) { + self.html_text_mode -= 1; } fn lex(&mut self) -> TokenizeResult<'s> { - if !self.emit_whitespace { - self.skip_whitespace(); - } + self.skip_whitespace(); try_all_paths!( self.single_char_token(), @@ -181,6 +191,7 @@ impl<'s> Lexer<'s> { self.single_char_token_case(']', Token::CloseBracket), self.single_char_token_case(';', Token::Semicolon), self.single_char_token_case(',', Token::Comma), + self.single_char_token_case('!', Token::Bang), ) } diff --git a/src/luax/mod.rs b/src/luax/mod.rs index 1a3db12..ddbb2ef 100644 --- a/src/luax/mod.rs +++ b/src/luax/mod.rs @@ -9,13 +9,22 @@ mod tests; mod tokens; pub fn table_to_html<W: std::io::Write>(table: rlua::Table, f: &mut W) -> Result<()> { - let type_name: Option<String> = table.get("tag").unwrap(); + let tag_name: Option<String> = table.get("tag").unwrap(); - if type_name.is_none() { - return Err(LuaXError::MissingField("tag".to_string()).into()); + if tag_name.is_none() { + // we might be in a list + for child in table.sequence_values::<rlua::Value>() { + match child? { + rlua::Value::Table(child) => table_to_html(child, f)?, + rlua::Value::String(s) => write!(f, "{}", s.to_str()?)?, + _ => return Err(LuaXError::NonTableChildren.into()), + } + } + + return Ok(()); } - let type_name = type_name.unwrap(); + let type_name = tag_name.unwrap(); write!(f, "<{}", type_name)?; let mut children = None; @@ -69,7 +78,13 @@ pub fn preprocess(s: &str) -> Result<String> { let mut buf = Vec::new(); let preprocessor = preprocessor::Preprocessor::new(s, &mut buf)?; - preprocessor.preprocess()?; + match preprocessor.preprocess() { + Ok(_) => {} + Err(e) => { + println!("got up to: {}", String::from_utf8_lossy(&buf)); + return Err(e); + } + } let s = String::from_utf8(buf).unwrap(); @@ -221,7 +236,6 @@ mod utils { param_name.push(pattern_char.unwrap()); pattern_char = pattern.next(); } - println!("param_name: {}", param_name); pattern_char = pattern.next(); let mut param_value = String::new(); while path_char != Some('/') && path_char.is_some() { @@ -295,16 +309,12 @@ mod utils { vec![("name", "a"), ("name2", "c")], ), ("/{name}/b/{name2}", "/a/b", vec![("name", "a")]), - ( - "/{name}/b/{name2}", - "/a/b/c/d/e", - vec![("name", "a"), ("name2", "c")], - ), ]; let lua = Lua::new(); - for (pattern, path, expected) in cases { + for (pattern, path, mut expected) in cases { + expected.sort_by_key(|(k, _)| k.to_owned()); lua.context(|ctx| { let res = super::url_extract(ctx, (pattern.to_string(), path.to_string())).unwrap(); @@ -313,6 +323,8 @@ mod utils { let (key, value) = pair.unwrap(); res_vec.push((key, value)); } + res_vec.sort_by_key(|(k, _)| k.clone()); + let equal = res_vec .iter() .zip(expected.iter()) diff --git a/src/luax/preprocessor.rs b/src/luax/preprocessor.rs index ba51a44..cd5d735 100644 --- a/src/luax/preprocessor.rs +++ b/src/luax/preprocessor.rs @@ -1,4 +1,5 @@ use std::io::Write; +use std::sync::OnceLock; use crate::error::LuaXError; @@ -799,37 +800,90 @@ impl<'s, W: Write> Preprocessor<'s, W> { } fn html_identifier(&mut self) -> Result<String> { - if let Token::Identifier(s) = self.current { + if let Token::Bang = self.current { + self.next_token_silent()?; + let mut s = "!".to_string(); + if let Token::Identifier(s2) = self.current { + s.push_str(s2); + self.next_token_silent()?; + } + Ok(s.to_string()) + } else if let Token::Identifier(s) = self.current { self.next_token_silent()?; + let mut s = s.to_string(); + // we might have - in the identifier, so we loop until we don't + let mut last_was_id = true; + loop { + if let Token::Minus = self.current { + s.push('-'); + self.next_token_silent()?; + last_was_id = false; + } else if let Token::Identifier(s2) = self.current { + if last_was_id { + break; + } + s.push_str(s2); + self.next_token_silent()?; + last_was_id = true; + } else { + break; + } + } Ok(s.to_string()) } else { Err(LuaXError::InvalidStart.into()) } } + fn default_html_tag(tag: &str) -> bool { + static TAGS: OnceLock<Vec<&str>> = OnceLock::new(); + + let tags = TAGS.get_or_init(|| include_str!("default_html_tags.txt").split('\n').collect()); + + tags.contains(&tag) + } + fn html_template(&mut self) -> Result<()> { if !self.match_token_silent(Token::Lt)? { return Err(LuaXError::InvalidStart.into()); } - self.lexer.allow_unknowns(); - let tag = require!( self.html_identifier(), LuaXError::NeededToken("identifier".to_string()) ); - write!(self.out_stream, " {{ tag=\"{}\", ", tag)?; + if Self::default_html_tag(&tag) { + write!(self.out_stream, " {{ tag=\"{}\", ", tag)?; + } else { + write!(self.out_stream, " {} ({{ ", tag)?; + } self.html_attributes()?; if self.match_token_silent(Token::Slash)? { self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))?; - write!(self.out_stream, "children={{}} }}")?; + write!(self.out_stream, "children={{}}")?; + if Self::default_html_tag(&tag) { + write!(self.out_stream, " }}")?; + } else { + write!(self.out_stream, " }}) ")?; + } return Ok(()); } - self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))?; + // there might be an invalid token immediately after the tag name + match self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string())) { + Ok(()) => {} + Err(e) => match e.downcast_ref::<LuaXError>() { + Some(LuaXError::UnexpectedCharacter(_)) => { + self.lexer.enable_html_text_mode(); + self.next_token_silent()?; + self.lexer.disable_html_text_mode(); + }, + _ => return Err(e), + }, + } self.html_children()?; @@ -849,9 +903,11 @@ impl<'s, W: Write> Preprocessor<'s, W> { self.consume_token_silent(Token::Gt, LuaXError::NeededToken(Token::Gt.to_string()))?; - write!(self.out_stream, " }}")?; - - self.lexer.disallow_unknowns(); + if Self::default_html_tag(&tag) { + write!(self.out_stream, " }}")?; + } else { + write!(self.out_stream, " }}) ")?; + } Ok(()) } @@ -869,7 +925,11 @@ impl<'s, W: Write> Preprocessor<'s, W> { self.consume_token_silent(Token::Eq, LuaXError::NeededToken(Token::Eq.to_string()))?; - write!(self.out_stream, "{}=", key)?; + if key.contains('-') { + write!(self.out_stream, "[\"{}\"]=", key)?; + } else { + write!(self.out_stream, "{}=", key)?; + } if self.match_token_silent(Token::OpenBrace)? { require!(self.expression(), LuaXError::ExpectedExpression); @@ -900,9 +960,7 @@ impl<'s, W: Write> Preprocessor<'s, W> { break; } if self.match_token_silent(Token::LuaStart)? { - self.lexer.disallow_unknowns(); require!(self.expression(), LuaXError::ExpectedExpression); - self.lexer.allow_unknowns(); self.consume_token_silent( Token::LuaEnd, LuaXError::NeededToken(Token::LuaEnd.to_string()), @@ -911,7 +969,16 @@ impl<'s, W: Write> Preprocessor<'s, W> { continue; } - if optionally!(self.html_template()).is_some() { + if match self.html_template() { + Ok(t) => Some(t), + Err(e) => match e.downcast_ref::<LuaXError>() { + Some(LuaXError::InvalidStart) => None, + Some(LuaXError::UnexpectedCharacter(_)) => None, + _ => return Err(e), + }, + } + .is_some() + { write!(self.out_stream, ",")?; continue; } @@ -926,7 +993,7 @@ impl<'s, W: Write> Preprocessor<'s, W> { // handle plain HTML text, which can really be anything. Needs to become // a string literal write!(self.out_stream, " \"")?; - self.lexer.emit_whitespace(); + self.lexer.enable_html_text_mode(); loop { if self.current == Token::Lt || self.current == Token::LuaStart @@ -938,7 +1005,7 @@ impl<'s, W: Write> Preprocessor<'s, W> { write!(self.out_stream, "{}", self.current)?; self.next_token_silent()?; } - self.lexer.hide_whitespace(); + self.lexer.disable_html_text_mode(); write!(self.out_stream, "\",")?; } write!(self.out_stream, "}}")?; diff --git a/src/luax/tests/preprocessor.rs b/src/luax/tests/preprocessor.rs index d5c2f42..6769288 100644 --- a/src/luax/tests/preprocessor.rs +++ b/src/luax/tests/preprocessor.rs @@ -34,7 +34,7 @@ fn compare_output(input: &str, expected: &str) -> Result<()> { .unwrap(); println!( "first mismatch: index {}: {:?} != {:?}", - first_mismatch.0, first_mismatch.1.0, first_mismatch.1.1 + first_mismatch.0, first_mismatch.1 .0, first_mismatch.1 .1 ); println!("expected: {}", expected); println!("output: {}", output); @@ -196,3 +196,80 @@ fn site() -> Result<()> { "#, ) } + +#[test] +fn html_with_code() -> Result<()> { + compare_output( + "return <div>{$ hello $}</div>", + "return { tag=\"div\", attrs={}, children={ hello,} }", + ) +} + +#[test] +fn html_with_code_and_text() -> Result<()> { + compare_output( + "return <div>hello {$ world $}</div>", + "return { tag=\"div\", attrs={}, children={ \"hello \", world,} }", + ) +} + +#[test] +fn function_returning_html() -> Result<()> { + compare_output( + "function hello() return <div></div> end", + "function hello ( ) return { tag=\"div\", attrs={}, children={} } end", + ) +} + +#[test] +fn function_returning_nested_html() -> Result<()> { + compare_output( + "function hello() return <div><span></span></div> end", + "function hello ( ) return { tag=\"div\", attrs={}, children={ { tag=\"span\", attrs={}, children={} },} } end", + ) +} + +#[test] +fn function_returning_nested_html_with_code() -> Result<()> { + compare_output( + "function hello() return <div>{$ world $}</div> end", + "function hello ( ) return { tag=\"div\", attrs={}, children={ world,} } end", + ) +} + +#[test] +fn component() -> Result<()> { + compare_output("return <Hello />", "return Hello ({ attrs={}, children={} })") +} + +#[test] +fn component_with_attrs() -> Result<()> { + compare_output( + "return <Hello name=\"world\" />", + "return Hello ({ attrs={name=\"world\", }, children={} })", + ) +} + +#[test] +fn component_with_children() -> Result<()> { + compare_output( + "return <Hello><span /></Hello>", + "return Hello ({ attrs={}, children={ { tag=\"span\", attrs={}, children={} },} })", + ) +} + +#[test] +fn html_with_attrs_with_dash() -> Result<()> { + compare_output( + "return <div data-hello=\"world\"></div>", + "return { tag=\"div\", attrs={[\"data-hello\"]=\"world\", }, children={} }", + ) +} + +#[test] +fn weird_symbols_in_html() -> Result<()> { + compare_output( + "return <div>@everyone</div>", + "return { tag=\"div\", attrs={}, children={ \"@everyone\",} }", + ) +} diff --git a/src/luax/tokens.rs b/src/luax/tokens.rs index 2c9448d..9c23acd 100644 --- a/src/luax/tokens.rs +++ b/src/luax/tokens.rs @@ -71,11 +71,12 @@ pub enum Token<'s> { Identifier(&'s str), // HTML extensions + Bang, OpenClosingTag, LuaStart, LuaEnd, Eof, - Unknown(char), + HtmlTextChar(char), Whitespace, } @@ -148,8 +149,9 @@ impl std::fmt::Display for Token<'_> { Token::LuaStart => write!(f, "{{$"), Token::LuaEnd => write!(f, "$}}"), + Token::Bang => write!(f, "!"), Token::Eof => write!(f, "EOF"), - Token::Unknown(c) => write!(f, "{}", c), + Token::HtmlTextChar(c) => write!(f, "{}", c), Token::Whitespace => write!(f, " "), } } |