properly scan escape chars

This commit is contained in:
Scott Richmond 2025-07-02 20:54:21 -04:00
parent 28d6dc24f0
commit 2ffff9edd9
4 changed files with 49 additions and 2 deletions

View File

@ -378,8 +378,9 @@ fn chars/safe {
fn ws? {
"Tells if a string is a whitespace character."
(" ") -> true
("\n") -> true
("\t") -> true
("\n") -> true
("\r") -> true
(_) -> false
}

View File

@ -425,7 +425,7 @@ function __wbg_get_imports() {
_assertBoolean(ret);
return ret;
};
imports.wbg.__wbindgen_closure_wrapper7777 = function() { return logError(function (arg0, arg1, arg2) {
imports.wbg.__wbindgen_closure_wrapper7787 = function() { return logError(function (arg0, arg1, arg2) {
const ret = makeMutClosure(arg0, arg1, 347, __wbg_adapter_22);
return ret;
}, arguments) };

Binary file not shown.

View File

@ -68,7 +68,19 @@ pub fn lexer(
let keyword = just(':').ignore_then(word).map(Token::Keyword);
let escape = just('\\')
.then(choice((
just('\\'),
just('"'),
just('n').to('\n'),
just('t').to('\t'),
just('r').to('\r'),
)))
.ignored();
let string = none_of("\\\"")
.ignored()
.or(escape)
.repeated()
.to_slice()
.map(Token::String)
@ -104,3 +116,37 @@ pub fn lexer(
.repeated()
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_lexes_nil() {
let spanned_toks = lexer().parse("nil").into_output_errors().0.unwrap();
let (token, _) = spanned_toks[0].clone();
assert_eq!(token, Token::Nil);
}
#[test]
fn it_lexes_strings() {
let spanned_toks = lexer()
.parse("\"foo bar baz\"")
.into_output_errors()
.0
.unwrap();
let (token, _) = spanned_toks[0].clone();
assert_eq!(token, Token::String("foo bar baz"));
}
#[test]
fn it_lexes_strings_w_escaped_quotes() {
let spanned_toks = lexer()
.parse("\"foo \\\"bar baz\"")
.into_output_errors()
.0
.unwrap();
let (token, _) = spanned_toks[0].clone();
assert_eq!(token, Token::String("foo \\\"bar baz"));
}
}