Compare commits

..

No commits in common. "e4f70dde42342e71583f20b14153beeffc6e9bf3" and "01a052f37a9cc4fd584d14a369b2f4befcbf0f10" have entirely different histories.

View File

@ -9,9 +9,9 @@
// * ignored words
// todo:
// * [x] rewrite fn parser to use chumsky::Recursive::declare/define
// - [x] do this to extract/simplify/DRY things like tuple patterns, fn clauses, etc.
// * [x] Work around chumsky::Stream::from_iter().spanned disappearing in most recent version
// * [ ] rewrite fn parser to use chumsky::Recursive::declare/define
// - [ ] do this to extract/simplify/DRY things like tuple patterns, fn clauses, etc.
// * [ ] Work around chumsky::Stream::from_iter().spanned disappearing in most recent version
// * [x] investigate using labels (which is behind a compiler flag, somehow)
// * [ ] wire up Ariadne parsing errors
// * [ ] validation
@ -28,7 +28,6 @@
use chumsky::{
input::{Stream, ValueInput},
prelude::*,
recursive::Recursive,
};
use imbl::{HashMap, Vector};
use std::fmt;
@ -345,400 +344,415 @@ fn parser<'src, I>(
where
I: ValueInput<'src, Token = Token<'src>, Span = Span>,
{
let mut expr = Recursive::declare();
let mut pattern = Recursive::declare();
let mut simple = Recursive::declare();
let mut nonbinding = Recursive::declare();
let separators = recursive(|separators| {
just(Token::Punctuation(","))
.or(just(Token::Punctuation("\n")))
.then(separators.clone().repeated())
});
let terminators = recursive(|terminators| {
just(Token::Punctuation(";"))
.or(just(Token::Punctuation("\n")))
.then(terminators.clone().repeated())
});
let placeholder_pattern =
select! {Token::Punctuation("_") => Pattern::Placeholder}.map_with(|p, e| (p, e.span()));
let word_pattern =
select! { Token::Word(w) => Pattern::Word(w) }.map_with(|w, e| (w, e.span()));
let atom_pattern = select! {
Token::Nil => Pattern::Atom(Value::Nil),
Token::Boolean(b) => Pattern::Atom(Value::Boolean(b)),
Token::Number(n) => Pattern::Atom(Value::Number(n)),
Token::Keyword(k) => Pattern::Atom(Value::Keyword(k)),
}
.map_with(|a, e| (a, e.span()));
let tuple_pattern = pattern
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|tuple, e| (Pattern::Tuple(tuple), e.span()))
.labelled("tuple pattern");
let list_pattern = pattern
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("[")), just(Token::Punctuation("]")))
.map_with(|list, e| (Pattern::List(list), e.span()));
let pair_pattern = select! {Token::Keyword(k) => Value::Keyword(k)}
.then(pattern.clone())
.map_with(|(kw, patt), e| (Pattern::Pair(kw, Box::new(patt)), e.span()));
let shorthand_pattern = select! {Token::Word(w) => w}.map_with(|w, e| {
(
Pattern::Pair(Value::Keyword(w), Box::new((Pattern::Word(w), e.span()))),
e.span(),
)
});
let dict_pattern = pair_pattern
.or(shorthand_pattern)
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(
just(Token::Punctuation("#{")),
just(Token::Punctuation("}")),
)
.map_with(|dict, e| (Pattern::Dict(dict), e.span()));
pattern.define(
atom_pattern
.or(word_pattern)
.or(placeholder_pattern)
.or(tuple_pattern.clone())
.or(list_pattern)
.or(dict_pattern)
.labelled("pattern"),
);
let placeholder =
select! {Token::Punctuation("_") => Ast::Placeholder}.map_with(|p, e| (p, e.span()));
let word = select! { Token::Word(w) => Ast::Word(w) }
.map_with(|w, e| (w, e.span()))
.labelled("word");
let value = select! {
Token::Nil => Ast::Value(Value::Nil),
Token::Boolean(b) => Ast::Value(Value::Boolean(b)),
Token::Number(n) => Ast::Value(Value::Number(n)),
Token::String(s) => Ast::Value(Value::String(s)),
}
.map_with(|v, e| (v, e.span()));
let keyword = select! {Token::Keyword(k) => Ast::Value(Value::Keyword(k)),}
.map_with(|k, e| (k, e.span()));
let tuple = simple
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|tuple, e| (Ast::Tuple(tuple), e.span()));
let args = simple
.clone()
.or(placeholder)
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|args, e| (Ast::Arguments(args), e.span()));
let synth_root = word.clone().or(keyword.clone());
let synth_term = keyword.clone().or(args);
let synthetic = synth_root
.then(synth_term.clone())
.then(synth_term.clone().repeated().collect())
.map_with(|((root, first), rest), e| {
(
Ast::Synthetic(Box::new(root), Box::new(first), rest),
e.span(),
)
recursive(|expr| {
let separators = recursive(|separators| {
just(Token::Punctuation(","))
.or(just(Token::Punctuation("\n")))
.then(separators.clone().repeated())
});
let list = simple
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("[")), just(Token::Punctuation("]")))
.map_with(|list, e| (Ast::List(list), e.span()));
let pattern = recursive(|pattern| {
let placeholder = select! {Token::Punctuation("_") => Pattern::Placeholder}
.map_with(|p, e| (p, e.span()));
let pair = select! {Token::Keyword(k) => Value::Keyword(k)}
.then(simple.clone())
.map_with(|(kw, expr), e| (Ast::Pair(kw, Box::new(expr)), e.span()));
let word =
select! { Token::Word(w) => Pattern::Word(w) }.map_with(|w, e| (w, e.span()));
let shorthand = select! {Token::Word(w) => w}.map_with(|w, e| {
(
Ast::Pair(Value::Keyword(w), Box::new((Ast::Word(w), e.span()))),
e.span(),
)
});
let atom = select! {
Token::Nil => Pattern::Atom(Value::Nil),
Token::Boolean(b) => Pattern::Atom(Value::Boolean(b)),
Token::Number(n) => Pattern::Atom(Value::Number(n)),
Token::Keyword(k) => Pattern::Atom(Value::Keyword(k)),
}
.map_with(|a, e| (a, e.span()));
let dict = pair
.or(shorthand)
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(
just(Token::Punctuation("#{")),
just(Token::Punctuation("}")),
)
.map_with(|dict, e| (Ast::Dict(dict), e.span()));
simple.define(
synthetic
.or(word)
.or(keyword)
.or(value)
.or(tuple)
.or(list)
.or(dict)
.labelled("simple expression"),
);
let block = expr
.clone()
.separated_by(terminators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}")))
.map_with(|block, e| (Ast::Block(block), e.span()))
.recover_with(via_parser(nested_delimiters(
Token::Punctuation("{"),
Token::Punctuation("}"),
[
(Token::Punctuation("("), Token::Punctuation(")")),
(Token::Punctuation("["), Token::Punctuation("]")),
],
|span| (Ast::Error, span),
)));
let if_ = just(Token::Reserved("if"))
.ignore_then(simple.clone())
.then_ignore(just(Token::Reserved("then")))
.then(expr.clone())
.then_ignore(just(Token::Reserved("else")))
.then(expr.clone())
.map_with(|((condition, then_branch), else_branch), e| {
(
Ast::If(
Box::new(condition),
Box::new(then_branch),
Box::new(else_branch),
),
e.span(),
)
});
let when_clause = simple
.clone()
.then_ignore(just(Token::Punctuation("->")))
.then(expr.clone())
.map_with(|(cond, body), e| (Ast::WhenClause(Box::new(cond), Box::new(body)), e.span()));
let when = just(Token::Reserved("when"))
.ignore_then(
when_clause
.separated_by(terminators.clone())
.allow_trailing()
.allow_leading()
.collect()
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}"))),
)
.map_with(|clauses, e| (Ast::When(clauses), e.span()));
let match_clause = pattern
.clone()
.then_ignore(just(Token::Punctuation("->")))
.then(expr.clone())
.map_with(|(patt, body), e| (Ast::MatchClause(Box::new(patt), Box::new(body)), e.span()));
let match_ = just(Token::Reserved("match"))
.ignore_then(simple.clone())
.then_ignore(just(Token::Reserved("with")))
.then(
match_clause
let tuple = pattern
.clone()
.separated_by(terminators.clone())
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}"))),
)
.map_with(|(expr, clauses), e| (Ast::Match(Box::new(expr), clauses), e.span()));
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|tuple, e| (Pattern::Tuple(tuple), e.span()));
let conditional = when.or(if_).or(match_);
//todo:
// * [x] do
// * [ ] loop
// * [ ] repeat
// * [x] panic!
let panic = just(Token::Reserved("panic!"))
.ignore_then(nonbinding.clone())
.map_with(|expr, e| (Ast::Panic(Box::new(expr)), e.span()));
let do_ = just(Token::Reserved("do"))
.ignore_then(
nonbinding
let list = pattern
.clone()
.separated_by(
just(Token::Punctuation(">")).then(just(Token::Punctuation("\n")).repeated()),
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("[")), just(Token::Punctuation("]")))
.map_with(|list, e| (Pattern::List(list), e.span()));
let pair = select! {Token::Keyword(k) => Value::Keyword(k)}
.then(pattern.clone())
.map_with(|(kw, patt), e| (Pattern::Pair(kw, Box::new(patt)), e.span()));
let shorthand = select! {Token::Word(w) => w}.map_with(|w, e| {
(
Pattern::Pair(Value::Keyword(w), Box::new((Pattern::Word(w), e.span()))),
e.span(),
)
.collect(),
)
.map_with(|exprs, e| (Ast::Do(exprs), e.span()));
});
let repeat = just(Token::Reserved("repeat"))
.ignore_then(simple.clone())
.then(block.clone())
.map_with(|(count, body), e| (Ast::Repeat(Box::new(count), Box::new(body)), e.span()));
let dict = pair
.or(shorthand)
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(
just(Token::Punctuation("#{")),
just(Token::Punctuation("}")),
)
.map_with(|dict, e| (Pattern::Dict(dict), e.span()));
let fn_clause = tuple_pattern
.clone()
.then_ignore(just(Token::Punctuation("->")))
.then(nonbinding.clone())
.map_with(|(pattern, body), e| (Ast::FnClause(Box::new(pattern), Box::new(body)), e.span()))
.labelled("function clause");
let lambda = just(Token::Reserved("fn"))
.ignore_then(fn_clause.clone())
.map_with(|clause, e| (Ast::Fn("anonymous", vec![clause]), e.span()));
nonbinding.define(
simple
.clone()
.or(conditional)
.or(block)
.or(lambda)
.or(panic)
.or(do_)
.or(repeat)
.labelled("nonbinding expression"),
);
let let_ = just(Token::Reserved("let"))
.ignore_then(pattern.clone())
.then_ignore(just(Token::Punctuation("=")))
.then(nonbinding.clone())
.map_with(|(pattern, expression), e| {
(Ast::Let(Box::new(pattern), Box::new(expression)), e.span())
atom.or(word)
.or(placeholder)
.or(tuple)
.or(list)
.or(dict)
.labelled("pattern")
});
let box_ = just(Token::Reserved("box"))
.ignore_then(word.clone())
.then_ignore(just(Token::Punctuation("=")))
.then(nonbinding.clone())
.map_with(|(word, expr), e| {
let name = if let Ast::Word(w) = word.0 {
w
} else {
unreachable!()
};
(Ast::Box(name, Box::new(expr)), e.span())
let placeholder =
select! {Token::Punctuation("_") => Ast::Placeholder}.map_with(|p, e| (p, e.span()));
let word = select! { Token::Word(w) => Ast::Word(w) }
.map_with(|w, e| (w, e.span()))
.labelled("word");
let simple = recursive(|simple| {
let value = select! {
Token::Nil => Ast::Value(Value::Nil),
Token::Boolean(b) => Ast::Value(Value::Boolean(b)),
Token::Number(n) => Ast::Value(Value::Number(n)),
Token::String(s) => Ast::Value(Value::String(s)),
}
.map_with(|v, e| (v, e.span()));
let keyword = select! {Token::Keyword(k) => Ast::Value(Value::Keyword(k)),}
.map_with(|k, e| (k, e.span()));
// let word = select! {Token::Word(w) => Ast::Word(w)}.map_with(|w, e| (w, e.span()));
let tuple = simple
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|tuple, e| (Ast::Tuple(tuple), e.span()));
// let placeholder = select! {Token::Punctuation("_") => Ast::Placeholder}
// .map_with(|p, e| (p, e.span()));
let args = simple
.clone()
.or(placeholder)
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|args, e| (Ast::Arguments(args), e.span()));
let synth_root = word.clone().or(keyword.clone());
let synth_term = keyword.clone().or(args);
let synthetic = synth_root
.then(synth_term.clone())
.then(synth_term.clone().repeated().collect())
.map_with(|((root, first), rest), e| {
(
Ast::Synthetic(Box::new(root), Box::new(first), rest),
e.span(),
)
});
let list = simple
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("[")), just(Token::Punctuation("]")))
.map_with(|list, e| (Ast::List(list), e.span()));
let pair = select! {Token::Keyword(k) => Value::Keyword(k)}
.then(simple.clone())
.map_with(|(kw, expr), e| (Ast::Pair(kw, Box::new(expr)), e.span()));
let shorthand = select! {Token::Word(w) => w}.map_with(|w, e| {
(
Ast::Pair(Value::Keyword(w), Box::new((Ast::Word(w), e.span()))),
e.span(),
)
});
let dict = pair
.or(shorthand)
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(
just(Token::Punctuation("#{")),
just(Token::Punctuation("}")),
)
.map_with(|dict, e| (Ast::Dict(dict), e.span()));
synthetic
.or(word)
.or(keyword)
.or(value)
.or(tuple)
.or(list)
.or(dict)
.labelled("simple expression")
});
let fn_decl = just(Token::Reserved("fn"))
.ignore_then(word.clone())
.map_with(|(word, _), e| {
let name = if let Ast::Word(w) = word {
w
} else {
unreachable!()
};
(Ast::FnDeclaration(name), e.span())
let terminators = recursive(|terminators| {
just(Token::Punctuation("\n"))
.or(just(Token::Punctuation(";")))
.then(terminators.clone().repeated())
.labelled("terminator")
});
// let tuple_pattern = pattern
// .clone()
// .separated_by(separators.clone())
// .allow_leading()
// .allow_trailing()
// .collect()
// .delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
// .map_with(|tuple, e| (Pattern::Tuple(tuple), e.span()));
let fn_named = just(Token::Reserved("fn"))
.ignore_then(word.clone())
.then(fn_clause.clone())
.map_with(|(word, clause), e| {
let name = if let Ast::Word(word) = word.0 {
word
} else {
unreachable!()
};
(Ast::Fn(name, vec![clause]), e.span())
});
let fn_compound = just(Token::Reserved("fn"))
.ignore_then(word.clone())
.then(
fn_clause
let nonbinding = recursive(|nonbinding| {
let block = expr
.clone()
.separated_by(terminators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}"))),
)
.map_with(|(word, clauses), e| {
let name = if let Ast::Word(word) = word.0 {
word
} else {
unreachable!()
};
(Ast::Fn(name, clauses), e.span())
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}")))
.map_with(|block, e| (Ast::Block(block), e.span()))
.recover_with(via_parser(nested_delimiters(
Token::Punctuation("{"),
Token::Punctuation("}"),
[
(Token::Punctuation("("), Token::Punctuation(")")),
(Token::Punctuation("["), Token::Punctuation("]")),
],
|span| (Ast::Error, span),
)));
let if_ = just(Token::Reserved("if"))
.ignore_then(simple.clone())
.then_ignore(just(Token::Reserved("then")))
.then(expr.clone())
.then_ignore(just(Token::Reserved("else")))
.then(expr.clone())
.map_with(|((condition, then_branch), else_branch), e| {
(
Ast::If(
Box::new(condition),
Box::new(then_branch),
Box::new(else_branch),
),
e.span(),
)
});
let when_clause = simple
.clone()
.then_ignore(just(Token::Punctuation("->")))
.then(expr.clone())
.map_with(|(cond, body), e| {
(Ast::WhenClause(Box::new(cond), Box::new(body)), e.span())
});
let when = just(Token::Reserved("when"))
.ignore_then(
when_clause
.separated_by(terminators.clone())
.allow_trailing()
.allow_leading()
.collect()
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}"))),
)
.map_with(|clauses, e| (Ast::When(clauses), e.span()));
let match_clause = pattern
.clone()
.then_ignore(just(Token::Punctuation("->")))
.then(expr.clone())
.map_with(|(patt, body), e| {
(Ast::MatchClause(Box::new(patt), Box::new(body)), e.span())
});
let match_ = just(Token::Reserved("match"))
.ignore_then(simple.clone())
.then_ignore(just(Token::Reserved("with")))
.then(
match_clause
.clone()
.separated_by(terminators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}"))),
)
.map_with(|(expr, clauses), e| (Ast::Match(Box::new(expr), clauses), e.span()));
let conditional = when.or(if_).or(match_);
//todo:
// * [x] do
// * [ ] loop
// * [ ] repeat
// * [x] panic!
let panic = just(Token::Reserved("panic!"))
.ignore_then(nonbinding.clone())
.map_with(|expr, e| (Ast::Panic(Box::new(expr)), e.span()));
let do_ = just(Token::Reserved("do"))
.ignore_then(
nonbinding
.clone()
.separated_by(
just(Token::Punctuation(">"))
.then(just(Token::Punctuation("\n")).repeated()),
)
.collect(),
)
.map_with(|exprs, e| (Ast::Do(exprs), e.span()));
let repeat = just(Token::Reserved("repeat"))
.ignore_then(simple.clone())
.then(block.clone())
.map_with(|(count, body), e| {
(Ast::Repeat(Box::new(count), Box::new(body)), e.span())
});
let tuple_pattern = pattern
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|tuple, e| (Pattern::Tuple(tuple), e.span()))
.labelled("tuple pattern");
let fn_clause = tuple_pattern
.then_ignore(just(Token::Punctuation("->")))
.then(nonbinding.clone())
.map_with(|(pattern, body), e| {
(Ast::FnClause(Box::new(pattern), Box::new(body)), e.span())
})
.labelled("function clause");
let lambda = just(Token::Reserved("fn"))
.ignore_then(fn_clause)
.map_with(|clause, e| (Ast::Fn("anonymous", vec![clause]), e.span()));
let loop_ = just(Token::Reserved("loop"));
simple
.clone()
.or(conditional)
.or(block)
.or(lambda)
.or(panic)
.or(do_)
.labelled("nonbinding expression")
});
let fn_ = fn_named.or(fn_compound).or(fn_decl);
let let_ = just(Token::Reserved("let"))
.ignore_then(pattern.clone())
.then_ignore(just(Token::Punctuation("=")))
.then(nonbinding.clone())
.map_with(|(pattern, expression), e| {
(Ast::Let(Box::new(pattern), Box::new(expression)), e.span())
});
let binding = let_.or(box_).or(fn_);
let box_ = just(Token::Reserved("box"))
.ignore_then(word.clone())
.then_ignore(just(Token::Punctuation("=")))
.then(nonbinding.clone())
.map_with(|(word, expr), e| {
let name = if let Ast::Word(w) = word.0 {
w
} else {
unreachable!()
};
(Ast::Box(name, Box::new(expr)), e.span())
});
expr.define(binding.or(nonbinding));
let fn_decl = just(Token::Reserved("fn"))
.ignore_then(word.clone())
.map_with(|(word, _), e| {
let name = if let Ast::Word(w) = word {
w
} else {
unreachable!()
};
(Ast::FnDeclaration(name), e.span())
});
let script = expr
.separated_by(terminators.clone())
.allow_trailing()
.allow_leading()
.collect()
.map_with(|exprs, e| (Ast::Block(exprs), e.span()));
let tuple_pattern = pattern
.clone()
.separated_by(separators.clone())
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("(")), just(Token::Punctuation(")")))
.map_with(|tuple, e| (Pattern::Tuple(tuple), e.span()));
script
let fn_clause = tuple_pattern
.then_ignore(just(Token::Punctuation("->")))
.then(nonbinding.clone())
.map_with(|(pattern, body), e| {
(Ast::FnClause(Box::new(pattern), Box::new(body)), e.span())
});
let fn_named = just(Token::Reserved("fn"))
.ignore_then(word.clone())
.then(fn_clause.clone())
.map_with(|(word, clause), e| {
let name = if let Ast::Word(word) = word.0 {
word
} else {
unreachable!()
};
(Ast::Fn(name, vec![clause]), e.span())
});
let fn_compound = just(Token::Reserved("fn"))
.ignore_then(word.clone())
.then(
fn_clause
.clone()
.separated_by(terminators)
.allow_leading()
.allow_trailing()
.collect()
.delimited_by(just(Token::Punctuation("{")), just(Token::Punctuation("}"))),
)
.map_with(|(word, clauses), e| {
let name = if let Ast::Word(word) = word.0 {
word
} else {
unreachable!()
};
(Ast::Fn(name, clauses), e.span())
});
let fn_ = fn_named.or(fn_compound).or(fn_decl);
let binding = let_.or(box_).or(fn_);
nonbinding.or(binding)
})
}
pub fn main() {
let src = "let #{a, :b b} = foo\na(b(c),d)";
let src = "let #{a, :b b} = foo";
let (tokens, lex_errs) = lexer().parse(src).into_output_errors();
if lex_errs.len() > 0 {
println!("{:?}", lex_errs);
@ -750,7 +764,7 @@ pub fn main() {
println!("{}", token)
}
let (ast, _) = parser()
.parse(Stream::from_iter(to_parse).map((0..src.len()).into(), |(t, s)| (t, s)))
.parse(Stream::from_iter(to_parse).spanned(SimpleSpan::new(0, src.len())))
.unwrap();
println!("{}", ast);
}