From ce11f1cd0fbc09640cea2ab138f780ab36c97d47 Mon Sep 17 00:00:00 2001 From: Scott Richmond Date: Sat, 21 Jun 2025 17:43:47 -0400 Subject: [PATCH] start work on getting prelude working; discover closure bug --- assets/prelude.ld | 27 +++-- assets/test_prelude.ld | 252 +++++++++++++++++++++++++++++++++++++++-- may_2025_thoughts.md | 55 +++++++++ src/base.rs | 4 + src/compiler.rs | 3 + src/errors.rs | 76 ++++++------- src/main.rs | 50 +++++++- src/parser.rs | 12 +- src/validator.rs | 46 ++++---- src/value.rs | 4 +- 10 files changed, 441 insertions(+), 88 deletions(-) diff --git a/assets/prelude.ld b/assets/prelude.ld index aa1a979..d1c5b4c 100644 --- a/assets/prelude.ld +++ b/assets/prelude.ld @@ -25,7 +25,6 @@ fn turn/rad fn unbox fn update! -& the very base: know something's type fn type { "Returns a keyword representing the type of the value passed in." (x) -> base :type (x) @@ -124,19 +123,19 @@ fn not { (_) -> false } -fn neq? { - "Returns true if none of the arguments have the same value." - (x) -> false - (x, y) -> not (eq? (x, y)) - (x, y, ...zs) -> if eq? (x, y) - then false - else loop (y, zs) with { - (a, []) -> neq? (a, x) - (a, [b, ...cs]) -> if neq? (a, x) - then recur (b, cs) - else false - } -} +& fn neq? { +& "Returns true if none of the arguments have the same value." +& (x) -> false +& (x, y) -> not (eq? (x, y)) +& (x, y, ...zs) -> if eq? (x, y) +& then false +& else loop (y, zs) with { +& (a, []) -> neq? (a, x) +& (a, [b, ...cs]) -> if neq? (a, x) +& then recur (b, cs) +& else false +& } +& } & tuples: not a lot you can do with them functionally fn tuple? { diff --git a/assets/test_prelude.ld b/assets/test_prelude.ld index 34e709f..1b4c138 100644 --- a/assets/test_prelude.ld +++ b/assets/test_prelude.ld @@ -1,10 +1,248 @@ -fn min { - "Returns the number in its arguments that is closest to negative infinity." - (x as :number) -> x - (x as :number, y as :number) -> if base :lt? (x, y) then x else y - (x, y, ...zs) -> (min, zs, min (x, y)) +& & the very base: know something's type +& fn type { +& "Returns a keyword representing the type of the value passed in." +& (x) -> base :type (x) +& } + +& & some helper type functions +& fn coll? { +& "Returns true if a value is a collection: dict, list, tuple, or set." +& (coll as :dict) -> true +& (coll as :list) -> true +& (coll as :tuple) -> true +& & (coll as :set) -> true +& (_) -> false +& } + +& fn ordered? { +& "Returns true if a value is an indexed collection: list or tuple." +& (coll as :list) -> true +& (coll as :tuple) -> true +& (coll as :string) -> true +& (_) -> false +& } + +& fn assoc? { +& "Returns true if a value is an associative collection: a dict or a pkg." +& (d as :dict) -> true +& (_) -> false +& } + +& &&& nil: working with nothing + +& fn nil? { +& "Returns true if a value is nil." +& (nil) -> true +& (_) -> false +& } + +& fn some? { +& "Returns true if a value is not nil." +& (nil) -> false +& (_) -> true +& } + +& fn some { +& "Takes a possibly nil value and a default value. Returns the value if it's not nil, returns the default if it's nil." +& (nil, default) -> default +& (value, _) -> value +& } + +& & ...and if two things are the same +& fn eq? { +& "Returns true if all arguments have the same value." +& (x) -> true +& (x, y) -> base :eq? (x, y) +& (x, y, ...zs) -> if eq? (x, y) +& then loop (y, zs) with { +& (a, []) -> eq? (a, x) +& (a, [b, ...cs]) -> if eq? (a, x) +& then recur (b, cs) +& else false +& } +& else false +& } + +& &&& true & false: boolean logic (part the first) +& fn bool? { +& "Returns true if a value is of type :boolean." +& (false) -> true +& (true) -> true +& (_) -> false +& } + +& fn true? { +& "Returns true if a value is boolean `true`. Useful to distinguish between `true` and anything else." +& (true) -> true +& (_) -> false +& } + +& fn false? { +& "Returns `true` if a value is `false`, otherwise returns `false`. Useful to distinguish between `false` and `nil`." +& (false) -> true +& (_) -> false +& } + +& fn bool { +& "Returns false if a value is nil or false, otherwise returns true." +& (nil) -> false +& (false) -> false +& (_) -> true +& } + +& fn not { +& "Returns false if a value is truthy, true if a value is falsy." +& (nil) -> true +& (false) -> true +& (_) -> false +& } + +& & tuples: not a lot you can do with them functionally +& fn tuple? { +& "Returns true if a value is a tuple." +& (tuple as :tuple) -> true +& (_) -> false +& } + +& &&& functions: getting things done +& fn fn? { +& "Returns true if an argument is a function." +& (f as :fn) -> true +& (_) -> false +& } + +& what we need for some very basic list manipulation +fn first { + "Retrieves the first element of an ordered collection--a tuple or a list. If the collection is empty, returns nil." + ([]) -> nil + (()) -> nil + ("") -> nil + (xs as :list) -> base :first (xs) + (xs as :tuple) -> base :first (xs) + (str as :string) -> base :slice (str, 0, 1) } -fn lt? (x as :number, y as :number) -> base :lt? (x, y) +fn rest { + "Returns all but the first element of a list or tuple, as a list." + ([]) -> [] + (()) -> () + (xs as :list) -> base :rest (xs) + (xs as :tuple) -> base :rest (xs) + (str as :string) -> base :rest (str) +} -#{min, lt?} +fn inc { + "Increments a number." + (x as :number) -> base :inc (x) +} + +& fn dec { +& "Decrements a number." +& (x as :number) -> base :dec (x) +& } + +& fn count { +& "Returns the number of elements in a collection (including string)." +& (xs as :list) -> base :count (xs) +& (xs as :tuple) -> base :count (xs) +& (xs as :dict) -> base :count (xs) +& (xs as :string) -> base :count (xs) +& & (xs as :set) -> base :count (xs) +& } + +& fn empty? { +& "Returns true if something is empty. Otherwise returns false (including for things that can't logically be empty, like numbers)." +& ([]) -> true +& (#{}) -> true +& & (s as :set) -> eq? (s, ${}) +& (()) -> true +& ("") -> true +& (_) -> false +& } + +& fn any? { +& "Returns true if something is not empty, otherwise returns false (including for things that can't be logically full, like numbers)." +& ([...]) -> true +& (#{...}) -> true +& & (s as :set) -> not (empty? (s)) +& ((...)) -> true +& (s as :string) -> not (empty? (s)) +& (_) -> false +& } + +& fn list? { +& "Returns true if the value is a list." +& (l as :list) -> true +& (_) -> false +& } + +& fn list { +& "Takes a value and returns it as a list. For values, it simply wraps them in a list. For collections, conversions are as follows. A tuple->list conversion preservers order and length. Unordered collections do not preserve order: sets and dicts don't have predictable or stable ordering in output. Dicts return lists of (key, value) tuples." +& (x) -> base :list (x) +& } + +fn append { + "Adds an element to a list." + () -> [] + (xs as :list) -> xs + (xs as :list, x) -> base :append (xs, x) +} + +fn fold { + "Folds a list." + (f as :fn, []) -> [] + (f as :fn, xs as :list) -> fold (f, xs, f ()) + (f as :fn, [], root) -> [] + (f as :fn, xs as :list, root) -> loop (root, first (xs), rest (xs)) with { + (prev, curr, []) -> f (prev, curr) + (prev, curr, remaining) -> recur ( + f (prev, curr) + first (remaining) + rest (remaining) + ) + } +} + +fn map { + "Maps a function over a list: returns a new list with elements that are the result of applying the function to each element in the original list. E.g., `map ([1, 2, 3], inc) &=> [2, 3, 4]`. With one argument, returns a function that is a mapper over lists; with two, it executes the mapping function right away." + & (f as :fn) -> map (f, _) + & (kw as :keyword) -> map (kw, _) + (f as :fn, xs) -> { + fn mapper (prev, curr) -> append (prev, f (curr)) + fold (mapper, xs, []) + } + & (kw as :keyword, xs) -> { + & fn mapper (prev, curr) -> append (prev, kw (curr)) + & fold (mapper, xs, []) + & } +} + +#{ + & type + & coll? + & ordered? + & assoc? + & nil? + & some? + & some + & eq? + & bool? + & true? + & false? + & bool + & not + & tuple? + & fn? + & rest + inc + & dec + & count + & empty? + & any? + & list? + & list + & first + & fold + & append + map +} diff --git a/may_2025_thoughts.md b/may_2025_thoughts.md index ac7aa0d..41c06df 100644 --- a/may_2025_thoughts.md +++ b/may_2025_thoughts.md @@ -319,6 +319,11 @@ So this is my near-term TODO: - [ ] FINALLY, test Rudus against Ludus test cases And then: quality of life improvements: +* [ ] refactor messes + - [ ] The compiler should abstract over some of the very titchy bytecode instruction code + - [ ] Pull apart some gargantuan modules into smaller chunks: e.g., `Op` and `Chunk` should be their own modules + - [ ] Identify code smells + - [ ] Fix some of them * [ ] improve validator - [ ] Tuples may not be longer than n members - [ ] Loops may not have splatterns @@ -367,3 +372,53 @@ The vec in question is the LFn::Defined.closed. Just patched up the `if` alternative branch unconditional jump, which was jumping too far. Now it really is just some pretty systematic testing of prelude functions, including the problems with upvalues, which I haven't yet been able to recreate. + +### On proceeding from here +#### 2025-06-021 +Rather than doing everything by hand right now, I think the way to go about things is to figure out how to do as much automated bugfixing as possible. +That means reprioritizing some things. +So here's a short punch list of things to do in that register: +* [x] Hook validator back in to both source AND prelude code + - [x] Validator should know about the environment for global/prelude function + - [x] Run validator on current prelude to fix current known errors +* [ ] Do what it takes to compile this interpreter into Ludus's JS environment + - [ ] JSONify Ludus values + - [ ] Write a function that's source code to JSON result + - [ ] Expose this to a WASM compiler + - [ ] Patch this into a JS file + - [ ] Automate this build process +* [ ] Start testing against the cases in `ludus-test` +* [ ] Systematically debug prelude + - [ ] Bring it in function by function, testing each in turn + +*** +I've started working on systematically going through the Prelude. +I've found a closure error. +This is in `map`/`mapping`. +What's happening is that the inner function, `mapping`, is closing over values directly from a stack that no longer exists. +What we need to have happen is that if a function is closing over values _inside_ a function, it needs to capture not the upvalues directly from the stack, but _from the enclosing closure_. +I think I need to consult Uncle Bob Nystrom to get a sense of what to do here. +*** +So I found the minimal test case: +``` +let foo = { + let thing = :thing + let bar = :bar + let baz = :baz + fn quux () -> { + fn frobulate () -> (bar, baz) + } +} + +foo () +``` +`frobulate` is closed over when `quux` is called, when the stack looks completely different than it does when `quux` is defined. +If you remove line 2, binding `thing`, then you don't get a panic, but when you call `foo () ()`, the result is `(fn quux, fn frobulate)`. +The problem is that `frobulate`'s upvalues are indexes into the stack, rather than having some relation to `quux`'s upvalues. +What needs to happen is that an enclosing function needs to capture, define, and pass down the upvalues for its enclosed functions. + +I'm having an exact problem that Uncle Bob is describing at +https://craftinginterpreters.com/closures.html#flattening-upvalues. +I need to study and adapt this exact set of problems. +I believe I need to take the strategy he uses with closures being different from functions, etc. +So: rework the closures strategy here. diff --git a/src/base.rs b/src/base.rs index 1e97a77..a25caa5 100644 --- a/src/base.rs +++ b/src/base.rs @@ -266,6 +266,10 @@ pub fn rest(ordered: &Value) -> Value { Value::Tuple(tuple) => { Value::List(Box::new(Vector::from_iter(tuple.iter().next().cloned()))) } + Value::Interned(str) => Value::String(Rc::new(str.get(1..).unwrap_or("").to_string())), + Value::String(str) => Value::String(Rc::new( + str.clone().as_str().get(1..).unwrap_or("").to_string(), + )), _ => unreachable!("internal Ludus error"), } } diff --git a/src/compiler.rs b/src/compiler.rs index c43df8d..d582bf3 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1258,6 +1258,7 @@ impl<'a> Compiler<'a> { MatchClause(..) => unreachable!(), Fn(name, body, doc) => { let is_anon = name.is_empty(); + let mut name = name; if !is_anon { let declared = self.chunk.constants.iter().any(|val| match val { @@ -1275,6 +1276,8 @@ impl<'a> Compiler<'a> { self.emit_constant(declaration); self.bind(name); } + } else { + name = &"_anon"; } let FnBody(fn_body) = &body.as_ref().0 else { diff --git a/src/errors.rs b/src/errors.rs index 182cd3a..a3e374c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,47 +1,47 @@ -use crate::process::{LErr, Trace}; +// use crate::process::{LErr, Trace}; use crate::validator::VErr; use crate::value::Value; use ariadne::{sources, Color, Label, Report, ReportKind}; use std::collections::HashSet; -pub fn report_panic(err: LErr) { - let mut srcs = HashSet::new(); - let mut stack = vec![]; - let mut order = 1; - for entry in err.trace.iter().rev() { - let Trace { - callee, - caller, - function, - arguments, - input, - src, - } = entry; - let (_, first_span) = callee; - let (_, second_span) = caller; - let Value::Fn(f) = function else { - unreachable!() - }; - let fn_name = f.borrow().name.clone(); - let i = first_span.start; - let j = second_span.end; - let label = Label::new((entry.input, i..j)) - .with_color(Color::Yellow) - .with_message(format!("({order}) calling `{fn_name}` with `{arguments}`")); - order += 1; - stack.push(label); - srcs.insert((*input, *src)); - } +// pub fn report_panic(err: LErr) { +// let mut srcs = HashSet::new(); +// let mut stack = vec![]; +// let mut order = 1; +// for entry in err.trace.iter().rev() { +// let Trace { +// callee, +// caller, +// function, +// arguments, +// input, +// src, +// } = entry; +// let (_, first_span) = callee; +// let (_, second_span) = caller; +// let Value::Fn(f) = function else { +// unreachable!() +// }; +// let fn_name = f.borrow().name.clone(); +// let i = first_span.start; +// let j = second_span.end; +// let label = Label::new((entry.input, i..j)) +// .with_color(Color::Yellow) +// .with_message(format!("({order}) calling `{fn_name}` with `{arguments}`")); +// order += 1; +// stack.push(label); +// srcs.insert((*input, *src)); +// } - Report::build(ReportKind::Error, (err.input, err.span.into_range())) - .with_message(format!("Ludus panicked! {}", err.msg)) - .with_label(Label::new((err.input, err.span.into_range())).with_color(Color::Red)) - .with_labels(stack) - .with_note(err.extra) - .finish() - .print(sources(srcs.iter().copied())) - .unwrap(); -} +// Report::build(ReportKind::Error, (err.input, err.span.into_range())) +// .with_message(format!("Ludus panicked! {}", err.msg)) +// .with_label(Label::new((err.input, err.span.into_range())).with_color(Color::Red)) +// .with_labels(stack) +// .with_note(err.extra) +// .finish() +// .print(sources(srcs.iter().copied())) +// .unwrap(); +// } pub fn report_invalidation(errs: Vec) { for err in errs { diff --git a/src/main.rs b/src/main.rs index 3496480..4764f3c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,10 @@ mod parser; use crate::parser::{parser, Ast}; mod validator; +use crate::validator::Validator; + +mod errors; +use crate::errors::report_invalidation; mod compiler; use crate::compiler::Compiler; @@ -27,7 +31,7 @@ use value::Value; mod vm; use vm::Vm; -const PRELUDE: &str = include_str!("../assets/prelude.ld"); +const PRELUDE: &str = include_str!("../assets/test_prelude.ld"); pub fn prelude() -> HashMap<&'static str, Value> { let tokens = lexer().parse(PRELUDE).into_output_errors().0.unwrap(); @@ -41,9 +45,25 @@ pub fn prelude() -> HashMap<&'static str, Value> { panic!(); } - let parsed: &'static Spanned = Box::leak(Box::new(parsed.unwrap())); - let mut compiler = Compiler::new(parsed, "prelude", PRELUDE, None, HashMap::new()); + let parsed = parsed.unwrap(); + let (ast, span) = &parsed; + let base = base::make_base(); + let mut base_env = imbl::HashMap::new(); + base_env.insert("base", base.clone()); + + let mut validator = Validator::new(ast, span, "prelude", PRELUDE, base_env); + + validator.validate(); + + if !validator.errors.is_empty() { + println!("VALIDATION ERRORS IN PRLUDE:"); + report_invalidation(validator.errors); + panic!(); + } + + let parsed: &'static Spanned = Box::leak(Box::new(parsed)); + let mut compiler = Compiler::new(parsed, "prelude", PRELUDE, None, HashMap::new()); compiler.emit_constant(base); compiler.bind("base"); compiler.compile(); @@ -80,6 +100,16 @@ pub fn run(src: &'static str) { let parsed: &'static Spanned = Box::leak(Box::new(parse_result.unwrap())); let prelude = prelude(); + + let mut validator = Validator::new(&parsed.0, &parsed.1, "user input", src, prelude.clone()); + validator.validate(); + + if !validator.errors.is_empty() { + println!("Ludus found some validation errors:"); + report_invalidation(validator.errors); + return; + } + // let prelude = imbl::HashMap::new(); let mut compiler = Compiler::new(parsed, "test", src, None, prelude); @@ -113,8 +143,16 @@ pub fn run(src: &'static str) { pub fn main() { env::set_var("RUST_BACKTRACE", "1"); - let src = " - -"; + let src = r#" +let foo = { + let bar = :bar + let baz = :baz + fn quux () -> { + fn frobulate () -> (bar, baz) + } +} + +foo () () +"#; run(src); } diff --git a/src/parser.rs b/src/parser.rs index 2b0e4dc..9978a95 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -176,7 +176,15 @@ impl fmt::Display for Ast { .collect::>() .join("\n") ), - Tuple(t) | Ast::Arguments(t) => write!( + Arguments(a) => write!( + f, + "Arguments: ({})", + a.iter() + .map(|(line, _)| line.to_string()) + .collect::>() + .join("\n") + ), + Tuple(t) => write!( f, "Tuple: ({})", t.iter() @@ -434,7 +442,7 @@ fn is_word_char(c: char) -> bool { } fn parse_string(s: &'static str, span: SimpleSpan) -> Result>, String> { - println!("parsing string pattern: {s}"); + // println!("parsing string pattern: {s}"); let mut parts = vec![]; let mut current_part = String::new(); let mut start = span.start; diff --git a/src/validator.rs b/src/validator.rs index d8a1af9..455012b 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -1,6 +1,8 @@ // TODO: // * [ ] ensure `or` and `and` never get passed by reference -// * [ ] ensure no placeholder in `or` and `and` +// * [ ] ensure no placeholder in `or` and `and` args +// * [ ] ensure loops have fixed arity (no splats) +// * [ ] ensure fn pattern splats are always highest (and same) arity use crate::parser::*; use crate::spans::{Span, Spanned}; @@ -8,15 +10,15 @@ use crate::value::Value; use std::collections::{HashMap, HashSet}; #[derive(Clone, Debug, PartialEq)] -pub struct VErr { +pub struct VErr<'a> { pub msg: String, - pub span: Span, + pub span: &'a Span, pub input: &'static str, pub src: &'static str, } -impl VErr { - pub fn new(msg: String, span: Span, input: &'static str, src: &'static str) -> VErr { +impl<'a> VErr<'a> { + pub fn new(msg: String, span: &'a Span, input: &'static str, src: &'static str) -> VErr<'a> { VErr { msg, span, @@ -58,13 +60,13 @@ fn match_arities(arities: &HashSet, num_args: u8) -> bool { #[derive(Debug, PartialEq)] pub struct Validator<'a> { - pub locals: Vec<(String, Span, FnInfo)>, - pub prelude: &'a Vec<(&'static str, Value)>, + pub locals: Vec<(String, &'a Span, FnInfo)>, + pub prelude: imbl::HashMap<&'static str, Value>, pub input: &'static str, pub src: &'static str, pub ast: &'a Ast, - pub span: Span, - pub errors: Vec, + pub span: &'a Span, + pub errors: Vec>, pub fn_info: HashMap<*const Ast, FnInfo>, status: VStatus, } @@ -72,10 +74,10 @@ pub struct Validator<'a> { impl<'a> Validator<'a> { pub fn new( ast: &'a Ast, - span: Span, + span: &'a Span, input: &'static str, src: &'static str, - prelude: &'a Vec<(&'static str, Value)>, + prelude: imbl::HashMap<&'static str, Value>, ) -> Validator<'a> { Validator { input, @@ -116,7 +118,7 @@ impl<'a> Validator<'a> { || self.prelude.iter().any(|(bound, _)| name == *bound) } - fn bound(&self, name: &str) -> Option<&(String, Span, FnInfo)> { + fn bound(&self, name: &str) -> Option<&(String, &Span, FnInfo)> { match self.locals.iter().rev().find(|(bound, ..)| name == bound) { Some(binding) => Some(binding), None => None, @@ -150,7 +152,7 @@ impl<'a> Validator<'a> { fn visit(&mut self, node: &'a Spanned) { let (expr, span) = node; self.ast = expr; - self.span = *span; + self.span = span; self.validate(); } @@ -169,7 +171,7 @@ impl<'a> Validator<'a> { Interpolated(parts) => { for part in parts { if let (StringPart::Word(name), span) = part { - self.span = *span; + self.span = span; if !self.resolved(name.as_str()) { self.err(format!("unbound name `{name}`")); } else { @@ -282,6 +284,9 @@ impl<'a> Validator<'a> { // check arity against fn info if first term is word and second term is args Synthetic(first, second, rest) => { match (&first.0, &second.0) { + (Ast::And, Ast::Arguments(_)) | (Ast::Or, Ast::Tuple(_)) => { + self.visit(second.as_ref()) + } (Ast::Word(_), Ast::Keyword(_)) => self.visit(first.as_ref()), (Ast::Keyword(_), Ast::Arguments(args)) => { if args.len() != 1 { @@ -302,7 +307,10 @@ impl<'a> Validator<'a> { } } } - _ => unreachable!(), + _ => unreachable!( + "malformed synthetic root with\nfirst: {}\nsecond: {}", + first.0, second.0 + ), } for term in rest { self.visit(term); @@ -393,7 +401,7 @@ impl<'a> Validator<'a> { // we have to do this explicitly here because of arity checking let (expr, span) = clause; self.ast = expr; - self.span = *span; + self.span = span; // add clause arity to arities arities.insert(self.arity()); self.validate(); @@ -462,7 +470,7 @@ impl<'a> Validator<'a> { for clause in body { let (expr, span) = clause; self.ast = expr; - self.span = *span; + self.span = span; let arity = self.arity(); // dbg!(&arity); match arity { @@ -515,7 +523,7 @@ impl<'a> Validator<'a> { InterpolatedPattern(parts, _) => { for (part, span) in parts { if let StringPart::Word(name) = part { - self.span = *span; + self.span = span; match self.bound(name) { Some(_) => self.err(format!("name `{name}` is already bound")), None => self.bind(name.to_string()), @@ -547,7 +555,7 @@ impl<'a> Validator<'a> { (PlaceholderPattern, _) => (), (WordPattern(name), span) => match self.bound(name) { Some(_) => { - self.span = *span; + self.span = span; self.err(format!("name `{name}` is already bound")) } None => self.bind(name.to_string()), diff --git a/src/value.rs b/src/value.rs index 6a21336..eb61c4a 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,7 +1,7 @@ use crate::base::BaseFn; use crate::compiler::Chunk; -use crate::parser::Ast; -use crate::spans::Spanned; +// use crate::parser::Ast; +// use crate::spans::Spanned; use imbl::{HashMap, Vector}; use std::cell::RefCell; use std::rc::Rc;