Compare commits

...

4 Commits

Author SHA1 Message Date
Scott Richmond
064b5df2dd better name for package names 2024-05-10 15:02:55 -04:00
Scott Richmond
f1a1e9ec62 notes towards an ast-validator 2024-05-10 15:02:37 -04:00
Scott Richmond
8f284f1e65 first draft of all the things; many bugs abound 2024-05-10 15:02:22 -04:00
Scott Richmond
232261b646 add uppercase pkg to scanner 2024-05-10 14:29:12 -04:00
3 changed files with 422 additions and 59 deletions

View File

@ -71,7 +71,7 @@
# errors
# terminators are what terminate expressions
(def terminators [:break :newline :semicolon :eof])
(def terminators [:break :newline :semicolon])
(defn- terminates?
"Returns true if the current token in the parser is a terminator"
@ -273,6 +273,10 @@
(def ast @{:type :args :data @[] :token origin :partial false})
(while (separates? parser) (advance parser)) # consume any separators
(while (not (check parser :rparen))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed paren"})
(array/push (parser :errors) err)
(error err))
(def origin (current parser))
(def term (if (check parser :placeholder)
(if (ast :partial)
@ -314,6 +318,10 @@
(def ast {:type :tuple :data @[] :token origin})
(while (separates? parser) (advance parser)) # consume any separators
(while (not (check parser :rparen))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed paren"})
(array/push (parser :errors) err)
(error err))
(def term (try (nonbinding parser) ([e] e)))
(array/push (ast :data) term)
(try (separators parser)
@ -327,6 +335,10 @@
(def ast {:type :list :data @[] :token origin})
(while (separates? parser) (advance parser))
(while (not (check parser :rbracket))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed bracket"})
(array/push (parser :errors) err)
(error err))
(def origin (current parser))
(def term (if (check parser :splat)
(do
@ -347,6 +359,10 @@
(def ast {:type :set :data @[] :token origin})
(while (separates? parser) (advance parser))
(while (not (check parser :rbrace))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed brace"})
(array/push (parser :errors) err)
(error err))
(def origin (current parser))
(def term (if (check parser :splat)
(do
@ -367,6 +383,10 @@
(def ast {:type :dict :data @[] :token origin})
(while (separates? parser) (advance parser))
(while (not (check parser :rbrace))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed brace"})
(array/push (parser :errors) err)
(error err))
(def origin (current parser))
(def term (case (type origin)
:splat {:type :splat :data (try (word (advance parser)) ([e] e)) :token origin}
@ -397,14 +417,78 @@
(advance parser)
{:type :word :data (origin :lexeme) :token origin})
(defn- tuple-pattern [parser])
(defn- tup-pattern [parser]
(def origin (current parser))
(advance parser) # consume the :lparen
(def ast {:type :tuple :data @[] :token origin})
(while (separates? parser) (advance parser)) # consume any separators
(while (not (check parser :rparen))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed paren"})
(array/push (parser :errors) err)
(error err))
(def origin (current parser))
(def term (if (check parser :splat)
(do
(advance parser)
(def splatted (if (check parser :word) (word parser) nil))
{:type :splat :data splatted :token origin})
(try (pattern parser) ([e] e))))
(array/push (ast :data) term)
(try (separators parser)
([e] (pp e) (array/push (ast :data) e))))
(advance parser)
ast)
(defn- list-pattern [parser])
(defn- list-pattern [parser]
(def origin (current parser))
(advance parser)
(def ast {:type :list :data @[] :token origin})
(while (separates? parser) (advance parser))
(while (not (check parser :rbracket))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed bracket"})
(array/push (parser :errors) err)
(error err))
(def origin (current parser))
(def term (if (check parser :splat)
(do
(advance parser)
(def splatted (if (check parser :word) (word parser) nil))
{:type :splat :data splatted :token origin})
(try (pattern parser) ([e] e))))
(array/push (ast :data) term)
(try (separators parser)
([e] (array/push (ast :data) e))))
(advance parser)
ast)
(defn- dict-pattern [parser])
(defn- string-pattern [parser])
(defn- dict-pattern [parser]
(def origin (current parser))
(advance parser)
(def ast {:type :dict :data @[] :token origin})
(while (separates? parser) (advance parser))
(while (not (check parser :rbrace))
(when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed brace"})
(array/push (parser :errors) err)
(error err))
(def origin (current parser))
(def term (case (type origin)
:splat {:type :splat :data (try (word (advance parser)) ([_] nil)) :token origin}
:word (try (word parser) ([e] e))
:keyword (do
(def key (capture kw parser))
(def value (capture pattern parser))
{:type :pair :data [key value] :token origin})
(try (panic parser (string expect "expected dict term, got " (type origin))) ([e] e))
))
(array/push (ast :data) term)
(try (separators parser) ([e] (array/push (ast :data) e))))
(advance parser)
ast)
### TODO: add as patterns
(defrec pattern [parser]
(case (-> parser current type)
:nil (nill parser)
@ -415,13 +499,27 @@
:string (str parser)
:word (word-pattern parser)
:placeholder (placeholder parser)
:lparen (tuple-pattern parser)
:ignored (placeholder parser)
:lparen (tup-pattern parser)
:lbracket (list-pattern parser)
:startdict (dict-pattern parser)
:interpolated (string-pattern parser)
:interpolated (interpolated parser)
(panic parser "expected pattern")
))
### let
# let {pattern} = {nonbinding}
(defn- lett [parser]
(def ast {:type :let :data @[] :token (current parser)})
(advance parser) # consume the let
(array/push (ast :data) (capture pattern parser))
(if-let [err (expect-ret parser :equals)]
(do (array/push (ast :data) err) (break ast))
(advance parser))
(accept-many parser :newline)
(array/push (ast :data) (capture nonbinding parser))
ast)
### conditional forms
# if {simple} then {nonbinding} else {nonbinding}
(defn- iff [parser]
@ -464,8 +562,10 @@
(accept-many parser :newline :semicolon :break) # ...and any additional ones
err)))
# when { {when-clause}+ }
(defn- whenn [parser]
(def ast {:type :when :data @[] :origin (current parser)})
(def origin (current parser))
(def ast {:type :when :data @[] :token origin})
(advance parser) # consume when
(if-let [err (expect-ret parser :lbrace)]
(do
@ -473,22 +573,256 @@
(break ast)) # early return; just bail if we don't have {
(advance parser))
(accept-many parser :newline)
(while (not (check parser :rbrace :eof)) # make sure we don't roll past eof
(while (not (check parser :rbrace )) # make sure we don't roll past eof
(when (check parser :eof) (error {:type :error :token origin :data ast :msg "unclosed brace"}))
(array/push (ast :data) (capture when-clause parser)))
(advance parser)
ast)
(defn- match [parser])
### TODO: add guards to patterns
(defn- match-clause [parser]
(try
(do
(def ast {:type :clause :data @[] :origin (current parser)})
(def lhs (pattern parser))
(expect parser :arrow)
(advance parser)
(accept-many parser :newline)
(def rhs (nonbinding parser))
(terminator parser)
[lhs rhs])
([err]
(accept-many parser ;terminators)
err)))
(defn- matchh [parser]
(def origin (current parser))
(def ast {:type :match :data @[] :token origin})
(expect parser :match)
(advance parser)
(try
(do
(simple parser)
(expect parser :with) (advance parser)
(expect parser :lbrace) (advance parser)
(accept-many parser :newline)
(while (not (check parser :rbrace))
(when (check parser :eof) (error {:type :error :token origin :data ast :msg "unclosed brace"}))
(array/push (ast :data) (match-clause parser)))
(advance parser)
ast)
([err] err)))
# {pattern} = {nonbinding} {terminators}
(defn- with-clause [parser]
(try
(do
(def lhs (pattern parser))
(expect parser :equals) (advance parser)
(def rhs (nonbinding parser))
(terminator parser)
[lhs rhs]
)
([err]
(accept-many parser ;terminators)
err)
)
)
# with { {clauses}+ } {terminators}? then {nonbinding} {terminators}? else {nonbinding}
(defn- withh [parser]
(def origin (current parser))
(expect parser :with) (advance parser)
(try
(do
(expect parser :lbrace) (advance parser)
(accept-many parser ;terminators)
(def data @[])
(def clauses @[])
(array/push clauses (with-clause parser))
(accept-many parser ;terminators)
(while (not (check parser :rbrace))
(if (check parser :eof)
(error {:type :error :data data :token origin :msg "unclosed brace"}))
(array/push clauses (with-clause parser))
(accept-many parser ;terminators))
(array/push data clauses)
(accept-many parser :newline)
(expect parser :then) (advance parser)
(array/push data (nonbinding parser))
(accept-many parser :newline)
(expect parser :else) (advance parser)
(array/push data (nonbinding parser))
{:type :with :data data :token origin})
([err] err)
)
)
### function forms
(defn- fnn [parser])
(defn- fn-simple [parser]
(try
(do
(def lhs (tuple-pattern parser))
(expect parser :arrow) (advance parser)
(def rhs (nonbinding parser))
[[lhs rhs]]
)
([err] err)
)
)
(defn- lambda [parser])
(defn- fn-clause [parser]
(def origin (current parser))
(try
(do
(def lhs (tuple-pattern parser))
(expect parser :arrow) (advance parser)
(def rhs (nonbinding parser))
(terminator parser)
[lhs rhs])
([err]
(advance parser)
(accept-many parser ;terminators)
err
)
)
)
(defn- fn-clauses [parser]
(def origin (current parser))
(expect parser :lbrace) (advance parser)
(accept-many parser ;terminators)
(def data @[])
(while (not (check parser :rbrace))
(if (check parser :eof)
(error {:type :error :token origin :data data :msg "unclosed brace"}))
(array/push data (capture fn-clause parser)))
data)
(defn- fnn [parser]
(try
(do
(def origin (current parser))
(expect parser :fn) (advance parser)
(def name (word parser))
(def data (case (-> parser peek type)
:lbrace (fn-clauses parser)
:lparen (fn-simple parser)
(panic parser (string "expected clause or clauses, got " (-> current parser type)))))
{:type :fn :name name :data data :token origin}
)
([err] err)))
(defn- lambda [parser]
(def origin (current parser))
(expect parser :fn) (advance parser)
{:type :fn :data (fn-simple parser) :token origin})
### compoound forms
(defn- block [parser])
(defn- block [parser]
(def origin (current parser))
(expect parser :lbrace) (advance parser)
(accept-many parser ;terminators)
(def data @[])
(while (not (check parser :rbrace))
(if (check parser :eof)
(error {:type :error :token origin :data data :msg "unclosed brace"}))
(array/push data (capture expr parser))
(terminator parser))
{:type :block :data data :token origin})
(defn- doo [parser])
### TODO: decide whether this design works
# newlines are allowed AFTER pipelines, but not before
# eg. `do foo > \n bar > \n baz`
# but not `do foo \n > bar \n > baz`
# Otherwise, this isn't LR
(defn- doo [parser]
(def origin (current parser))
(expect parser :do) (advance parser)
(def data @[])
(array/push data (capture simple parser))
(while (check parser :pipeline)
(accept-many parser :newline)
(array/push data (capture simple parser)))
{:type :do :data data :token origin})
### refs, pkgs, nses, etc.
(defn- ref [parser]
(def origin (current parser))
(expect parser :ref) (advance parser)
(try
(do
(def name (word parser))
(expect parser :equals) (advance parser)
(def value (nonbinding parser))
{:type :ref :data value :name name :token origin})
([err] err)))
(defn- pkg-name [parser]
(expect parser :pkg-name)
(def origin (current parser))
(advance parser)
{:type :pkg-name :data (origin :lexeme) :token origin})
(defn- usee [parser]
(def origin (current parser))
(expect parser :use) (advance parser)
(try
(do
{:type :use :data (pkg-name parser) :token origin})
([err] err)))
(defn- pkg [parser]
(try
(do
(def origin (current parser))
(expect :pkg) (advance parser)
(def name (pkg-name parser))
(expect :lbrace) (advance parser)
(accept-many parser ;terminators)
(def data @[])
(while (not (check parser :rbrace))
(when (check parser :eof)
(def err {:type :error :token origin :data data :msg "unclosed brace"})
(array/push (parser :errors) err)
(error err))
(case (-> parser current type)
:keyword (do
(def origin (current parser))
(def key (capture kw parser))
(def value (capture simple parser))
(array/push data {:type :pair :data [key value] :token origin}))
:word (array/push (capture word parser))
(panic parser "expected dict term"))
(terminator parser))
{:type :pkg :data data :token origin :name name})
([err] err)))
(defn- ns [parser]
(try
(do
(def origin (current parser))
(expect :ns) (advance parser)
(def name (pkg-name parser))
(def body (block parser))
{:type :ns :data body :name name :token origin})
([err] err)))
(defn- importt [parser]
(def origin (current parser))
(expect parser :import) (advance parser)
(def path (str parser))
(expect parser :as) (advance parser)
(def name (pkg-name parser))
{:type :import :data path :name name :token origin})
### tests
(defn- testt [parser]
(def origin (current parser))
(expect parser :test) (advance parser)
(def desc (str parser))
(def body (nonbinding parser))
{:type :test :data [desc body] :token origin})
### expressions
# four levels of expression complexity:
@ -519,6 +853,7 @@
# non-binding expressions
# the rhs of lets, clauses, inside conditional forms, etc.
# any form that does not bind a name
(defrec nonbinding [parser]
(def curr (current parser))
(case (type curr)
@ -532,7 +867,7 @@
# strings
:string (str parser)
### TODO: interpolated strings
:interpolated (unreachable)
:interpolated (interpolated parser)
# collection literals
:lparen (tup parser)
@ -546,17 +881,17 @@
# conditional forms
:if (iff parser)
:when (whenn parser)
:match (unreachable)
:with (unreachable)
:match (matchh parser)
:with (withh parser)
# do
:do (unreachable)
:do (doo parser)
# fn: but only lambda
:fn (unreachable)
:fn (lambda parser)
# blocks
:lbrace (unreachable)
:lbrace (block parser)
(panic parser (string "expected nonbinding expression, got " (type curr)))
)
@ -565,9 +900,12 @@
(defrec expr [parser]
(def curr (current parser))
(case (type curr)
:let (unreachable)
:fn (unreachable)
:ref (unreachable)
# binding forms
:let (lett parser)
:fn (fnn parser)
:ref (ref parser)
# nonbinding forms
:nil (nill parser)
:true (bool parser)
:false (bool parser)
@ -581,10 +919,10 @@
:word (word parser)
:if (iff parser)
:when (whenn parser)
:match (unreachable)
:with (unreachable)
:do (unreachable)
:lbrace (unreachable)
:match (matchh parser)
:with (withh parser)
:do (doo parser)
:lbrace (block parser)
(panic parser (string "expected expression, got " (type curr)))
)
)
@ -592,41 +930,41 @@
(defrec toplevel [parser]
(def curr (current parser))
(case (type curr)
:pkg (unreachable)
:ns (unreachable)
:test (unreachable)
:import (unreachable)
:use (unreachable)
:let (unreachable)
:fn (unreachable)
:ref (unreachable)
:nil (nill parser)
:true (bool parser)
:false (bool parser)
:number (num parser)
:keyword (kw parser)
:string (str parser)
:lparen (tup parser)
:lbracket (list parser)
:startdict (dict parser)
:startset (sett parser)
:word (word parser)
:if (iff parser)
:when (whenn parser)
:match (unreachable)
:with (unreachable)
:do (unreachable)
:lbrace (unreachable)
(panic parser (string "expected expression, got " (type curr)))
# toplevel-only
:pkg (pkg parser)
:ns (ns parser)
:test (testt parser)
:import (importt parser)
:use (usee parser)
# all the other expressions
(expr parser)
)
)
(def script [parser]
(def origin (current parser))
(def lines @[])
(while (not (check parser :eof))
(array/push lines (capture toplevel parser))
(capture terminator parser))
{:type :script :data lines :token origin})
(do
#(comment
(def source `"foo { bar } baz \{quux} {fuzz}"`)
(def source `match foo with {
1 -> 2 4
_ -> :foo
}
`)
(def scanned (s/scan source))
(def a-parser (new-parser scanned))
(def parsed (simple a-parser))
(-> parsed)
(def parsed (try (matchh a-parser) ([e] e)))
)
# FIXME:
# TODO:
# - if guards on patterns
# DECIDE:
# - when to use a flat try/catch format, and when to use capture/expect-ret to get values instead of errors

View File

@ -217,6 +217,16 @@
:else (add-error scanner (string "Unexpected " curr " after word " word ".")))))
(recur scanner (buffer char)))
(defn- add-pkg
[char scanner]
(defn recur [scanner pkg]
(let [curr (current-char scanner)]
(cond
(terminates? curr) (add-token scanner :pkg-name :none)
(word-char? curr) (recur (advance scanner) (buffer/push pkg curr))
:else (add-error scanner (string "unexpected " curr " after pkg name " pkg)))))
(recur scanner (buffer char)))
(defn- add-ignored
[scanner]
(defn recur [scanner ignored]
@ -302,7 +312,7 @@
(cond
(whitespace? char) scanner ## for now just skip whitespace characters
(digit? char) (add-number char scanner)
(upper? char) (add-word char scanner) ## no datatypes for now
(upper? char) (add-pkg char scanner)
(lower? char) (add-word char scanner)
:else (add-error scanner (string "Unexpected character: " char))))))

15
janet/validate.janet Normal file
View File

@ -0,0 +1,15 @@
### A validator for a Ludus AST
(comment
Tracking here, before I start writing this code, the kinds of validation we're hoping to accomplish:
* splats come at the end of list, tuple, and dict patterns
* no unbound names
* no re-bound names
* correct imports
* no unbound names with `use` forms
* no circular imports
* first-level property access with pkg, e.g. `Foo :bar`--bar must be on Foo
)