start fixing off-by-one errors: script, block, tuple, args, tup-patt

This commit is contained in:
Scott Richmond 2024-07-16 19:40:40 -04:00
parent d416511b48
commit cb7098ac4e

View File

@ -3,6 +3,9 @@
### We still need to scan some things ### We still need to scan some things
(import /src/scanner :as s) (import /src/scanner :as s)
# stash janet type
(def janet-type type)
(defmacro declare (defmacro declare
"Forward-declares a function name, so that it can be called in a mutually recursive manner." "Forward-declares a function name, so that it can be called in a mutually recursive manner."
[& names] [& names]
@ -75,7 +78,7 @@
(has-value? terminators ttype)) (has-value? terminators ttype))
# breakers are what terminate panics # breakers are what terminate panics
(def breaking [:break :newline :semicolon :comma :eof :then :else]) (def breaking [:break :newline :semicolon :comma :eof :then :else :arrow])
(defn- breaks? (defn- breaks?
"Returns true if the current token in the parser should break a panic" "Returns true if the current token in the parser should break a panic"
@ -89,11 +92,15 @@
[parser message] [parser message]
# (print "Panic in the parser: " message) # (print "Panic in the parser: " message)
(def origin (current parser)) (def origin (current parser))
(advance parser) # (advance parser)
(def skipped @[origin]) (def skipped @[origin])
(while (not (breaks? parser)) (while (not (breaks? parser))
(array/push skipped (current parser)) (array/push skipped (current parser))
(advance parser)) (advance parser))
# but we actually don't want to advance past all breaking tokens
# this is the source of the off-by-one errors
# only newlines and semicolons should be skipped here; everything else needs to be parsed normaly
# wait, except for we don't actually advance past the breaking token
(array/push skipped (current parser)) (array/push skipped (current parser))
(def err {:type :error :data skipped :token origin :msg message}) (def err {:type :error :data skipped :token origin :msg message})
(update parser :errors array/push err) (update parser :errors array/push err)
@ -279,8 +286,8 @@
(def origin (current parser)) (def origin (current parser))
(advance parser) # consume the :lparen (advance parser) # consume the :lparen
(def ast @{:type :args :data @[] :token origin :partial false}) (def ast @{:type :args :data @[] :token origin :partial false})
(while (separates? parser) (advance parser)) # consume any separators
(while (not (check parser :rparen)) (while (not (check parser :rparen))
(accept-many parser ;separates)
(when (check parser :eof) (when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed paren"}) (def err {:type :error :token origin :msg "unclosed paren"})
(array/push (parser :errors) err) (array/push (parser :errors) err)
@ -299,8 +306,7 @@
{:type :placeholder :token origin})) {:type :placeholder :token origin}))
(capture nonbinding parser))) (capture nonbinding parser)))
(array/push (ast :data) term) (array/push (ast :data) term)
(try (separators parser) (capture separators parser))
([e] (array/push (ast :data) e))))
(advance parser) (advance parser)
ast) ast)
@ -337,16 +343,16 @@
(def origin (current parser)) (def origin (current parser))
(advance parser) # consume the :lparen (advance parser) # consume the :lparen
(def ast {:type :tuple :data @[] :token origin}) (def ast {:type :tuple :data @[] :token origin})
(while (separates? parser) (advance parser)) # consume any separators # (while (separates? parser) (advance parser)) # consume any separators
(while (not (check parser :rparen)) (while (not (check parser :rparen))
(accept-many parser ;separates)
(when (check parser :eof) (when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed paren"}) (def err {:type :error :token origin :msg "unclosed paren"})
(array/push (parser :errors) err) (array/push (parser :errors) err)
(error err)) (error err))
(def term (capture nonbinding parser)) (def term (capture nonbinding parser))
(array/push (ast :data) term) (array/push (ast :data) term)
(try (separators parser) (capture separators parser))
([e] (array/push (ast :data) e))))
(advance parser) (advance parser)
ast) ast)
@ -452,8 +458,8 @@
(def origin (current parser)) (def origin (current parser))
(advance parser) # consume the :lparen (advance parser) # consume the :lparen
(def ast {:type :tuple :data @[] :token origin}) (def ast {:type :tuple :data @[] :token origin})
(while (separates? parser) (advance parser)) # consume any separators
(while (not (check parser :rparen)) (while (not (check parser :rparen))
(accept-many parser ;separates)
(when (check parser :eof) (when (check parser :eof)
(def err {:type :error :token origin :msg "unclosed paren"}) (def err {:type :error :token origin :msg "unclosed paren"})
(array/push (parser :errors) err) (array/push (parser :errors) err)
@ -466,8 +472,7 @@
{:type :splat :data splatted :token origin}) {:type :splat :data splatted :token origin})
(capture pattern parser))) (capture pattern parser)))
(array/push (ast :data) term) (array/push (ast :data) term)
(try (separators parser) (capture separators parser))
([e] (array/push (ast :data) e))))
(advance parser) (advance parser)
ast) ast)
@ -573,9 +578,12 @@
(array/push (ast :data) (capture nonbinding parser)) (array/push (ast :data) (capture nonbinding parser))
ast) ast)
(defn- literal-terminator? [token]
(def tok-type (token :type))
(or (= :newline tok-type) (= :semicolon tok-type)))
(defn- terminator [parser] (defn- terminator [parser]
(if-not (terminates? parser) (if-not (terminates? parser)
# this line panics, captures the panic, advances the parser, and re-throws the error; solves an off-by-one error
(panic parser "expected terminator")) (panic parser "expected terminator"))
(advance parser) (advance parser)
(while (terminates? parser) (advance parser))) (while (terminates? parser) (advance parser)))
@ -798,13 +806,13 @@
(defn- block [parser] (defn- block [parser]
(def origin (current parser)) (def origin (current parser))
(expect parser :lbrace) (advance parser) (expect parser :lbrace) (advance parser)
(accept-many parser ;terminators)
(def data @[]) (def data @[])
(while (not (check parser :rbrace)) (while (not (check parser :rbrace))
(accept-many parser ;terminators)
(if (check parser :eof) (if (check parser :eof)
(error {:type :error :token origin :data data :msg "unclosed brace"})) (error {:type :error :token origin :data data :msg "unclosed brace"}))
(array/push data (capture expr parser)) (array/push data (capture expr parser))
(terminator parser)) (capture terminator parser))
(advance parser) (advance parser)
{:type :block :data data :token origin}) {:type :block :data data :token origin})
@ -1103,7 +1111,7 @@
(def origin (current parser)) (def origin (current parser))
(def lines @[]) (def lines @[])
(while (not (check parser :eof)) (while (not (check parser :eof))
(accept-many parser :newline) (accept-many parser ;terminators)
(array/push lines (capture toplevel parser)) (array/push lines (capture toplevel parser))
(capture terminator parser)) (capture terminator parser))
{:type :script :data lines :token origin}) {:type :script :data lines :token origin})
@ -1114,13 +1122,35 @@
(set (parser :ast) ast) (set (parser :ast) ast)
parser) parser)
# (do (defn- pp-tok [token]
(comment (if (not token) (break "nil"))
(def {:line line :lexeme lex :type type :start start} token)
(string "<" line "[" start "]" ": " type ": " lex ">"))
(defn- pp-ast [ast &opt indent]
(default indent 0)
(def {:token token :data data :type type} ast)
(def pretty-tok (pp-tok token))
(def data-rep (if (= :array (janet-type data))
(string "[\n"
(string/join (map (fn [x] (pp-ast x (inc indent))) data)
(string (string/repeat " " indent) "\n"))
"\n" (string/repeat " " indent) "]")
data
))
(string (string/repeat " " indent) type ": " pretty-tok " " data-rep)
)
(do
# (comment
(def source ` (def source `
[...x] (,,,,,,1, 2, foo, :three)
`) `)
(def scanned (s/scan source)) (def scanned (s/scan source))
# (print "\n***NEW PARSE***\n") # (print "\n***NEW PARSE***\n")
(def a-parser (new-parser scanned)) (def a-parser (new-parser scanned))
(try (def parsed (pattern a-parser)) ([e] (pp e))) (def parsed (script a-parser))
(pp (map (fn [err] (err :msg)) ((parse scanned) :errors)))
(print (pp-ast ((parse scanned) :ast)))
) )