twc/ludus
4
0
Fork 0
Code Issues 28 Pull Requests Packages Projects 8 Releases Wiki Activity

Individuate reserved words in tokens

Browse Source
This commit is contained in:
Scott Richmond 2022-02-14 16:46:42 -05:00
parent 3806683c99
commit e2f8b0cb6c
1 changed files with 28 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
src/ludus/scanner.clj
View File

@ -8,33 +8,34 @@
(def reserved-words (def reserved-words
"List of Ludus reserved words." "List of Ludus reserved words."
;; see ludus-spec repo for more info ;; see ludus-spec repo for more info
#{ {
"as" "as" ::token/as
"cond" "cond" ::token/cond
"else" "else" ::token/else
"false" "false" ::token/false
"fn" "fn" ::token/fn
"if" "if" ::token/if
"match" "let" ::token/let
"mut" "match" ::token/match
"nil" "mut" ::token/mut
"panic!" "nil" ::token/nil
"then" "panic!" ::token/panic
"true" "then" ::token/then
"var" "true" ::token/true
"with" "var" ::token/var
"with" ::token/with
;; below here, probable ;; below here, probable
"defer" "defer" ::token/defer
"gen" "gen" ::token/gen
"loop" "loop" ::token/loop
"ns" "ns" ::token/ns
"recur" "recur" ::token/recur
"repeat" "repeat" ::token/repeat
"test" "test" ::token/test
"wait" "wait" ::token/wait
"yield" "yield" ::token/yield
;; below here, possible ;; below here, possible
"when" "when" ::token/when
}) })
@ -174,9 +175,7 @@
word (str char)] word (str char)]
(let [curr (current-char scanner)] (let [curr (current-char scanner)]
(cond (cond
(terminates? curr) (if (contains? reserved-words word) (terminates? curr) (add-token scanner (get reserved-words word ::token/word))
(add-token scanner ::token/reserved)
(add-token scanner ::token/word))
(word-char? curr) (recur (advance scanner) (str word curr)) (word-char? curr) (recur (advance scanner) (str word curr))
:else (add-error scanner (str "Unexpected " curr " after word " word ".")))))) :else (add-error scanner (str "Unexpected " curr " after word " word "."))))))
@ -281,11 +280,7 @@
;; word matches ;; word matches
(cond (cond
(whitespace? char) (loop [scanner scanner ws (str char)] ;; for now, don't add whitespace tokens
(let [curr (current-char scanner)]
(if (whitespace? curr)
(recur (advance scanner) (str ws curr))
(add-token scanner ::token/ws))))
(digit? char) (add-number char scanner) (digit? char) (add-number char scanner)
(alpha? char) (add-word char scanner) (alpha? char) (add-word char scanner)
:else (add-error scanner (str "Unexpected character: " char)))))) :else (add-error scanner (str "Unexpected character: " char))))))