Scan strings

src/ludus/scanner.clj

@@ -26,6 +26,7 @@
     "ns"
     "recur"
     "repeat"
+    ;; below here, possible
     "when"
     })
 
@@ -91,10 +92,16 @@
   ([scanner token-type]
    (add-token scanner token-type nil))
   ([scanner token-type literal]
-   (update scanner ::tokens conj (token/token token-type (current-lexeme scanner) literal (::line scanner)))))
+   (update scanner ::tokens conj 
+     (token/token 
+       token-type 
+       (current-lexeme scanner) 
+       literal 
+       (::line scanner) 
+       (::start scanner)))))
 
 (defn- add-error [scanner msg]
-  (update scanner ::errors conj {:msg msg :line (::line scanner)}))
+  (update scanner ::errors conj {:msg msg :line (::line scanner) :start (::start scanner)}))
 
 (defn- scan-keyword 
   ([scanner] (scan-keyword scanner scanner))
@@ -114,11 +121,18 @@
     (if (nonzero-digit? current)
       (loop [current current]))))
 
-;; I am working here--trying to figure out how to add a string token
 (defn- add-string 
-  ([scanner] (add-string scanner "")
-  ([scanner string]
-    (let [char (current-char scanner)]))))
+  [scanner]
+  (loop [scanner scanner
+         string ""]
+    (let [char (current-char scanner)]
+      (case char
+        \newline (add-error scanner "Unterminated string.")
+        \" (add-token (advance scanner) ::token/string string)
+        \\ (recur (advance (advance scanner)) (str string (next-char scanner)))
+        (if (at-end? scanner)
+          (add-error scanner "Unterminated string.")
+          (recur (advance scanner) (str string char)))))))
 
 (defn- add-word [scanner])
 
@@ -211,6 +225,12 @@
       (recur (-> scanner (scan-token) (next-token))))))
 
 
-(let [source "|)"]
+(let [source "\"foo\\\nbar\"\n)"]
   (scan source))
 
+;; string scanning is (I think) working
+;; line counting is not working
+;; do I just save a location and then calculate line numbers if an error happens?
+;; next up: numbers!
+
+

src/ludus/token.clj

@@ -3,11 +3,12 @@
 (defn token
   ([type text]
    (token type text nil 1))
-  ([type text literal line]
+  ([type text literal line start]
    {::type type
     ::lexeme text
     ::literal literal
-    ::line line}))
+    ::line line
+    ::start start}))
 
 (defn from-keyword [kw]
   (keyword "ludus.token" kw))