Skip to content

Commit

Permalink
Upgrade tools.reader
Browse files Browse the repository at this point in the history 
Fixes #413
Closes #297
  • Loading branch information
@vemv
vemv committed Aug 1, 2021
1 parent 1b87b1f commit 8ae1262
Show file tree
Hide file tree
Showing 11 changed files with 182 additions and 143 deletions.
16 changes: 16 additions & 0 deletions .circleci/crucible.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,22 @@ fi

grep --silent "== Warnings: 34. Exceptions thrown: 0" output || exit 1

# Exercise tools.reader, see https://github.com/jonase/eastwood/issues/413
cd ../tools.reader || exit 1

if lein with-profile +test update-in :plugins conj "[jonase/eastwood \"RELEASE\"]" -- eastwood | tee output; then
echo "Should have failed! Emitted output:"
cat output
exit 1
fi

if grep --silent "Reflection warning" output; then
echo "Reflection warnings should have been detected as an Eastwood linter and not as mere stdout!"
exit 1
fi

grep --silent "== Warnings: 14. Exceptions thrown: 0" output || exit 1

# Exercise clojurescript as it's large and interesting
cd ../clojurescript || exit 1

Expand Down
1 change: 1 addition & 0 deletions .circleci/tools.reader
Submodule tools.reader added at 3b0211
3 changes: 3 additions & 0 deletions .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@
[submodule ".circleci/clojurescript"]
path = .circleci/clojurescript
url = git@github.com:clojure/clojurescript.git
[submodule ".circleci/tools.reader"]
path = .circleci/tools.reader
url = git@github.com:clojure/tools.reader.git
133 changes: 83 additions & 50 deletions copied-deps/eastwood/copieddeps/dep10/clojure/tools/reader.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
(ns ^{:doc "A clojure reader in clojure"
:author "Bronsa"}
eastwood.copieddeps.dep10.clojure.tools.reader
(:refer-clojure :exclude [read read-line read-string char
(:refer-clojure :exclude [read read-line read-string char read+string
default-data-readers *default-data-reader-fn*
*read-eval* *data-readers* *suppress-read*])
(:require [eastwood.copieddeps.dep10.clojure.tools.reader.reader-types :refer
[read-char unread peek-char indexing-reader?
[read-char unread peek-char indexing-reader? source-logging-push-back-reader source-logging-reader?
get-line-number get-column-number get-file-name string-push-back-reader log-source]]
[eastwood.copieddeps.dep10.clojure.tools.reader.impl.utils :refer :all] ;; [char ex-info? whitespace? numeric? desugar-meta]
[eastwood.copieddeps.dep10.clojure.tools.reader.impl.errors :as err]
Expand All @@ -22,8 +22,10 @@
(:import (clojure.lang PersistentHashSet IMeta
RT Symbol Reflector Var IObj
PersistentVector IRecord Namespace)
eastwood.copieddeps.dep10.clojure.tools.reader.reader_types.SourceLoggingPushbackReader
java.lang.reflect.Constructor
(java.util regex.Pattern List LinkedList)))
java.util.regex.Pattern
(java.util List LinkedList)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; helpers
Expand Down Expand Up @@ -283,7 +285,7 @@
(read-unicode-char rdr ch 16 4 true)))
(if (numeric? ch)
(let [ch (read-unicode-char rdr ch 8 3 false)]
(if (> (int ch) 0337)
(if (> (int ch) 0377)
(err/throw-bad-octal-number rdr)
ch))
(err/throw-bad-escape-char rdr ch)))))
Expand Down Expand Up @@ -672,8 +674,8 @@
ret))

(defn- syntax-quote-coll [type coll]
;; We use sequence rather than seq here to fix http://dev.clojure.org/jira/browse/CLJ-1444
;; But because of http://dev.clojure.org/jira/browse/CLJ-1586 we still need to call seq on the form
;; We use sequence rather than seq here to fix https://clojure.atlassian.net/browse/CLJ-1444
;; But because of https://clojure.atlassian.net/browse/CLJ-1586 we still need to call seq on the form
(let [res (list 'clojure.core/sequence
(list 'clojure.core/seq
(cons 'clojure.core/concat
Expand Down Expand Up @@ -750,7 +752,8 @@

(defn- read-namespaced-map
[rdr _ opts pending-forms]
(let [token (read-token rdr :namespaced-map (read-char rdr))]
(let [[start-line start-column] (starting-line-col-info rdr)
token (read-token rdr :namespaced-map (read-char rdr))]
(if-let [ns (cond
(= token ":")
(ns-name *ns*)
Expand All @@ -763,13 +766,22 @@

(let [ch (read-past whitespace? rdr)]
(if (identical? ch \{)
(let [items (read-delimited :namespaced-map \} rdr opts pending-forms)]
(let [items (read-delimited :namespaced-map \} rdr opts pending-forms)
[end-line end-column] (ending-line-col-info rdr)]
(when (odd? (count items))
(err/throw-odd-map rdr nil nil items))
(let [keys (take-nth 2 items)
vals (take-nth 2 (rest items))]

(RT/map (to-array (mapcat list (namespace-keys (str ns) keys) vals)))))
(with-meta
(RT/map (to-array (mapcat list (namespace-keys (str ns) keys) vals)))
(when start-line
(merge
(when-let [file (get-file-name rdr)]
{:file file})
{:line start-line
:column start-column
:end-line end-line
:end-column end-column})))))
(err/throw-ns-map-no-map rdr token)))
(err/throw-bad-ns rdr token))))

Expand Down Expand Up @@ -828,7 +840,7 @@
(loop [i 0]
(if (>= i ctors-num)
(err/reader-error rdr "Unexpected number of constructor arguments to " (str class)
": got" numargs)
": got " numargs)
(if (== (count (.getParameterTypes ^Constructor (aget all-ctors i)))
numargs)
(Reflector/invokeConstructor class entries)
Expand Down Expand Up @@ -910,47 +922,46 @@
(err/reader-error "Reading disallowed - *read-eval* bound to :unknown"))
(try
(loop []
(log-source reader
(if (seq pending-forms)
(.remove ^List pending-forms 0)
(let [ch (read-char reader)]
(cond
(whitespace? ch) (recur)
(nil? ch) (if eof-error? (err/throw-eof-error reader nil) sentinel)
(= ch return-on) READ_FINISHED
(number-literal? reader ch) (read-number reader ch)
:else (let [f (macros ch)]
(if f
(let [res (f reader ch opts pending-forms)]
(if (identical? res reader)
(recur)
res))
(read-symbol reader ch))))))))
(catch Exception e
(if (ex-info? e)
(let [d (ex-data e)]
(if (= :reader-exception (:type d))
(throw e)
(throw (ex-info (.getMessage e)
(merge {:type :reader-exception}
d
(if (indexing-reader? reader)
{:line (get-line-number reader)
:column (get-column-number reader)
:file (get-file-name reader)}))
e))))
(throw (ex-info (.getMessage e)
(merge {:type :reader-exception}
(if (indexing-reader? reader)
{:line (get-line-number reader)
:column (get-column-number reader)
:file (get-file-name reader)}))
e)))))))
(let [ret (log-source reader
(if (seq pending-forms)
(.remove ^List pending-forms 0)
(let [ch (read-char reader)]
(cond
(whitespace? ch) reader
(nil? ch) (if eof-error? (err/throw-eof-error reader nil) sentinel)
(= ch return-on) READ_FINISHED
(number-literal? reader ch) (read-number reader ch)
:else (if-let [f (macros ch)]
(f reader ch opts pending-forms)
(read-symbol reader ch))))))]
(if (identical? ret reader)
(recur)
ret)))
(catch Exception e
(if (ex-info? e)
(let [d (ex-data e)]
(if (= :reader-exception (:type d))
(throw e)
(throw (ex-info (.getMessage e)
(merge {:type :reader-exception}
d
(if (indexing-reader? reader)
{:line (get-line-number reader)
:column (get-column-number reader)
:file (get-file-name reader)}))
e))))
(throw (ex-info (.getMessage e)
(merge {:type :reader-exception}
(if (indexing-reader? reader)
{:line (get-line-number reader)
:column (get-column-number reader)
:file (get-file-name reader)}))
e)))))))

(defn read
"Reads the first object from an IPushbackReader or a java.io.PushbackReader.
Returns the object read. If EOF, throws if eof-error? is true.
Otherwise returns sentinel. If no stream is providen, *in* will be used.
Otherwise returns sentinel. If no stream is provided, *in* will be used.
Opts is a persistent map with valid keys:
:read-cond - :allow to process reader conditionals, or
Expand All @@ -970,8 +981,16 @@
{:arglists '([] [reader] [opts reader] [reader eof-error? eof-value])}
([] (read *in* true nil))
([reader] (read reader true nil))
([{eof :eof :as opts :or {eof :eofthrow}} reader] (read* reader (= eof :eofthrow) eof nil opts (LinkedList.)))
([reader eof-error? sentinel] (read* reader eof-error? sentinel nil {} (LinkedList.))))
([{eof :eof :as opts :or {eof :eofthrow}} reader]
(when (source-logging-reader? reader)
(let [^StringBuilder buf (:buffer @(.source-log-frames ^SourceLoggingPushbackReader reader))]
(.setLength buf 0)))
(read* reader (= eof :eofthrow) eof nil opts (LinkedList.)))
([reader eof-error? sentinel]
(when (source-logging-reader? reader)
(let [^StringBuilder buf (:buffer @(.source-log-frames ^SourceLoggingPushbackReader reader))]
(.setLength buf 0)))
(read* reader eof-error? sentinel nil {} (LinkedList.))))

(defn read-string
"Reads one object from the string s.
Expand All @@ -996,3 +1015,17 @@
[form]
(binding [gensym-env {}]
(syntax-quote* form)))

(defn read+string
"Like read, and taking the same args. reader must be a SourceLoggingPushbackReader.
Returns a vector containing the object read and the (whitespace-trimmed) string read."
([] (read+string (source-logging-push-back-reader *in*)))
([stream] (read+string stream true nil))
([^SourceLoggingPushbackReader stream eof-error? eof-value]
(let [o (log-source stream (read stream eof-error? eof-value))
s (.trim (str (:buffer @(.source-log-frames stream))))]
[o s]))
([opts ^SourceLoggingPushbackReader stream]
(let [o (log-source stream (read opts stream))
s (.trim (str (:buffer @(.source-log-frames stream))))]
[o s])))
17 changes: 8 additions & 9 deletions copied-deps/eastwood/copieddeps/dep10/clojure/tools/reader/edn.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,15 @@

:else
(loop [sb (StringBuilder.)
ch (do (unread rdr initch) initch)]
ch initch]
(if (or (whitespace? ch)
(macro-terminating? ch)
(nil? ch))
(str sb)
(do (unread rdr ch)
(str sb))
(if (not-constituent? ch)
(err/throw-bad-char rdr kind ch)
(recur (doto sb (.append (read-char rdr))) (peek-char rdr))))))))
(recur (doto sb (.append ch)) (read-char rdr))))))))



Expand All @@ -70,9 +71,7 @@
(if-let [ch (read-char rdr)]
(if-let [dm (dispatch-macros ch)]
(dm rdr ch opts)
(if-let [obj (read-tagged (doto rdr (unread ch)) ch opts)]
obj
(err/throw-no-dispatch rdr ch)))
(read-tagged (doto rdr (unread ch)) ch opts))
(err/throw-eof-at-dispatch rdr)))

(defn- read-unmatched-delimiter
Expand Down Expand Up @@ -231,7 +230,7 @@
(read-unicode-char rdr ch 16 4 true)))
(if (numeric? ch)
(let [ch (read-unicode-char rdr ch 8 3 false)]
(if (> (int ch) 0337)
(if (> (int ch) 0377)
(err/throw-bad-octal-number rdr)
ch))
(err/throw-bad-escape-char rdr ch)))))
Expand Down Expand Up @@ -272,9 +271,9 @@
(let [^String ns (s 0)
^String name (s 1)]
(if (identical? \: (nth token 0))
(err/throw-invalid reader :keyword token) ; No ::kw in edn.
(err/throw-invalid reader :keyword (str \: token)) ; No ::kw in edn.
(keyword ns name)))
(err/throw-invalid reader :keyword token)))
(err/throw-invalid reader :keyword (str \: token))))
(err/throw-single-colon reader))))

(defn- wrapping-reader
Expand Down
31 changes: 12 additions & 19 deletions copied-deps/eastwood/copieddeps/dep10/clojure/tools/reader/impl/errors.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@
(defn throw-invalid-unicode-literal [rdr token]
(throw
(illegal-arg-error rdr
(str "Invalid unicode literal: \\" token "."))))
"Invalid unicode literal: \\" token ".")))

(defn throw-invalid-unicode-escape [rdr ch]
(reader-error
Expand All @@ -105,52 +105,45 @@
(defn throw-eof-at-dispatch [rdr]
(eof-error rdr "Unexpected EOF while reading dispatch character."))

(defn throw-bad-dispatch [rdr ch]
(reader-error rdr "No dispatch macro for " ch "."))

(defn throw-unmatch-delimiter [rdr ch]
(reader-error rdr "Unmatched delimiter " ch "."))

(defn throw-eof-reading [rdr kind & start]
(let [init (case kind :regex "#\"" :string \")]
(eof-error rdr "Unexpected EOF reading " (name kind) " starting " (apply str init start) ".")))

(defn throw-no-dispatch [rdr ch]
(throw-bad-dispatch rdr ch))

(defn throw-invalid-unicode-char[rdr token]
(throw
(illegal-arg-error rdr
(str "Invalid unicode character \\" token "."))))
"Invalid unicode character \\" token ".")))

(defn throw-invalid-unicode-digit-in-token [rdr ch token]
(throw
(illegal-arg-error rdr
(str "Invalid digit " ch " in unicode character \\" token "."))))
"Invalid digit " ch " in unicode character \\" token ".")))

(defn throw-invalid-unicode-digit[rdr ch]
(throw
(illegal-arg-error rdr
(str "Invalid digit " ch " in unicode character."))))
"Invalid digit " ch " in unicode character.")))

(defn throw-invalid-unicode-len[rdr actual expected]
(throw
(illegal-arg-error rdr
(str
"Invalid unicode literal. Unicode literals should be "
expected
"characters long. "
"value suppled is "
actual
"characters long."))))
"Invalid unicode literal. Unicode literals should be "
expected
" characters long. "
"Value supplied is "
actual
" characters long.")))

(defn throw-invalid-character-literal[rdr token]
(reader-error rdr "Invalid character literal \\u" token "."))

(defn throw-invalid-octal-len[rdr token]
(reader-error
rdr
"Invalid octal escape sequence in a character literal:"
"Invalid octal escape sequence in a character literal: "
token
". Octal escape sequences must be 3 or fewer digits."))

Expand Down Expand Up @@ -193,7 +186,7 @@
rdr
"Feature cannot be "
(i/inspect feature)
" Features must be keywords."))
". Features must be keywords."))

(defn throw-ns-map-no-map [rdr ns-name]
(reader-error rdr "Namespaced map with namespace " ns-name " does not specify a map."))
Expand Down
Loading

0 comments on commit 8ae1262

Please sign in to comment.