Retain resource URL in java-parser results, improve caching

.clj-kondo/config.edn

@@ -1,4 +1,4 @@
 {:output {:progress true
           :exclude-files ["analysis.cljc" "meta.cljc" "inspect_test.clj"]}
- :linters {:unused-private-var {:level :warning :exclude [orchard.query-test/a-private orchard.query-test/docd-fn]}}}
-
+ :linters {:unused-private-var {:level :warning :exclude [orchard.query-test/a-private orchard.query-test/docd-fn]}
+           :refer-all {:exclude [clojure.test]}}}

CHANGELOG.md

@@ -21,6 +21,7 @@
 * [#123](https://github.com/clojure-emacs/orchard/pull/123): Fix info lookups from namespaces that don't yet exist
 * [#125](https://github.com/clojure-emacs/orchard/issues/125): Don't fail if the classpath references a non-existing .jar
 * [#128](https://github.com/clojure-emacs/orchard/issues/128): Strengthen `apropos`
+* [#124](https://github.com/clojure-emacs/orchard/pull/124): Remove costly `io/resource` lookup
 
 ## 0.7.1 (2021-04-18)
 

dev/user.clj

@@ -14,5 +14,6 @@
 
 (cond->> ["dev" "src" "test"]
   jdk8?       (into ["src-jdk8"])
-  (not jdk8?) (into ["src-newer-jdks"])
+  (not jdk8?) (into ["src-newer-jdks"
+                     "test-newer-jdks"])
   true        (apply set-refresh-dirs))

project.clj

@@ -87,6 +87,9 @@
              "-Dclojure.main.report=stderr"]
 
   :source-paths ["src" "src-jdk8" "src-newer-jdks"]
+  :test-paths ~(cond-> ["test"]
+                 (not jdk8?)
+                 (conj "test-newer-jdks"))
 
   :profiles {
              ;; Clojure versions matrix
@@ -96,8 +99,8 @@
                                   [org.clojure/clojure "1.8.0" :classifier "sources"]]}
              :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]
                                   [org.clojure/clojure "1.9.0" :classifier "sources"]]}
-             :1.10 {:dependencies [[org.clojure/clojure "1.10.1"]
-                                   [org.clojure/clojure "1.10.1" :classifier "sources"]]}
+             :1.10 {:dependencies [[org.clojure/clojure "1.10.3"]
+                                   [org.clojure/clojure "1.10.3" :classifier "sources"]]}
              :master {:repositories [["snapshots"
                                       "https://oss.sonatype.org/content/repositories/snapshots"]]
                       :dependencies [[org.clojure/clojure "1.11.0-master-SNAPSHOT"]

src-jdk8/orchard/java/legacy_parser.clj

@@ -273,6 +273,8 @@
         (assoc (->> (map parse-info (.classes root))
                     (filter #(= klass (:class %)))
                     (first))
+               ;; relative path on the classpath
                :file path
-               :path (. (io/resource path) getPath))))
+               ;; Full URL, e.g. file:.. or jar:...
+               :resource-url (io/resource path))))
     (catch Abort _)))

src-newer-jdks/orchard/java/parser.clj

@@ -304,7 +304,9 @@
                       (map #(parse-info % root))
                       (filter #(= klass (:class %)))
                       (first))
+                 ;; relative path on the classpath
                  :file path
-                 :path (.getPath (io/resource path)))
+                 ;; Full URL, e.g. file:.. or jar:...
+                 :resource-url (io/resource path))
           (finally (.close (.getJavaFileManager root))))))
     (catch Throwable _)))

src/orchard/info.clj

@@ -195,3 +195,7 @@
   classes. If no source is available, return the relative path as is."
   [^String path]
   {:javadoc (java/resolve-javadoc-path path)})
+
+(comment
+  (info-java 'clojure.lang.RT 'baseLoader)
+  (file-info "clojure/core.clj"))

src/orchard/java.clj

@@ -250,16 +250,6 @@
 
 (def cache (atom {}))
 
-(defn- immutable-source-file?
-  "Return true if the source file is effectively immutable. Specifically, this
-  returns true if no source file is available, or if the source file is in a
-  jar/zip archive."
-  [info]
-  (let [path (:file info)
-        src  (when path (io/resource path))]
-    (or (not src)
-        (re-find #"\.(jar|zip)!" (str src)))))
-
 (defn class-info
   "For the class symbol, return (possibly cached) Java class and member info.
   Members are indexed first by name, and then by argument types to list all
@@ -269,9 +259,11 @@
         info (if cached
                (:info cached)
                (class-info* class))
-        last-modified (if (immutable-source-file? info)
+        resource (:resource-url info)
+        last-modified (if (or (nil? resource)
+                              (util.io/url-to-file-within-archive? resource))
                         0
-                        (.lastModified ^File (io/file (:path info))))
+                        (util.io/last-modified-time resource))
         stale (not= last-modified (:last-modified cached))
         ;; If last-modified in cache mismatches last-modified of the file,
         ;; regenerate class-info.

src/orchard/java/resource.clj

@@ -15,7 +15,9 @@
     s))
 
 (defn project-resources
-  "Get a list of classpath resources."
+  "Get a list of classpath resources, i.e. files that are not clojure/java source
+  or class files. Only consider classpath entries that are directories, does not
+  consider jars."
   []
   (mapcat
    (fn [^File directory]
@@ -32,7 +34,7 @@
                    {:root directory
                     :file file
                     :relpath relpath
-                    :url (io/resource relpath)})))
+                    :url (io/as-url file)})))
           (remove #(.startsWith ^String (:relpath %) "META-INF/"))
           (remove #(re-matches #".*\.(clj[cs]?|java|class)" (:relpath %)))))
    (filter (memfn ^File isDirectory) (map io/as-file (cp/classpath (cp/boot-aware-classloader))))))

src/orchard/util/io.clj

@@ -1,4 +1,10 @@
-(ns orchard.util.io)
+(ns orchard.util.io
+  "Utility functions for dealing with file system objects, and in/out streams."
+  (:require
+   [clojure.java.io :as io])
+  (:import
+   (java.io File)
+   (java.net URL JarURLConnection)))
 
 (defn wrap-silently
   "Middleware that executes `(f)` without printing to `System/out` or `System/err`.
@@ -22,3 +28,55 @@
             (System/setOut old-out))
           (when (= ps System/err) ;; `System/err` may have changed in the meantime (in face of concurrency)
             (System/setErr old-err)))))))
+
+(defn url-protocol
+  "Get the URL protocol as a string, e.g. http, file, jar."
+  [^java.net.URL url]
+  (.getProtocol url))
+
+(defn url-to-file-within-archive?
+  "Does this URL point to a file inside a jar (or zip) archive.
+  i.e. does it use the jar: protocol."
+  [^java.net.URL url]
+  (= "jar" (url-protocol url)))
+
+(defn resource-jarfile
+  "Given a jar:file:...!/... URL, return the location of the jar file on the
+  filesystem. Returns nil on any other URL."
+  ^File [^URL jar-resource]
+  (assert (= "jar" (url-protocol jar-resource)))
+  (let [^JarURLConnection conn (.openConnection jar-resource)
+        inner-url (.getJarFileURL conn)]
+    (when (= "file" (url-protocol inner-url))
+      (io/as-file inner-url))))
+
+(defn resource-artifact
+  "Return the File from which the given resource URL would be loaded.
+
+  For `file:` URLs returns the location of the resource itself, for
+  `jar:..!/...` URLs returns the location of the archive containing the
+  resource. Returns a fully qualified File, even when the URL is relative.
+  Throws when the URL is not a `file:` or `jar:` URL."
+  ^File [^java.net.URL resource]
+  (let [protocol (url-protocol resource)]
+    (case protocol
+      "file"
+      (io/as-file resource)
+      "jar"
+      (resource-jarfile resource)
+      #_else
+      (throw (ex-info (str "URLs with a " protocol
+                           " protocol can't be situated on the filesystem.")
+                      {:resource resource})))))
+
+(defprotocol LastModifiedTime
+  (last-modified-time [this]
+    "Return the last modified time of a File or resource URL."))
+
+(extend-protocol LastModifiedTime
+  java.net.URL
+  (last-modified-time [this]
+    (last-modified-time (resource-artifact this)))
+  java.io.File
+  (last-modified-time [this]
+    (.lastModified this)))

test-newer-jdks/orchard/java/parser_test.clj

@@ -0,0 +1,65 @@
+(ns orchard.java.parser-test
+  (:require [orchard.misc :as misc]
+            [orchard.java.parser :as parser]
+            [clojure.test :refer :all]))
+
+(defn compile-class-from-source
+  "Compile a java file on the classpath.
+  Returns true if all went well."
+  [classname]
+  (let [compiler (javax.tools.ToolProvider/getSystemJavaCompiler)]
+    (.. compiler
+        (getTask
+         nil ;; out
+         nil ;; fileManager
+         nil ;; diagnosticListener
+         nil ;; compilerOptions
+         nil ;; classnames for annotation processing
+         ;; compilationUnits
+         [(.. compiler
+              (getStandardFileManager nil nil nil)
+              (getJavaFileForInput javax.tools.StandardLocation/CLASS_PATH
+                                   classname
+                                   javax.tools.JavaFileObject$Kind/SOURCE))])
+        call)))
+
+(deftest source-info-test
+  (is (compile-class-from-source "orchard.java.DummyClass"))
+
+  (testing "file on the filesystem"
+    (is (= {:class 'orchard.java.DummyClass,
+            :members
+            '{orchard.java.DummyClass
+              {[]
+               {:name orchard.java.DummyClass,
+                :type void,
+                :argtypes [],
+                :argnames [],
+                :doc nil,
+                :line 12,
+                :column 8}},
+              dummyMethod
+              {[]
+               {:name dummyMethod,
+                :type java.lang.String,
+                :argtypes [],
+                :argnames [],
+                :doc "Method-level docstring. @returns the string \"hello\"",
+                :line 18,
+                :column 3}}},
+            :doc
+            "Class level docstring.\n\n```\n   DummyClass dc = new DummyClass();\n```\n\n@author Arne Brasseur",
+            :line 12,
+            :column 1,
+            :file "orchard/java/DummyClass.java"
+            :resource-url (java.net.URL. (str "file:"
+                                              (System/getProperty "user.dir")
+                                              "/test/orchard/java/DummyClass.java"))}
+           ((resolve 'orchard.java.parser/source-info) 'orchard.java.DummyClass))))
+
+  (testing "java file in a jar"
+    (let [rt-info ((resolve 'orchard.java.parser/source-info) 'clojure.lang.RT)]
+      (is (= {:file "clojure/lang/RT.java"}
+             (select-keys rt-info [:file])))
+      (is (re-find #"jar:file:/.*/.m2/repository/org/clojure/clojure/.*/clojure-.*-sources.jar!/clojure/lang/RT.java"
+                   (str (:resource-url rt-info)))))))

test/orchard/java/DummyClass.java

@@ -0,0 +1,21 @@
+package orchard.java;
+
+/**
+ * Class level docstring.
+ *
+ * <pre>
+ *   DummyClass dc = new DummyClass();
+ * </pre>
+ *
+ * @author Arne Brasseur
+ */
+public class DummyClass {
+  /**
+   * Method-level docstring.
+   *
+   * @returns the string "hello"
+   */
+  public String dummyMethod() {
+    return "hello";
+  }
+}

test/orchard/java_test.clj

@@ -81,14 +81,16 @@
 (deftest map-structure-test
   (when jdk-parser?
     (testing "Parsed map structure = reflected map structure"
-      (let [cols #{:file :line :column :doc :argnames :argtypes :path}
+      (let [cols #{:file :line :column :doc :argnames :argtypes :resource-url}
             keys= #(= (set (keys (apply dissoc %1 cols)))
                       (set (keys %2)))
             c1 (class-info* 'clojure.lang.Compiler)
             c2 (with-redefs [source-info (constantly nil)]
                  (class-info* 'clojure.lang.Compiler))]
         ;; Class info
-        (is (keys= c1 c2))
+        (is (keys= c1 c2) (str "Difference: "
+                               (pr-str [(remove (set (keys c1)) (keys c2))
+                                        (remove (set (keys c2)) (keys c1))])))
         ;; Members
         (is (keys (:members c1)))
         (is (= (keys (:members c1))