diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b2b543b..f093facc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## master (unreleased) +## 0.9.0 (2022-01-8) + +### Changes + +* [#51](https://github.com/clojure-emacs/orchard/issues/51): extend find-usages + * `orchard.xref/fn-deps` now also finds anonymous function dependencies + * added `orchard.xref/fn-deps-class` as a lower level API so you can still get the main functions deps only + * added `orchard.xref/fn-transitive-deps` + ## 0.8.0 (2021-12-15) ### Changes diff --git a/README.md b/README.md index 7edd6e8b..c90438a0 100644 --- a/README.md +++ b/README.md @@ -84,12 +84,26 @@ Just add `orchard` as a dependency and start hacking. Consult the [API documentation](https://cljdoc.org/d/cider/orchard/CURRENT) to get a better idea about the functionality that's provided. -#### Using `enrich-classpath` for best results +### Using `enrich-classpath` for best results There are features that Orchard intends to provide (especially, those related to Java interaction) which need to assume a pre-existing initial classpath that already has various desirable items, such as the JDK sources, third-party sources, special jars such as `tools` (for JDK8), a given project's own Java sources... all that is a domain in itself, which is why our [enrich-classpath](https://github.com/clojure-emacs/enrich-classpath) project does it. For getting the most out of Orchard, it is therefore recommended/necessary to use `enrich-classpath`. Please refer to its installation/usage instructions. +### xref/fn-deps and xref/fn-refs limitations + +These functions use a Clojure compiler implementation detail to find references to other function var dependencies. + +You can find a more in-depth explanation in this [post](https://lukas-domagala.de/blog/clojure-analysis-and-introspection.html). + +The important implications from this are: + +* very fast +* functions marked with meta :inline will not be found (inc, +, ...) +* redefining function vars that include lambdas will still return the dependencies of the old plus the new ones +([explanation](https://lukas-domagala.de/blog/clojure-compiler-class-cache.html)) +* does not work on AoT compiled functions + ## Configuration options So far, Orchard follows these options, which can be specified as Java system properties @@ -98,6 +112,12 @@ So far, Orchard follows these options, which can be specified as Java system pro * `"-Dorchard.initialize-cache.silent=true"` (default: `true`) * if `false`, the _class info cache_ initialization may print warnings (possibly spurious ones). +## Tests and formatting + +To run the CI tasks locally use: + +`make test cljfmt kondo eastwood` + ## History Originally [SLIME][] was the most diff --git a/src/orchard/xref.clj b/src/orchard/xref.clj index c11dfb90..84416430 100644 --- a/src/orchard/xref.clj +++ b/src/orchard/xref.clj @@ -3,9 +3,12 @@ references." {:added "0.5"} (:require + [clojure.repl :as repl] + [clojure.set :as set] + [clojure.string :as str] [orchard.query :as q])) -(defn- as-val +(defn- to-fn "Convert `thing` to a function value." [thing] (cond @@ -13,20 +16,71 @@ (symbol? thing) (var-get (find-var thing)) (fn? thing) thing)) +(defn- fn-name [^java.lang.Class f] + (-> f .getName repl/demunge symbol)) + +(defn fn-deps-class + "Returns a set with all the functions invoked by `v`. + `v` can be a function class or a symbol." + {:added "0.9"} + [v] + (let [^java.lang.Class v (if (class? v) + v + (eval v))] + (into #{} (keep (fn [^java.lang.reflect.Field f] + (or (and (identical? clojure.lang.Var (.getType f)) + (java.lang.reflect.Modifier/isPublic (.getModifiers f)) + (java.lang.reflect.Modifier/isStatic (.getModifiers f)) + (-> f .getName (.startsWith "const__")) + (.get f (fn-name v))) + nil)) + (.getDeclaredFields v))))) + +(def ^:private class-cache + "Reference to Clojures class cache. + This holds of classes compiled by the Clojure compiler, + one class per function and one per repl eval. + This field is package private, so it has to be set to + accessible otherwise an IllegalAccess exception would + be thrown." + (let [classCache* (.getDeclaredField clojure.lang.DynamicClassLoader "classCache")] + (.setAccessible classCache* true) + (.get classCache* clojure.lang.DynamicClassLoader))) + (defn fn-deps - "Returns a set with all the functions invoked by `val`. - `val` can be a function value, a var or a symbol." + "Returns a set with all the functions invoked inside `v` or any contained anonymous functions. + `v` can be a function value, a var or a symbol. + If a function was defined multiple times, old lambda deps will + be returned. + This does not return functions marked with meta :inline like `+` + since they are already compiled away at this point." {:added "0.5"} - [val] - (let [val (as-val val)] - (set (some->> val class .getDeclaredFields - (keep (fn [^java.lang.reflect.Field f] - (or (and (identical? clojure.lang.Var (.getType f)) - (java.lang.reflect.Modifier/isPublic (.getModifiers f)) - (java.lang.reflect.Modifier/isStatic (.getModifiers f)) - (-> f .getName (.startsWith "const__")) - (.get f val)) - nil))))))) + [v] + (when-let [^clojure.lang.AFn v (to-fn v)] + (let [f-class-name (-> v .getClass .getName)] + ;; this uses the implementation detail that the clojure compiler always + ;; prefixes names of lambdas with the name of its surrounding function class + (into #{} (comp (filter (fn [[k _v]] (clojure.string/includes? k f-class-name))) + (map (fn [[_k value]] (.get ^java.lang.ref.Reference value))) + (mapcat fn-deps-class)) + class-cache)))) + +(defn fn-transitive-deps + "Returns a set with all the functions invoked inside `v` or inside those functions. + `v` can be a function value, a var or a symbol." + {:added "0.9"} + [v] + (let [deps (fn-deps v)] + (loop [checked #{} + to-check (into [] deps) + deps deps] + (cond + (empty? to-check) deps + :else (let [[current & remaining] to-check + new-deps (fn-deps current)] + (recur (conj checked current) + (concat remaining (filter #(contains? deps %) new-deps)) + (set/union deps new-deps))))))) (defn- fn->sym "Convert a function value `f` to symbol." @@ -45,8 +99,37 @@ "Find all functions that refer `var`. `var` can be a function value, a var or a symbol." {:added "0.5"} - [var] - (let [var (as-var var) + [v] + (let [var (as-var v) all-vars (q/vars {:ns-query {:project? true} :private? true}) deps-map (zipmap all-vars (map fn-deps all-vars))] (map first (filter (fn [[_k v]] (contains? v var)) deps-map)))) + +(comment + ;; this can be used to blow up memory, which will clear the class cache of old references + (defn oom [] + (try (let [memKiller (java.util.ArrayList.)] + (loop [free 10000000] + (.add memKiller (object-array free)) + (.get memKiller 0) + (recur 100000 #_(if (< (Math/abs (.. Runtime (getRuntime) (freeMemory))) Integer/MAX_VALUE) + (Math/abs (.. Runtime (getRuntime) (freeMemory))) + Integer/MAX_VALUE)))) + (catch OutOfMemoryError _ + (println "freed")))) + + (fn-deps #'fn-refs) + (fn-deps #'orchard.xref/fn-deps) + (fn-refs #'orchard.xref/fn->sym) + + ;; returns all classes in this ns, even repl eval'd + (let [f-class-name "orchard.xref" #_(-> orchard.xref/fn-deps .getClass .getName) + classes (into #{} (comp (filter (fn [[k _v]] (clojure.string/includes? k f-class-name))) + (map (fn [[_k v]] (.get ^java.lang.ref.Reference v)))) + class-cache)] + classes) + + (oom) + (def vars (q/vars {:ns-query {:project? true} :private? true})) + + (map fn-deps vars)) diff --git a/test/orchard/xref_test.clj b/test/orchard/xref_test.clj index a7c1c46d..56044ecf 100644 --- a/test/orchard/xref_test.clj +++ b/test/orchard/xref_test.clj @@ -3,22 +3,28 @@ [clojure.test :refer [deftest is testing]] [orchard.xref :as xref])) +(defn- fn-transitive-dep [a b] + (* a b)) + +(defn- fn-dep [a b] + (fn-transitive-dep a b)) + (defn- dummy-fn [_x] - (map #(* % 2) (filter even? (range 1 10)))) + (map #(fn-dep % 2) (filter even? (range 1 10)))) (deftest fn-deps-test (testing "with a fn value" (is (= (xref/fn-deps dummy-fn) #{#'clojure.core/map #'clojure.core/filter - #'clojure.core/even? #'clojure.core/range}))) + #'clojure.core/even? #'clojure.core/range #'orchard.xref-test/fn-dep}))) (testing "with a var" (is (= (xref/fn-deps #'dummy-fn) #{#'clojure.core/map #'clojure.core/filter - #'clojure.core/even? #'clojure.core/range}))) + #'clojure.core/even? #'clojure.core/range #'orchard.xref-test/fn-dep}))) (testing "with a symbol" (is (= (xref/fn-deps 'orchard.xref-test/dummy-fn) #{#'clojure.core/map #'clojure.core/filter - #'clojure.core/even? #'clojure.core/range})))) + #'clojure.core/even? #'clojure.core/range #'orchard.xref-test/fn-dep})))) ;; The mere presence of this var can reproduce a certain issue. See: ;; https://github.com/clojure-emacs/orchard/issues/135#issuecomment-939731698 @@ -38,4 +44,16 @@ (is (contains? (into #{} (xref/fn-refs #'map)) #'orchard.xref-test/dummy-fn))) (testing "with a symbol" (is (= (xref/fn-refs 'orchard.xref-test/dummy-fn) '())) - (is (contains? (into #{} (xref/fn-refs #'map)) #'orchard.xref-test/dummy-fn)))) + (is (contains? (into #{} (xref/fn-refs #'map)) #'orchard.xref-test/dummy-fn))) + (testing "that usage from inside an anonymous function is found" + (is (contains? (into #{} (xref/fn-refs #'fn-dep)) #'orchard.xref-test/dummy-fn)))) + +(deftest fn-transitive-deps-test + (testing "basics" + (let [expected #{#'orchard.xref-test/fn-deps-test #'orchard.xref-test/fn-dep #'clojure.core/even? + #'clojure.core/filter #'orchard.xref-test/fn-transitive-dep #'clojure.core/map + #'clojure.test/test-var #'clojure.core/range}] + (is (contains? expected #'orchard.xref-test/fn-transitive-dep) + "Specifically includes `#'fn-transitive-dep`, which is a transitive dep of `#'dummy-fn` (via `#'fn-dep`)") + (is (= expected + (xref/fn-transitive-deps dummy-fn))))))