From 69af33d799c1b5701bb5aecf6e4c32cbe7803236 Mon Sep 17 00:00:00 2001
From: Carsten Behring <carsten.behring@gmail.com>
Date: Sat, 4 May 2024 11:32:37 +0200
Subject: [PATCH] added dynamic var to disable validation

---
 src/scicloj/metamorph/ml/tidy_models.clj | 54 +++++++++++++++---------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/src/scicloj/metamorph/ml/tidy_models.clj b/src/scicloj/metamorph/ml/tidy_models.clj
index 2616090..e3a9f08 100644
--- a/src/scicloj/metamorph/ml/tidy_models.clj
+++ b/src/scicloj/metamorph/ml/tidy_models.clj
@@ -4,41 +4,55 @@
    [clojure.set :as set]
    [tech.v3.dataset :as ds]))
 
-
-(def allowed-glance-columns
+(def ^:dynamic
+ ^{:doc "Controls if the result columns of the tidy fns of a model
+(glance-fn, tidy-fn, augment-fn is validated against these base
+https://github.com/scicloj/metamorph.ml/tree/main/resources/*.edn
+ and if on violation they fail."
+   :added "1.0"}
+ *validate-tidy-fns* true)
+
+(defn allowed-glance-columns []
   (keys
-   (edn/read-string (slurp "resources/columms-glance.edn"))))
+   (edn/read-string (slurp "https://raw.githubusercontent.com/scicloj/metamorph.ml/linearRegression/resources/columms-glance.edn"))))
 
-(def allowed-tidy-columns
+(defn allowed-tidy-columns []
   (keys
-   (edn/read-string (slurp "resources/columms-tidy.edn"))))
+   (edn/read-string (slurp "https://raw.githubusercontent.com/scicloj/metamorph.ml/linearRegression/resources/columms-tidy.edn"))))
 
-(def allowed-augment-columns
+(defn allowed-augment-columns []
   (keys
-   (edn/read-string (slurp "resources/columms-augment.edn"))))
+   (edn/read-string (slurp "https://raw.githubusercontent.com/scicloj/metamorph.ml/linearRegression/resources/columms-augment.edn"))))
+
+(defn _get-allowed-keys []
+  {:glance (allowed-glance-columns)
+   :tidy (allowed-tidy-columns)
+   :augment (allowed-augment-columns)})
+
+(def get-allowed-keys (memoize _get-allowed-keys))
 
 
 
 (defn- validate-ds [ds allowed-columns fn-name]
-  (let [
-        invalid-keys
-        (set/difference
-         (into #{} (ds/column-names ds))
-         (into #{} allowed-columns))]
-    (if (empty? invalid-keys)
-      ds
-      (throw (Exception. (format "invalid keys from %s: %s" fn-name invalid-keys))))))
+  (if (true? *validate-tidy-fns*)
+    (let [
+          invalid-keys
+          (set/difference
+           (into #{} (ds/column-names ds))
+           (into #{} allowed-columns))]
+      (if (empty? invalid-keys)
+        ds
+        (throw (Exception. (format "invalid keys from %s: %s" fn-name invalid-keys)))))
+    ds))
 
 (defn validate-tidy-ds [ds]
-  (validate-ds ds allowed-tidy-columns "tidy-fn"))
+  (validate-ds ds (:tidy (get-allowed-keys))  "tidy-fn"))
 
 (defn validate-glance-ds [ds]
-  (validate-ds ds allowed-glance-columns "glance-fn"))
-
-
+  (validate-ds ds (:glance (get-allowed-keys)) "glance-fn"))
 
 (defn validate-augment-ds [ds data]
   (validate-ds
    ds
-   (concat allowed-augment-columns (ds/column-names data))
+   (concat (:augment (get-allowed-keys)) (ds/column-names data))
    "augment-fn"))