Merge pull request #270 from psarando/tapis-v3

CORE-1889 Tapis v3 Migration

project.clj

@@ -22,15 +22,15 @@
                  [medley "1.4.0"]
                  [metosin/compojure-api "1.1.14"]
                  [org.cyverse/async-tasks-client "0.0.5"]
-                 [org.cyverse/authy "2.8.0"]
+                 [org.cyverse/authy "3.0.1"]
                  [org.cyverse/clojure-commons "3.0.10"]
                  [org.cyverse/debug-utils "2.9.0"]
                  [org.cyverse/kameleon "3.0.10"]
-                 [org.cyverse/mescal "3.1.12"]
+                 [org.cyverse/mescal "4.0.0"]
                  [org.cyverse/metadata-client "3.1.2"]
                  [org.cyverse/common-cli "2.8.2"]
                  [org.cyverse/common-cfg "2.8.3"]
-                 [org.cyverse/common-swagger-api "3.4.7"]
+                 [org.cyverse/common-swagger-api "3.4.8"]
                  [org.cyverse/cyverse-groups-client "0.1.9"]
                  [org.cyverse/permissions-client "2.8.4"]
                  [org.cyverse/service-logging "2.8.4"]

src/apps/constants.clj

@@ -1,5 +1,5 @@
 (ns apps.constants
-  (:require [mescal.agave-de-v2.constants :as c]))
+  (:require [mescal.tapis-de-v3.constants :as c]))
 
 (def de-system-id "de")
 (def hpc-system-id c/hpc-system-id)

src/apps/persistence/app_groups.clj

@@ -7,10 +7,10 @@
 
 (defn get-app-group-hierarchy
   "Gets the app group hierarchy rooted at the node with the given identifier."
-  [root-id {:keys [agave-enabled app-ids] :or {agave-enabled "false"}}]
+  [root-id {:keys [tapis-enabled app-ids] :or {tapis-enabled "false"}}]
   (if (seq app-ids)
-    (select (sqlfn :app_category_hierarchy root-id (Boolean/parseBoolean agave-enabled) (sql-array "uuid" app-ids)))
-    (select (sqlfn :app_category_hierarchy root-id (Boolean/parseBoolean agave-enabled)))))
+    (select (sqlfn :app_category_hierarchy root-id (Boolean/parseBoolean tapis-enabled) (sql-array "uuid" app-ids)))
+    (select (sqlfn :app_category_hierarchy root-id (Boolean/parseBoolean tapis-enabled)))))
 
 (defn get-visible-workspaces
   "Gets the list of workspaces that are visible to the user with the given workspace

src/apps/persistence/app_listing.clj

@@ -85,10 +85,10 @@
     (where query {:id [not-in omitted-app-ids]})
     query))
 
-(defn- add-agave-pipeline-where-clause
-  [query {agave-enabled? :agave-enabled :or {agave-enaled? "false"}}]
-  (let [agave-enabled? (Boolean/parseBoolean agave-enabled?)]
-    (if-not agave-enabled?
+(defn- add-tapis-pipeline-where-clause
+  [query {tapis-enabled? :tapis-enabled :or {tapis-enabled? "false"}}]
+  (let [tapis-enabled? (Boolean/parseBoolean tapis-enabled?)]
+    (if-not tapis-enabled?
       (where query {:step_count :task_count})
       query)))
 
@@ -151,7 +151,7 @@
       (add-app-id-where-clause query-opts)
       (add-app-type-where-clause query-opts)
       (add-omitted-app-id-where-clause query-opts)
-      (add-agave-pipeline-where-clause query-opts)))
+      (add-tapis-pipeline-where-clause query-opts)))
 
 (defn- get-app-count-base-query
   "Adds a where clause to the get-all-apps-count-base-query, filtering out `deleted` apps."
@@ -275,7 +275,7 @@
       (add-app-id-where-clause query_opts)
       (add-app-type-where-clause query_opts)
       (add-omitted-app-id-where-clause query_opts)
-      (add-agave-pipeline-where-clause query_opts)))
+      (add-tapis-pipeline-where-clause query_opts)))
 
 (defn- get-app-listing-base-query
   "Adds a where clause to the get-all-apps-listing-base-query, filtering out `deleted` apps."
@@ -453,7 +453,7 @@
        (add-app-id-where-clause params)
        (add-app-type-where-clause params)
        (add-omitted-app-id-where-clause params)
-       (add-agave-pipeline-where-clause params)
+       (add-tapis-pipeline-where-clause params)
        (select))))
 
 (defn list-public-apps-by-user

src/apps/persistence/app_search.clj

@@ -35,10 +35,10 @@
              (where {:es.app_version_id :v.id
                      :es.task_id        nil})))
 
-(defn- add-agave-pipeline-where-clause
-  [q {agave-enabled? :agave-enabled :or {agave-enaled? "false"}}]
-  (let [agave-enabled? (Boolean/parseBoolean agave-enabled?)]
-    (if-not agave-enabled?
+(defn- add-tapis-pipeline-where-clause
+  [q {tapis-enabled? :tapis-enabled :or {tapis-enabled? "false"}}]
+  (let [tapis-enabled? (Boolean/parseBoolean tapis-enabled?)]
+    (if-not tapis-enabled?
       (where q (not (exists (external-step-subselect))))
       q)))
 
@@ -88,7 +88,7 @@
     (add-app-id-where-clause q query-opts)
     (add-app-type-where-clause q query-opts)
     (add-omitted-app-id-where-clause q query-opts)
-    (add-agave-pipeline-where-clause q query-opts)
+    (add-tapis-pipeline-where-clause q query-opts)
     (add-search-term-where-clauses q search-term (:pre-matched-app-ids query-opts))
     (add-non-admin-where-clauses q query-opts)))
 

src/apps/persistence/jobs.clj

@@ -24,11 +24,12 @@
 
 (def de-job-type "DE")
 (def agave-job-type "Agave")
+(def tapis-job-type "Tapis")
 (def interactive-job-type "Interactive")
 (def osg-job-type "OSG")
 
 (def de-client-name c/de-system-id)
-(def agave-client-name c/hpc-system-id)
+(def tapis-client-name c/hpc-system-id)
 (def interactive-client-name c/interactive-system-id)
 (def osg-client-name c/osg-system-id)
 (def combined-client-name "combined")

src/apps/persistence/oauth.clj

@@ -7,13 +7,6 @@
   (:import [java.sql Timestamp]
            [java.util UUID]))
 
-(defn- validate-token-type
-  "Verifies that the token type is supported."
-  [token-type]
-  (when-not (= "bearer" token-type)
-    (throw+ {:type  :clojure-commons.exception/illegal-argument
-             :error (str "OAuth 2.0 token type, " token-type ", is not supported.")})))
-
 (defn- user-id-subselect
   "Returns a subselect statement to find a user ID."
   [username]
@@ -73,8 +66,7 @@
 
 (defn store-access-token
   "Stores information about an OAuth access token in the database."
-  [api-name username {:keys [token-type expires-at refresh-token access-token]}]
-  (validate-token-type token-type)
+  [api-name username {:keys [expires-at refresh-token access-token]}]
   (if (has-access-token api-name username)
     (replace-access-token api-name username expires-at refresh-token access-token)
     (insert-access-token api-name username expires-at refresh-token access-token)))

src/apps/routes/callbacks.clj

@@ -2,22 +2,20 @@
   (:use [apps.routes.schemas.callback]
         [common-swagger-api.schema]
         [common-swagger-api.schema.analyses :only [AnalysisIdPathParam]]
-        [common-swagger-api.schema.callbacks :only [AgaveJobStatusUpdateParams AgaveJobStatusUpdate]]
+        [common-swagger-api.schema.callbacks :only [TapisJobStatusUpdate]]
         [ring.util.http-response :only [ok]])
   (:require [apps.service.callbacks :as callbacks]))
 
 (defroutes callbacks
-  (POST "/de-job" []
-    :body [body (describe DeJobStatusUpdate "The updated job status information.")]
-    :summary "Update the status of of a DE analysis."
-    :description "The jex-events service calls this endpoint when the status of a DE analysis
-         changes"
-    (ok (callbacks/update-de-job-status body)))
+           (POST "/de-job" []
+                 :body [body (describe DeJobStatusUpdate "The updated job status information.")]
+                 :summary "Update the status of of a DE analysis."
+                 :description "The jex-events service calls this endpoint when the status of a DE analysis changes"
+                 (ok (callbacks/update-de-job-status body)))
 
-  (POST "/agave-job/:job-id" []
-    :path-params [job-id :- AnalysisIdPathParam]
-    :body [body (describe AgaveJobStatusUpdate "The updated job status information.")]
-    :query [params AgaveJobStatusUpdateParams]
-    :summary "Update the status of an Agave analysis."
-    :description "The DE registers this endpoint as a callback when it submts jobs to Agave."
-    (ok (callbacks/update-agave-job-status job-id (:lastUpdated body) params))))
+           (POST "/tapis-job/:job-id" []
+                 :path-params [job-id :- AnalysisIdPathParam]
+                 :body [body (describe TapisJobStatusUpdate "The updated job status information.")]
+                 :summary "Update the status of an Tapis analysis."
+                 :description "The DE registers this endpoint as a callback when it submts jobs to Tapis."
+                 (ok (callbacks/update-tapis-job-status job-id body))))

src/apps/service/apps/combined.clj

@@ -23,7 +23,7 @@
     jp/combined-client-name)
 
   (getJobTypes [_]
-    (mapcat #(.getJobTypes %) clients))
+    (conj (mapcat #(.getJobTypes %) clients) jp/agave-job-type))
 
   (listSystemIds [_]
     (mapcat #(.listSystemIds %) clients))
@@ -368,4 +368,5 @@
     (.hasAppPermission (util/get-apps-client clients system-id) username system-id app-id required-level))
 
   (supportsJobSharing [_ job-step]
-    (.supportsJobSharing (util/apps-client-for-job-step clients job-step) job-step)))
+    (and (not= (:job_type job-step) jp/agave-job-type)
+         (.supportsJobSharing (util/apps-client-for-job-step clients job-step) job-step))))

src/apps/service/apps/combined/jobs.clj

@@ -85,7 +85,7 @@
   [{app-version-id :app_version_id} {app-step-number :app_step_number}]
   (nth (ap/load-app-steps app-version-id) (dec app-step-number)))
 
-(defn- prepare-agave-job-step-submission
+(defn- prepare-tapis-job-step-submission
   [job-info job-step submission]
   (let [app-step   (get-current-app-step job-info job-step)
         submission (prepare-common-job-step-submission job-info job-step submission)]
@@ -97,7 +97,7 @@
   [job-info job-step submission]
   (if (cu/is-de-job-step? job-step)
     (prepare-de-job-step-submission job-info job-step submission)
-    (prepare-agave-job-step-submission job-info job-step submission)))
+    (prepare-tapis-job-step-submission job-info job-step submission)))
 
 (defn- record-step-submission
   [job-id step-number external-id]

src/apps/service/apps/jobs/util.clj

@@ -2,6 +2,7 @@
   (:use [slingshot.slingshot :only [try+ throw+]])
   (:require [apps.clients.data-info :as data-info]
             [apps.persistence.jobs :as jp]
+            [apps.persistence.app-metadata :as am]
             [apps.util.service :as service]
             [clojure.string :as string]
             [clojure.tools.logging :as log]
@@ -51,8 +52,7 @@
    "FolderInput"       "collection"
    "MultiFileSelector" "many"})
 
-(def input-types
-  (set (keys input-multiplicities)))
+(def input-types am/param-ds-input-types)
 
 (def output-multiplicities
   {"FileOutput"      "single"