Merge pull request #472 from nasa/harmony-398

Harmony 398/1491 - Bulk job status change

app/frontends/jobs.ts

@@ -7,7 +7,7 @@ import JobLink from '../models/job-link';
 import { needsStacLink } from '../util/stac';
 import { getRequestRoot } from '../util/url';
 import { getCloudAccessJsonLink, getCloudAccessShLink, getJobStateChangeLinks, getStacCatalogLink, getStatusLink, Link } from '../util/links';
-import { RequestValidationError, NotFoundError } from '../util/errors';
+import { RequestValidationError, NotFoundError, ServerError } from '../util/errors';
 import { getPagingParams, getPagingLinks, setPagingHeaders } from '../util/pagination';
 import HarmonyRequest from '../models/harmony-request';
 import db from '../util/db';
@@ -235,6 +235,38 @@ export async function changeJobState(
   }
 }
 
+/**
+ * Helper function for canceling, pausing, or resuming jobs in a batch
+ *
+ * @param req - The request sent by the client
+ * @param next - The next function in the call chain
+ * @param jobFn - The function to call to change the job state
+ */
+export async function changeJobsState(
+  req: HarmonyRequest,
+  next: NextFunction,
+  jobFn: (jobID: string, logger: Logger, username: string, token: string) => Promise<void>,
+): Promise<void> {
+  let processedCount = 0;
+  try {
+    const { jobIDs } = req.body;
+    let username: string;
+    const isAdmin = await isAdminUser(req);
+    if (!isAdmin) {
+      // undefined username => admin=true
+      username = req.user;
+    }
+    for (const jobID of jobIDs) {
+      validateJobId(jobID);
+      await jobFn(jobID, req.context.logger, username, req.accessToken);
+      processedCount += 1;
+    }
+  } catch (e) {
+    const message = `Could not change all job statuses. Proccessed ${processedCount}.`;
+    next(new ServerError(message));
+  }
+}
+
 /**
  * Express.js handler that cancels a single job `(POST /jobs/{jobID}/cancel)`. A user can cancel their own
  * request. An admin can cancel any user's request.
@@ -299,4 +331,68 @@ export async function pauseJob(
 ): Promise<void> {
   req.context.logger.info(`Pause requested for job ${req.params.jobID} by user ${req.user}`);
   await changeJobState(req, res, next, pauseAndSaveJob);
+}
+
+/**
+ * Express.js handler that cancels jobs.
+ *
+ * @param req - The request sent by the client
+ * @param res - The response to send to the client
+ * @param next - The next function in the call chain
+ * @returns Resolves when the request is complete
+ */
+export async function cancelJobs(
+  req: HarmonyRequest, res: Response, next: NextFunction,
+): Promise<void> {
+  req.context.logger.info(`Cancel requested for jobs ${req.body.jobIDs} by user ${req.user}`);
+  await changeJobsState(req, next, cancelAndSaveJob);
+  res.status(200).json({ status: 'canceled' });
+}
+
+/**
+ * Express.js handler that resumes jobs.
+ *
+ * @param req - The request sent by the client
+ * @param res - The response to send to the client
+ * @param next - The next function in the call chain
+ * @returns Resolves when the request is complete
+ */
+export async function resumeJobs(
+  req: HarmonyRequest, res: Response, next: NextFunction,
+): Promise<void> {
+  req.context.logger.info(`Resume requested for jobs ${req.body.jobIDs} by user ${req.user}`);
+  await changeJobsState(req, next, resumeAndSaveJob);
+  res.status(200).json({ status: 'running' });
+}
+
+/**
+ * Express.js handler that skips the preview of jobs.
+ *
+ * @param req - The request sent by the client
+ * @param res - The response to send to the client
+ * @param next - The next function in the call chain
+ * @returns Resolves when the request is complete
+ */
+export async function skipJobsPreview(
+  req: HarmonyRequest, res: Response, next: NextFunction,
+): Promise<void> {
+  req.context.logger.info(`Skip preview requested for jobs ${req.body.jobIDs} by user ${req.user}`);
+  await changeJobsState(req, next, skipPreviewAndSaveJob);
+  res.status(200).json({ status: 'running' });
+}
+
+/**
+ * Express.js handler that pauses jobs.
+ *
+ * @param req - The request sent by the client
+ * @param res - The response to send to the client
+ * @param next - The next function in the call chain
+ * @returns Resolves when the request is complete
+ */
+export async function pauseJobs(
+  req: HarmonyRequest, res: Response, next: NextFunction,
+): Promise<void> {
+  req.context.logger.info(`Pause requested for jobs ${req.body.jobIDs} by user ${req.user}`);
+  await changeJobsState(req, next, pauseAndSaveJob);
+  res.status(200).json({ status: 'paused' });
 }

app/frontends/workflow-ui.ts

@@ -1,6 +1,6 @@
 import { Response, NextFunction } from 'express';
 import { sanitizeImage, truncateString } from '@harmony/util/string';
-import { getJobIfAllowed } from '../util/job';
+import { getJobIfAllowed, validateJobId } from '../util/job';
 import { Job, JobStatus, JobQuery, TEXT_LIMIT } from '../models/job';
 import { getWorkItemById, queryAll } from '../models/work-item';
 import { ForbiddenError, NotFoundError, RequestValidationError } from '../util/errors';
@@ -131,9 +131,10 @@ function parseQuery( /* eslint-disable @typescript-eslint/no-explicit-any */
  * a row of the jobs table.
  * @param logger - the logger to use
  * @param requestQuery - the query parameters from the request
+ * @param checked - whether the job should be selected
  * @returns an object with rendering functions
  */
-function jobRenderingFunctions(logger: Logger, requestQuery: Record<string, any>): object {
+function jobRenderingFunctions(logger: Logger, requestQuery: Record<string, any>, checked?: boolean): object {
   return {
     jobBadge(): string {
       return statusClass[this.status];
@@ -174,6 +175,12 @@ function jobRenderingFunctions(logger: Logger, requestQuery: Record<string, any>
         return truncateString(this.message, 100);
       }
     },
+    jobSelectBox(): string {
+      if (this.hasTerminalStatus()) {
+        return '';
+      }
+      return `<input class="select-job" type="checkbox" data-id="${this.jobID}" data-status="${this.status}" autocomplete="off" ${checked ? 'checked' : ''}></input>`;
+    },
     sortGranulesLinks(): string {
       // return links that lets the user apply or unapply an asc or desc sort
       const [ asc, desc ] = [ 'asc', 'desc' ].map((sortValue) => {
@@ -199,13 +206,14 @@ function jobRenderingFunctions(logger: Logger, requestQuery: Record<string, any>
 }
 
 /**
- * Transform a TableQuery to a WorkItem db query.
+ * Transform a TableQuery to a Job db query.
  * @param tableQuery - the constraints parsed from the query string of the request
  * @param isAdmin - is the requesting user an admin
  * @param user - the requesting user's username
+ * @param jobIDs - optional list of job IDs to match on
  * @returns JobQuery
  */
-function tableQueryToJobQuery(tableQuery: TableQuery, isAdmin: boolean, user: string): JobQuery {
+function tableQueryToJobQuery(tableQuery: TableQuery, isAdmin: boolean, user: string, jobIDs?: string[]): JobQuery {
   const jobQuery: JobQuery = { where: {}, whereIn: {} };
   if (tableQuery.sortGranules) {
     jobQuery.orderBy = {
@@ -239,6 +247,12 @@ function tableQueryToJobQuery(tableQuery: TableQuery, isAdmin: boolean, user: st
     jobQuery.dates.from = tableQuery.from;
     jobQuery.dates.to = tableQuery.to;
   }
+  if (jobIDs && jobIDs.length > 0) {
+    jobQuery.whereIn.jobID = {
+      values: jobIDs,
+      in: true,
+    };
+  }
   return jobQuery;
 }
 
@@ -273,6 +287,8 @@ export async function getJobs(
     const paginationInfo = { from: (pagination.from + 1).toLocaleString(),
       to: pagination.to.toLocaleString(), total: pagination.total.toLocaleString(),
       currentPage: pagination.currentPage.toLocaleString(), lastPage: pagination.lastPage.toLocaleString() };
+    const selectAllBox = jobs.some((j) => !j.hasTerminalStatus()) ?
+      '<input id="select-jobs" type="checkbox" title="select/deselect all jobs" autocomplete="off">' : '';
     res.render('workflow-ui/jobs/index', {
       version,
       page,
@@ -281,6 +297,7 @@ export async function getJobs(
       currentUser: req.user,
       isAdminRoute,
       jobs,
+      selectAllBox,
       serviceNames: JSON.stringify(serviceNames),
       sortGranules: requestQuery.sortgranules,
       disallowStatusChecked: !tableQuery.allowStatuses ? 'checked' : '',
@@ -559,6 +576,52 @@ export async function getWorkItemTableRow(
   }
 }
 
+/**
+ * Render rows of the jobs table for the workflow UI.
+ *
+ * @param req - The request sent by the client
+ * @param res - The response to send to the client
+ * @param next - The next function in the call chain
+ * @returns The job rows HTML
+ */
+export async function getJobTableRows(
+  req: HarmonyRequest, res: Response, next: NextFunction,
+): Promise<void> {
+  const { jobIDs } = req.body;
+  try {
+    jobIDs.forEach((id: string) => validateJobId(id));
+    const { isAdmin } = await getEdlGroupInformation(
+      req.user, req.context.logger,
+    );
+    const requestQuery = keysToLowerCase(req.query);
+    const { tableQuery } = parseQuery(requestQuery, JobStatus, req.context.isAdminAccess);
+    const jobQuery = tableQueryToJobQuery(tableQuery, isAdmin, req.user, jobIDs);
+    const jobs = (await Job.queryAll(db, jobQuery, false, 0, jobIDs.length)).data;
+    const resJson = {};
+    for (const job of jobs) {
+      const context = {
+        ...job,
+        ...jobRenderingFunctions(req.context.logger, requestQuery, true),
+        isAdminRoute: req.context.isAdminAccess,
+        hasTerminalStatus: job.hasTerminalStatus,
+        message: job.message,
+      };
+      const renderedHtml = await new Promise<string>((resolve, reject) => req.app.render(
+        'workflow-ui/jobs/job-table-row', context, (err, html) => {
+          if (err) {
+            reject('Could not get job rows');
+          }
+          resolve(html);
+        }));
+      resJson[job.jobID] = renderedHtml;
+    }
+    res.send(resJson);
+  } catch (e) {
+    req.context.logger.error(e);
+    next(e);
+  }
+}
+
 /**
  * Get the logs for a work item.
  *

app/models/job.ts

@@ -128,6 +128,7 @@ export interface JobQuery {
     status?: { in: boolean, values: string[] };
     service_name?: { in: boolean, values: string[] };
     username?: { in: boolean, values: string[] };
+    jobID?: { in: boolean, values: string[] };
   }
   orderBy?: {
     field: string;

app/routers/router.ts

@@ -1,5 +1,5 @@
 import process from 'process';
-import express, { RequestHandler } from 'express';
+import express, { json, RequestHandler } from 'express';
 import asyncHandler from 'express-async-handler';
 import cookieParser from 'cookie-parser';
 import swaggerUi from 'swagger-ui-express';
@@ -12,8 +12,8 @@ import earthdataLoginTokenAuthorizer from '../middleware/earthdata-login-token-a
 import earthdataLoginOauthAuthorizer from '../middleware/earthdata-login-oauth-authorizer';
 import admin from '../middleware/admin';
 import wmsFrontend from '../frontends/wms';
-import { getJobsListing, getJobStatus, cancelJob, resumeJob, pauseJob, skipJobPreview } from '../frontends/jobs';
-import { getJobs, getJob, getWorkItemsTable, getJobLinks, getWorkItemLogs, retry, getWorkItemTableRow, redirectWithoutTrailingSlash } from '../frontends/workflow-ui';
+import { getJobsListing, getJobStatus, cancelJob, resumeJob, pauseJob, skipJobPreview, skipJobsPreview, cancelJobs, resumeJobs, pauseJobs } from '../frontends/jobs';
+import { getJobs, getJob, getWorkItemsTable, getJobLinks, getWorkItemLogs, retry, getWorkItemTableRow, redirectWithoutTrailingSlash, getJobTableRows } from '../frontends/workflow-ui';
 import { getStacCatalog, getStacItem } from '../frontends/stac';
 import { getServiceResult } from '../frontends/service-results';
 import cmrGranuleLocator from '../middleware/cmr-granule-locator';
@@ -237,6 +237,12 @@ export default function router({ skipEarthdataLogin = 'false' }: RouterConfig):
   result.get('/admin/jobs', asyncHandler(getJobsListing));
   result.get('/admin/jobs/:jobID', asyncHandler(getJobStatus));
 
+  const jsonParser = json();
+  result.post('/jobs/cancel', jsonParser, asyncHandler(cancelJobs));
+  result.post('/jobs/resume', jsonParser, asyncHandler(resumeJobs));
+  result.post('/jobs/skip-preview', jsonParser, asyncHandler(skipJobsPreview));
+  result.post('/jobs/pause', jsonParser, asyncHandler(pauseJobs));
+
   result.get('/admin/request-metrics', asyncHandler(getRequestMetrics));
 
   result.get('/workflow-ui', asyncHandler(getJobs));
@@ -245,11 +251,14 @@ export default function router({ skipEarthdataLogin = 'false' }: RouterConfig):
   result.get('/workflow-ui/:jobID/work-items/:id', asyncHandler(getWorkItemTableRow));
   result.get('/workflow-ui/:jobID/links', asyncHandler(getJobLinks));
   result.post('/workflow-ui/:jobID/:id/retry', asyncHandler(retry));
+  result.post('/workflow-ui/jobs', jsonParser, asyncHandler(getJobTableRows));
+
   result.get('/admin/workflow-ui', asyncHandler(getJobs));
   result.get('/admin/workflow-ui/:jobID', asyncHandler(getJob));
   result.get('/admin/workflow-ui/:jobID/work-items', asyncHandler(getWorkItemsTable));
   result.get('/admin/workflow-ui/:jobID/work-items/:id', asyncHandler(getWorkItemTableRow));
   result.get('/admin/workflow-ui/:jobID/links', asyncHandler(getJobLinks));
+  result.post('/admin/workflow-ui/jobs', jsonParser, asyncHandler(getJobTableRows));
 
   result.get('/logs/:jobID/:id', asyncHandler(getWorkItemLogs));
 

app/views/capabilities/index.mustache.html

@@ -6,7 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <link rel="stylesheet" href="../../../css/eui.min.css">
     <link rel="stylesheet" href="../../../css/default.css">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.9.1/font/bootstrap-icons.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
     <link href="https://cdn.jsdelivr.net/npm/@yaireo/tagify@4.16.4/dist/tagify.css" rel="stylesheet" type="text/css" />
     <link rel="stylesheet" href="../../../css/workflow-ui/default.css">
     <script src="https://cdn.jsdelivr.net/npm/@yaireo/tagify@4.16.4"></script>

app/views/workflow-ui/job/index.mustache.html

@@ -6,7 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <link rel="stylesheet" href="../../../css/eui.min.css">
     <link rel="stylesheet" href="../../../css/default.css">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.9.1/font/bootstrap-icons.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
     <link href="https://cdn.jsdelivr.net/npm/@yaireo/tagify@4.16.4/dist/tagify.css" rel="stylesheet" type="text/css" />
     <link rel="stylesheet" href="../../../css/workflow-ui/default.css">
     <script src="https://cdn.jsdelivr.net/npm/@yaireo/tagify@4.16.4"></script>

app/views/workflow-ui/jobs/index.mustache.html

@@ -6,7 +6,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <link rel="stylesheet" href="../../../css/eui.min.css">
     <link rel="stylesheet" href="../../../css/default.css">
-    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.9.1/font/bootstrap-icons.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.0/font/bootstrap-icons.css">
     <link href="https://cdn.jsdelivr.net/npm/@yaireo/tagify@4.16.4/dist/tagify.css" rel="stylesheet" type="text/css" />
     <link rel="stylesheet" href="../../../css/workflow-ui/default.css">
     <script src="https://cdn.jsdelivr.net/npm/@yaireo/tagify@4.16.4"></script>
@@ -38,9 +38,14 @@
     <nav class="ml-0 pl-0"
         style="--bs-breadcrumb-divider: url(&#34;data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='8' height='8'%3E%3Cpath d='M2.5 0L1 1.5 3.5 4 1 6.5 2.5 8l4-4-4-4z' fill='currentColor'/%3E%3C/svg%3E&#34;);"
         aria-label="breadcrumb">
-        <ol class="breadcrumb">
-            <li class="breadcrumb-item active" aria-current="page">Jobs</li>
-        </ol>
+        <div class="breadcrumb  d-flex flex-row justify-content-between">
+            <ol class="breadcrumb p-0 m-0">
+                <li class="breadcrumb-item active" aria-current="page">Jobs</li>
+            </ol>
+            <div id="job-state-links-container">
+                <!-- job state change links will go here -->
+            </div>
+        </div>
     </nav>
     <div class="container-fluid">
         <div class="row pb-4">

app/views/workflow-ui/jobs/job-table-row.mustache.html

@@ -0,0 +1,18 @@
+<tr id="job-{{jobID}}" class='job-table-row'>
+    <td>{{{jobSelectBox}}}</td>
+    <th scope="row"><a href="workflow-ui/{{jobID}}{{dateQuery}}">{{jobID}}</a></th>
+    <td>{{service_name}}</td>
+    {{#isAdminRoute}}
+    <td>{{username}}</td>
+    {{/isAdminRoute}}
+    <td><span class="badge bg-{{jobBadge}}">{{status}}</span></td>
+    <td id="message-td" title="{{message}}">{{jobMessage}}</td>
+    <td>{{numInputGranules}}</td>
+    <td>{{progress}}%</td>
+    <td class="date-td" data-time="{{jobCreatedAt}}"></td>
+    <td class="date-td" data-time="{{jobUpdatedAt}}"></td>
+</tr>
+<th id="copy-{{jobID}}" class="job-url-th" colspan="9">
+    <i class="bi bi-copy text-primary copy-request" data-text="{{request}}" data-truncated="{{jobRequestIsTruncated}}"></i>
+    <span title="{{jobRequest}}" class="text-muted job-url-text">{{jobRequestDisplay}}</span>
+</th>