Merge pull request #668 from nasa/harmony-1938

HARMONY-1938: Support running harmony without using an EDL app

ENV_CHANGELOG.md

@@ -2,6 +2,14 @@
 Any changes to the environment variables will be documented in this file in chronological
 order with the most recent changes first.
 
+# 2024-12-04
+### Added
+- USE_EDL_CLIENT_APP - whether to use an EDL client application to enable admin and deployment endpoints and allow OAuth workflows.
+- EDL_TOKEN - required if USE_EDL_CLIENT_APP is set to false. An EDL token to use for all requests to the CMR and to download data in backend services.
+
+### Changed
+- OAUTH_CLIENT_ID, OAUTH_UID, OAUTH_PASSWORD, and OAUTH_REDIRECT_URI are no longer required if USE_EDL_CLIENT_APP is false.
+
 # 2024-12-03
 ### Added
 - Added enviroment defaults for Harmony SMAP L2 Gridding Service

README.md

@@ -22,7 +22,7 @@ This is the quickest way to get started with Harmony (by running Harmony in a co
 built-in Kubernetes cluster (including `kubectl`) which can be enabled in preferences. Minikube is a popular Linux alternative for running Kubernetes locally.
 * [openssl](https://www.openssl.org/) Read [this installation guide](https://github.com/openssl/openssl/blob/master/NOTES-WINDOWS.md) if you're a Windows user and openssl is not installed on your machine already.
 * [envsubst](https://pypi.org/project/envsubst) - Used to substitute environment variable placeholders inside configuration files.
-* [Earthdata Login application in UAT](docs/edl-requirement.md)
+* [Earthdata Login token in UAT](https://uat.urs.earthdata.nasa.gov) - You will need to create an account and then use the "Generate Token" link from your profile page to obtain a token.
 
 2. Download this repository (or download the zip file from GitHub)
 ```bash

bin/bootstrap-harmony

@@ -23,6 +23,14 @@ else
     source .env
 fi
 
+# Validate environment
+if [ "${USE_EDL_CLIENT_APP}" == "false" ]; then
+  if [ -z "${EDL_TOKEN}" ]; then
+    echo "Error: EDL_TOKEN must be set when USE_EDL_CLIENT_APP is 'false'."
+    exit 1
+  fi
+fi
+
 # Used to decide whether or not to run harmony in k8s
 export LOCAL_DEV
 export KUBE_CONTEXT

bin/create-dotenv

@@ -20,29 +20,43 @@ else
 
 EOF
     cat <<-PROMPT_EDL
-		Running Harmony requires creating an Earthdata Login (EDL) application. If you need instructions for this or do not understand what this means, please contact the Harmony team for help. Once you have your EDL application created, please enter the credentials as prompted.
+    Harmony can optionally use a user created Earthdata Login (EDL) client application to enable access
+    to administrative routes such as deployment endpoints as well as directly accessing logs from the
+    workflow-ui. These features are not generally useful for locally testing services, and so creation
+    of an EDL client is not necessary. To simplify getting started you can instead provide an EDL token
+    that will be used.
 
 PROMPT_EDL
     if [ "$EXEC_CONTEXT" != "workflow" ]; then
-        echo "# See the \"OAuth 2 (Earthdata Login)\" section in the env-defaults file" >> .env
-        echo "# Contact a harmony developer if unsure what values to use" >> .env
-        # prompt for the users EDL app credentials
-        read -p "EDL Application Client ID: " OAUTH_CLIENT_ID
-        read -s -p "EDL Application UID: " OAUTH_UID
-        echo ""
-        read -s -p "EDL Application Password: " OAUTH_PASSWORD
-        echo ""
+        read -p "To provide an EDL client application respond 'y', otherwise to supply just an EDL token hit <Enter>: " USE_EDL_CLIENT_APP
+    else
+        USE_EDL_CLIENT_APP="y"
+    fi
+    if [ "${USE_EDL_CLIENT_APP}" != "y" ]; then
+      read -s -p "EDL token to use for authentication (obtained using the Generate Token functionality from https://uat.urs.earthdata.nasa.gov/profile): " EDL_TOKEN
+      echo "# Note that EDL tokens expire. If the token is no longer valid obtain a new one from https://uat.urs.earthdata.nasa.gov/profile" >> .env
+      echo "EDL_TOKEN=$EDL_TOKEN" >> .env
+      echo "USE_EDL_CLIENT_APP=false" >> .env
+    else
+      echo "# See the \"OAuth 2 (Earthdata Login)\" section in the env-defaults file" >> .env
+      echo "# Contact a harmony developer if unsure what values to use" >> .env
+      # prompt for the users EDL app credentials
+      read -p "EDL Application Client ID: " OAUTH_CLIENT_ID
+      read -s -p "EDL Application UID: " OAUTH_UID
+      echo ""
+      read -s -p "EDL Application Password: " OAUTH_PASSWORD
+      echo ""
+      cat <<-OAUTH_EOF >> .env
+OAUTH_CLIENT_ID=$OAUTH_CLIENT_ID
+OAUTH_UID=$OAUTH_UID
+OAUTH_PASSWORD=$OAUTH_PASSWORD
+OAUTH_EOF
     fi
-
-    cat <<-EOF >> .env
-		OAUTH_CLIENT_ID=$OAUTH_CLIENT_ID
-		OAUTH_UID=$OAUTH_UID
-		OAUTH_PASSWORD=$OAUTH_PASSWORD
-EOF
 
     if [ "$EXEC_CONTEXT" != "workflow" ]; then
+        echo ""
         # prompt for LOCALLY_DEPLOYED_SERVICES
-        echo "Enter services to deploy (comma separated list):"
+        echo "Enter services to deploy (comma separated list)."
         read -p "Hit <Enter> to use the default services: " LOCALLY_DEPLOYED_SERVICES
 
         # Add locally LOCALLY_DEPLOYED_SERVICES if it is defined (not zero length)

docs/edl-requirement.md → docs/edl-application.md

@@ -1,6 +1,8 @@
-# Earthdata Login Application Requirement
+# Earthdata Login Application (Optional)
 
-To use Earthdata Login with a locally running Harmony, you must first [set up a new application](https://wiki.earthdata.nasa.gov/display/EL/How+To+Register+An+Application) in the Earthdata Login UAT environment using the Earthdata Login UI.  This is a four step process:
+For most Harmony functionality an Earthdata Login (EDL) application is not required. However for developers that need to work on functionality including admin endpoints such as /service-image-tag, /admin/jobs, or /admin/workflow-ui an EDL client application is needed.
+
+To use an EDL client application with a locally running Harmony, you must first [set up a new application](https://wiki.earthdata.nasa.gov/display/EL/How+To+Register+An+Application) in the Earthdata Login **UAT** environment using the Earthdata Login UI.  This is a four step process:
 
 1. Request and receive permission to be an Application Creator
 2. Create a local/dev Harmony Application in the EDL web interface
@@ -9,4 +11,4 @@ To use Earthdata Login with a locally running Harmony, you must first [set up a
 
 Select "OAuth 2" as the application type. Set the redirect URL to http://localhost:3000/oauth2/redirect for local Harmony. Leave `Required User Information` and `Redirect Time for Earthdata Login Splash page` empty. Check the checkbox for `By checking this box, I confirm that my application is compatible with EDL policy`. Leave the other checkbox unchecked. Then create the new application. After the application is created, you can use the "manage" -> "App Groups" tab to add the "EOSDIS Enterprise" group to the application. This "EOSDIS Enterprise" group will allow CMR searches issued by Harmony to be able to use your Earthdata Login tokens.
 
-If you have an .env file ready to go (see bin/create-dotenv), set `OAUTH_CLIENT_ID`, `OAUTH_UID` and `OAUTH_PASSWORD` with the information from your Earthdata Login application. If you are not ready to create a .env file, refer back to these values later on when you are ready to populate it.
+If you have an .env file ready to go (see bin/create-dotenv), set `OAUTH_CLIENT_ID`, `OAUTH_UID` and `OAUTH_PASSWORD` with the information from your Earthdata Login application. If you are not ready to create a .env file, refer back to these values later on when you are ready to populate it. If you previously set up your harmony environment to not use an EDL client be sure to remove `USE_EDL_CLIENT_APP` from your .env file to use the default setting, or set `USE_EDL_CLIENT_APP=true`.

docs/guides/develop.md

@@ -22,7 +22,7 @@ Required:
 * The [AWS CLI](https://aws.amazon.com/cli/) - Used to interact with both localstack and real AWS accounts
 * [SQLite3 commandline](https://sqlite.org/index.html) - Used to create the local development and test databases. Install using your OS package manager, or [download precompiled binaries from SQLite](https://www.sqlite.org/download.html)
 * PostgreSQL (required by the pg-native library) - `brew install postgresql` on OSX
-* [Earthdata Login application in UAT](../edl-requirement.md)
+* [Earthdata Login application in UAT](../edl-application.md)
 * [envsubst](https://pypi.org/project/envsubst) - Used to substitute environment variable placeholders inside configuration files.
 * [openssl](https://www.openssl.org/) Read [this installation guide](https://github.com/openssl/openssl/blob/master/NOTES-WINDOWS.md) if you're a Windows user and openssl is not installed on your machine already.
 

services/harmony/app/frontends/service-image-tags.ts

@@ -556,7 +556,7 @@ export async function updateServiceImageTag(
  * @param res  - The response object
  * @param _next  - The next middleware in the chain
  */
-export async function getServiceImageTagState(
+export async function getServiceDeploymentsState(
   req: HarmonyRequest, res: Response, _next: NextFunction,
 ): Promise<void> {
   if (!hasCookieSecret(req) && ! await validateUserIsInDeployerOrCoreGroup(req, res)) return;
@@ -571,7 +571,7 @@ export async function getServiceImageTagState(
  * @param req - The request object
  * @param res  - The response object
  */
-export async function setServiceImageTagState(
+export async function setServiceDeploymentsState(
   req: HarmonyRequest, res: Response,
 ): Promise<void> {
   const validations = [

services/harmony/app/middleware/cmr-collection-reader.ts

@@ -6,6 +6,7 @@ import HarmonyRequest from '../models/harmony-request';
 import { listToText } from '@harmony/util/string';
 import { EdlUserEulaInfo, verifyUserEula } from '../util/edl-api';
 import RequestContext from '../models/request-context';
+import env from '../util/env';
 
 // CMR Collection IDs separated by delimiters of single "+" or single whitespace
 // (some clients may translate + to space)
@@ -39,26 +40,28 @@ async function loadVariablesForCollection(context: RequestContext, collection: C
  * @throws ServerError, ForbiddenError, NotFoundError
  */
 async function verifyEulaAcceptance(collections: CmrCollection[], req: HarmonyRequest): Promise<void> {
-  const acceptEulaUrls = [];
-  for (const collection of collections) {
-    if (collection.eula_identifiers) {
-      for (const eulaId of collection.eula_identifiers) {
-        const eulaInfo: EdlUserEulaInfo = await verifyUserEula(req.context, req.user, eulaId);
-        if (eulaInfo.statusCode == 404 && eulaInfo.acceptEulaUrl) { // EULA wasn't accepted
-          acceptEulaUrls.push(eulaInfo.acceptEulaUrl);
-        } else if (eulaInfo.statusCode == 404) {
-          req.context.logger.error(`EULA (${eulaId}) verfification failed with statusCode 404. Error: ${eulaInfo.error}`);
-          throw new NotFoundError(`EULA ${eulaId} could not be found.`);
-        } else if (eulaInfo.statusCode !== 200) {
-          req.context.logger.error(`EULA (${eulaId}) verfification failed. Error: ${eulaInfo.error}`);
-          throw new ServerError(`EULA (${eulaId}) verfification failed unexpectedly.`);
+  if (env.useEdlClientApp) {
+    const acceptEulaUrls = [];
+    for (const collection of collections) {
+      if (collection.eula_identifiers) {
+        for (const eulaId of collection.eula_identifiers) {
+          const eulaInfo: EdlUserEulaInfo = await verifyUserEula(req.context, req.user, eulaId);
+          if (eulaInfo.statusCode == 404 && eulaInfo.acceptEulaUrl) { // EULA wasn't accepted
+            acceptEulaUrls.push(eulaInfo.acceptEulaUrl);
+          } else if (eulaInfo.statusCode == 404) {
+            req.context.logger.error(`EULA (${eulaId}) verfification failed with statusCode 404. Error: ${eulaInfo.error}`);
+            throw new NotFoundError(`EULA ${eulaId} could not be found.`);
+          } else if (eulaInfo.statusCode !== 200) {
+            req.context.logger.error(`EULA (${eulaId}) verfification failed. Error: ${eulaInfo.error}`);
+            throw new ServerError(`EULA (${eulaId}) verfification failed unexpectedly.`);
+          }
         }
       }
     }
-  }
-  if (acceptEulaUrls.length > 0) {
-    throw new ForbiddenError('You may access the requested data by resubmitting your request ' +
-      `after accepting the following EULA(s): ${acceptEulaUrls.join(', ')}.`);
+    if (acceptEulaUrls.length > 0) {
+      throw new ForbiddenError('You may access the requested data by resubmitting your request ' +
+        `after accepting the following EULA(s): ${acceptEulaUrls.join(', ')}.`);
+    }
   }
 }
 

services/harmony/app/middleware/earthdata-login-oauth-authorizer.ts

@@ -8,11 +8,13 @@ import { ForbiddenError, RequestValidationError } from '../util/errors';
 import HarmonyRequest from '../models/harmony-request';
 import env from '../util/env';
 
-const vars = ['OAUTH_CLIENT_ID', 'OAUTH_UID', 'OAUTH_PASSWORD', 'OAUTH_REDIRECT_URI', 'OAUTH_HOST', 'COOKIE_SECRET'];
+if (process.env.USE_EDL_CLIENT_APP === 'true') {
+  const vars = ['OAUTH_CLIENT_ID', 'OAUTH_UID', 'OAUTH_PASSWORD', 'OAUTH_REDIRECT_URI', 'OAUTH_HOST'];
 
-const missingVars = vars.filter((v) => !process.env[v]);
-if (missingVars.length > 0) {
-  throw new Error(`Earthdata Login configuration error: You must set ${listToText(missingVars)} in the environment`);
+  const missingVars = vars.filter((v) => !process.env[v]);
+  if (missingVars.length > 0) {
+    throw new Error(`Earthdata Login configuration error: When USE_EDL_CLIENT_APP is true you must set ${listToText(missingVars)} in the environment`);
+  }
 }
 
 export const oauthOptions: ModuleOptions = {

services/harmony/app/middleware/earthdata-login-skipped.ts

@@ -0,0 +1,26 @@
+import HarmonyRequest from '../models/harmony-request';
+import env from '../util/env';
+
+if (process.env.USE_EDL_CLIENT_APP === 'false' && !process.env.EDL_TOKEN) {
+  throw new Error(
+    'Earthdata Login configuration error: You must set EDL_TOKEN in the environment ' +
+    'when USE_EDL_CLIENT_APP is false',
+  );
+}
+
+/**
+ * Builds Express.js middleware for bypassing EDL client authentication. This should only
+ * be used for local testing and will limit harmony functionality to endpoints that do
+ * not require an EDL client to perform checks. EDL token verification will not be
+ * performed directly by the harmony app, but the EDL_TOKEN environment variable will
+ * be passed to CMR and to download sites when trying to retrieve data at which point
+ * those applications will validate the token.
+ *
+ * @returns Express.js middleware for doing EDL token authentication
+ */
+export default async function edlSkipped(req: HarmonyRequest, res, next): Promise<void> {
+  req.user = 'anonymous';
+  req.accessToken = env.edlToken;
+  req.authorized = true;
+  return next();
+}

services/harmony/app/routers/router.ts

@@ -29,7 +29,7 @@ import { setLogLevel } from '../frontends/configuration';
 import getVersions from '../frontends/versions';
 import serviceInvoker from '../backends/service-invoker';
 import HarmonyRequest, { addRequestContextToOperation } from '../models/harmony-request';
-import { getServiceImageTag, getServiceImageTags, updateServiceImageTag, getServiceImageTagState, setServiceImageTagState, getServiceDeployment, getServiceDeployments } from '../frontends/service-image-tags';
+import { getServiceImageTag, getServiceImageTags, updateServiceImageTag, getServiceDeploymentsState, setServiceDeploymentsState, getServiceDeployment, getServiceDeployments } from '../frontends/service-image-tags';
 import cmrCollectionReader = require('../middleware/cmr-collection-reader');
 import cmrUmmCollectionReader = require('../middleware/cmr-umm-collection-reader');
 import env from '../util/env';
@@ -44,14 +44,16 @@ import { getAdminHealth, getHealth } from '../frontends/health';
 import handleLabelParameter from '../middleware/label';
 import { addJobLabels, deleteJobLabels } from '../frontends/labels';
 import handleJobIDParameter from '../middleware/job-id';
+import earthdataLoginSkipped from '../middleware/earthdata-login-skipped';
+
 export interface RouterConfig {
   PORT?: string | number; // The port to run the frontend server on
   BACKEND_PORT?: string | number; // The port to run the backend server on
   CALLBACK_URL_ROOT?: string; // The base URL for callbacks to use
   // True if we should run example services, false otherwise.  Should be false
   // in production.  Defaults to true until we have real HTTP services.
   EXAMPLE_SERVICES?: string;
-  skipEarthdataLogin?: string; // True if we should skip using EDL
+  USE_EDL_CLIENT_APP?: string; // True if we use the EDL client app
   USE_HTTPS?: string; // True if the server should use https
 }
 
@@ -151,10 +153,10 @@ const authorizedRoutes = [
  * Creates and returns an express.Router instance that has the middleware
  * and handlers necessary to respond to frontend service requests
  *
- * @param skipEarthdataLogin - Opt to skip Earthdata Login
+ * @param USE_EDL_CLIENT_APP - Opt to skip Earthdata Login
  * @returns A router which can respond to frontend service requests
  */
-export default function router({ skipEarthdataLogin = 'false' }: RouterConfig): express.Router {
+export default function router({ USE_EDL_CLIENT_APP = 'false' }: RouterConfig): express.Router {
   const result = express.Router();
 
   const secret = process.env.COOKIE_SECRET;
@@ -174,10 +176,11 @@ export default function router({ skipEarthdataLogin = 'false' }: RouterConfig):
   // a bucket.
   result.post(collectionPrefix('(ogc-api-coverages)'), asyncHandler(shapefileUpload()));
 
-  result.use(logged(earthdataLoginTokenAuthorizer(authorizedRoutes)));
-
-  if (`${skipEarthdataLogin}` !== 'true') {
+  if (`${USE_EDL_CLIENT_APP}` !== 'false') {
+    result.use(logged(earthdataLoginTokenAuthorizer(authorizedRoutes)));
     result.use(logged(earthdataLoginOauthAuthorizer(authorizedRoutes)));
+  } else {
+    result.use(logged(earthdataLoginSkipped));
   }
 
   result.use('/core/*', core);
@@ -313,8 +316,8 @@ export default function router({ skipEarthdataLogin = 'false' }: RouterConfig):
   result.put('/service-image-tag/:service', jsonParser, asyncHandler(updateServiceImageTag));
   result.get('/service-deployment', asyncHandler(getServiceDeployments));
   result.get('/service-deployment/:id', asyncHandler(getServiceDeployment));
-  result.get('/service-deployments-state', asyncHandler(getServiceImageTagState));
-  result.put('/service-deployments-state', jsonParser, asyncHandler(setServiceImageTagState));
+  result.get('/service-deployments-state', asyncHandler(getServiceDeploymentsState));
+  result.put('/service-deployments-state', jsonParser, asyncHandler(setServiceDeploymentsState));
 
   result.get('/*', () => { throw new NotFoundError('The requested page was not found.'); });
   result.post('/*', () => { throw new NotFoundError('The requested POST page was not found.'); });

services/harmony/app/util/edl-api.ts

@@ -113,6 +113,11 @@ export interface EdlGroupMembership {
  */
 export async function getEdlGroupInformation(context: RequestContext, username: string)
   : Promise<EdlGroupMembership> {
+
+  if (!env.useEdlClientApp) {
+    return { isAdmin: false, isLogViewer: false, isServiceDeployer: false, hasCorePermissions: false };
+  }
+
   const groups = await getUserGroups(context, username);
   let isAdmin = false;
   if (groups.includes(env.adminGroupId)) {
@@ -142,6 +147,8 @@ export async function getEdlGroupInformation(context: RequestContext, username:
  * @param req - the harmony request
  */
 export async function isAdminUser(req: HarmonyRequest): Promise<boolean> {
+  if (!env.useEdlClientApp) return false;
+
   const isAdmin = req.context.isAdminAccess ||
     (await getEdlGroupInformation(req.context, req.user)).isAdmin;
   return isAdmin;

services/harmony/app/util/env.ts

@@ -16,16 +16,12 @@ class HarmonyServerEnv extends HarmonyEnv {
   @IsNotEmpty()
   adminGroupId: string;
 
-  @IsNotEmpty()
   oauthClientId: string;
 
-  @IsNotEmpty()
   oauthHost: string;
 
-  @IsNotEmpty()
   oauthPassword: string;
 
-  @IsNotEmpty()
   oauthUid: string;
 
   @IsNotEmpty()
@@ -123,6 +119,11 @@ class HarmonyServerEnv extends HarmonyEnv {
 
   @IsBoolean()
   uiLabeling: boolean;
+
+  @IsBoolean()
+  useEdlClientApp: boolean;
+
+  edlToken: string;
 }
 
 const localPath = path.resolve(__dirname, '../../env-defaults');