From 8c0800f44f39a2cd591d56ca1d3c125eee2ad190 Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 09:16:09 +0100 Subject: [PATCH 01/10] NCI-Agency/anet#3231: Pull up logger, and always pass AnetConfiguration to constructor Move worker creation closer to where it's scheduled. Make some variables final. --- .../java/mil/dds/anet/AnetApplication.java | 37 ++++++++++--------- .../mil/dds/anet/threads/AbstractWorker.java | 7 +++- .../threads/AccountDeactivationWorker.java | 9 +---- .../mil/dds/anet/threads/AnetEmailWorker.java | 10 +---- .../anet/threads/FutureEngagementWorker.java | 11 ++---- .../MaterializedViewRefreshWorker.java | 11 ++---- .../anet/threads/ReportApprovalWorker.java | 10 +---- .../anet/threads/ReportPublicationWorker.java | 10 +---- .../db/FutureEngagementWorkerTest.java | 5 ++- .../db/ReportPublicationWorkerTest.java | 4 +- .../emails/AnetEmailWorkerTest.java | 2 +- .../anet/test/resources/TaskApprovalTest.java | 2 +- 12 files changed, 45 insertions(+), 73 deletions(-) diff --git a/src/main/java/mil/dds/anet/AnetApplication.java b/src/main/java/mil/dds/anet/AnetApplication.java index 1ebf9028be..fd71e0a09d 100644 --- a/src/main/java/mil/dds/anet/AnetApplication.java +++ b/src/main/java/mil/dds/anet/AnetApplication.java @@ -207,31 +207,32 @@ public void run(AnetConfiguration configuration, Environment environment) } else { logger.info("AnetApplication is starting scheduled workers"); // Schedule any tasks that need to run on an ongoing basis. - ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - AnetEmailWorker emailWorker = new AnetEmailWorker(engine.getEmailDao(), configuration); - FutureEngagementWorker futureWorker = new FutureEngagementWorker(engine.getReportDao()); - ReportPublicationWorker reportPublicationWorker = - new ReportPublicationWorker(engine.getReportDao(), configuration); - final ReportApprovalWorker reportApprovalWorker = - new ReportApprovalWorker(engine.getReportDao(), configuration); + final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); // Check for any reports that need to be published every 5 minutes. // And run once in 5 seconds from boot-up. (give the server time to boot up). + final ReportPublicationWorker reportPublicationWorker = + new ReportPublicationWorker(configuration, engine.getReportDao()); scheduler.scheduleAtFixedRate(reportPublicationWorker, 5, 5, TimeUnit.MINUTES); scheduler.schedule(reportPublicationWorker, 5, TimeUnit.SECONDS); // Check for any emails that need to be sent every 5 minutes. // And run once in 10 seconds from boot-up. (give the server time to boot up). + final AnetEmailWorker emailWorker = new AnetEmailWorker(configuration, engine.getEmailDao()); scheduler.scheduleAtFixedRate(emailWorker, 5, 5, TimeUnit.MINUTES); scheduler.schedule(emailWorker, 10, TimeUnit.SECONDS); // Check for any future engagements every 3 hours. // And run once in 15 seconds from boot-up. (give the server time to boot up). + final FutureEngagementWorker futureWorker = + new FutureEngagementWorker(configuration, engine.getReportDao()); scheduler.scheduleAtFixedRate(futureWorker, 0, 3, TimeUnit.HOURS); scheduler.schedule(futureWorker, 15, TimeUnit.SECONDS); // Check for any reports that need to be approved every 5 minutes. // And run once in 20 seconds from boot-up. (give the server time to boot up). + final ReportApprovalWorker reportApprovalWorker = + new ReportApprovalWorker(configuration, engine.getReportDao()); scheduler.scheduleAtFixedRate(reportApprovalWorker, 5, 5, TimeUnit.MINUTES); scheduler.schedule(reportApprovalWorker, 5, TimeUnit.SECONDS); @@ -241,22 +242,22 @@ public void run(AnetConfiguration configuration, Environment environment) // Wait 60 seconds between updates of PostgreSQL materialized views, // starting 30 seconds after boot-up. final MaterializedViewRefreshWorker materializedViewRefreshWorker = - new MaterializedViewRefreshWorker(engine.getAdminDao()); + new MaterializedViewRefreshWorker(configuration, engine.getAdminDao()); scheduler.scheduleWithFixedDelay(materializedViewRefreshWorker, 30, 60, TimeUnit.SECONDS); } } // Create all of the HTTP Resources. - LoggingResource loggingResource = new LoggingResource(); - PersonResource personResource = new PersonResource(engine, configuration); - TaskResource taskResource = new TaskResource(engine, configuration); - LocationResource locationResource = new LocationResource(engine); - OrganizationResource orgResource = new OrganizationResource(engine); - PositionResource positionResource = new PositionResource(engine); - ReportResource reportResource = new ReportResource(engine, configuration); - AdminResource adminResource = new AdminResource(engine, configuration); - HomeResource homeResource = new HomeResource(engine, configuration); - SavedSearchResource savedSearchResource = new SavedSearchResource(engine); + final LoggingResource loggingResource = new LoggingResource(); + final PersonResource personResource = new PersonResource(engine, configuration); + final TaskResource taskResource = new TaskResource(engine, configuration); + final LocationResource locationResource = new LocationResource(engine); + final OrganizationResource orgResource = new OrganizationResource(engine); + final PositionResource positionResource = new PositionResource(engine); + final ReportResource reportResource = new ReportResource(engine, configuration); + final AdminResource adminResource = new AdminResource(engine, configuration); + final HomeResource homeResource = new HomeResource(engine, configuration); + final SavedSearchResource savedSearchResource = new SavedSearchResource(engine); final TagResource tagResource = new TagResource(engine); final AuthorizationGroupResource authorizationGroupResource = new AuthorizationGroupResource(engine); diff --git a/src/main/java/mil/dds/anet/threads/AbstractWorker.java b/src/main/java/mil/dds/anet/threads/AbstractWorker.java index 01ac078dec..6a48b1d5b1 100644 --- a/src/main/java/mil/dds/anet/threads/AbstractWorker.java +++ b/src/main/java/mil/dds/anet/threads/AbstractWorker.java @@ -4,19 +4,22 @@ import java.time.Instant; import mil.dds.anet.AnetObjectEngine; import mil.dds.anet.beans.JobHistory; +import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.JobHistoryDao; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public abstract class AbstractWorker implements Runnable { - private static final Logger logger = + protected static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected final AnetConfiguration config; private final String startMessage; private JobHistoryDao jobHistoryDao; - public AbstractWorker(String startMessage) { + public AbstractWorker(AnetConfiguration config, String startMessage) { + this.config = config; this.startMessage = startMessage; this.jobHistoryDao = AnetObjectEngine.getInstance().getJobHistoryDao(); } diff --git a/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java b/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java index bd024adfb4..167aa584fc 100644 --- a/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java +++ b/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java @@ -1,6 +1,5 @@ package mil.dds.anet.threads; -import java.lang.invoke.MethodHandles; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -19,14 +18,9 @@ import mil.dds.anet.emails.AccountDeactivationWarningEmail; import mil.dds.anet.utils.AnetAuditLogger; import mil.dds.anet.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class AccountDeactivationWorker extends AbstractWorker { - private static final Logger logger = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final PersonDao dao; private final List daysTillEndOfTourWarnings; @@ -36,7 +30,8 @@ public class AccountDeactivationWorker extends AbstractWorker { public AccountDeactivationWorker(AnetConfiguration config, PersonDao dao, int warningIntervalInSecs) { - super("Deactivation Warning Worker waking up to check for Future Account Deactivations"); + super(config, + "Deactivation Warning Worker waking up to check for Future Account Deactivations"); this.dao = dao; @SuppressWarnings("unchecked") diff --git a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java index b443e8ab19..ecdb1d1a7d 100644 --- a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java +++ b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java @@ -10,7 +10,6 @@ import freemarker.template.TemplateException; import java.io.IOException; import java.io.StringWriter; -import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.format.DateTimeFormatter; @@ -42,14 +41,9 @@ import org.simplejavamail.api.email.Email; import org.simplejavamail.email.EmailBuilder; import org.simplejavamail.mailer.MailerBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class AnetEmailWorker extends AbstractWorker { - private static final Logger logger = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static AnetEmailWorker instance; private EmailDao dao; @@ -70,8 +64,8 @@ public class AnetEmailWorker extends AbstractWorker { private final List activeDomainNames; @SuppressWarnings("unchecked") - public AnetEmailWorker(EmailDao dao, AnetConfiguration config) { - super("AnetEmailWorker waking up to send emails!"); + public AnetEmailWorker(AnetConfiguration config, EmailDao dao) { + super(config, "AnetEmailWorker waking up to send emails!"); this.dao = dao; this.mapper = MapperUtils.getDefaultMapper(); this.fromAddr = config.getEmailFromAddr(); diff --git a/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java b/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java index fea5578d50..ce6d077875 100644 --- a/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java +++ b/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java @@ -1,6 +1,5 @@ package mil.dds.anet.threads; -import java.lang.invoke.MethodHandles; import java.time.Instant; import java.util.List; import java.util.Map; @@ -9,20 +8,16 @@ import mil.dds.anet.beans.AnetEmail; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.beans.Report; +import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.ReportDao; import mil.dds.anet.emails.FutureEngagementUpdated; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class FutureEngagementWorker extends AbstractWorker { - private static final Logger logger = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private ReportDao dao; - public FutureEngagementWorker(ReportDao dao) { - super("Future Engagement Worker waking up to check for Future Engagements"); + public FutureEngagementWorker(AnetConfiguration config, ReportDao dao) { + super(config, "Future Engagement Worker waking up to check for Future Engagements"); this.dao = dao; } diff --git a/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java b/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java index fa986d5b5d..4061a275c5 100644 --- a/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java +++ b/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java @@ -1,25 +1,20 @@ package mil.dds.anet.threads; -import java.lang.invoke.MethodHandles; import java.time.Instant; import mil.dds.anet.beans.JobHistory; +import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.AdminDao; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class MaterializedViewRefreshWorker extends AbstractWorker { - private static final Logger logger = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String[] materializedViews = {"mv_fts_authorizationGroups", "mv_fts_locations", "mv_fts_organizations", "mv_fts_people", "mv_fts_positions", "mv_fts_reports", "mv_fts_tags", "mv_fts_tasks"}; private final AdminDao dao; - public MaterializedViewRefreshWorker(AdminDao dao) { - super("Refreshing materialized views"); + public MaterializedViewRefreshWorker(AnetConfiguration config, AdminDao dao) { + super(config, "Refreshing materialized views"); this.dao = dao; } diff --git a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java index d763da78d1..fd07a2d55d 100644 --- a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java @@ -1,6 +1,5 @@ package mil.dds.anet.threads; -import java.lang.invoke.MethodHandles; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Collections; @@ -17,19 +16,14 @@ import mil.dds.anet.database.ReportDao; import mil.dds.anet.utils.AnetAuditLogger; import mil.dds.anet.utils.DaoUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ReportApprovalWorker extends AbstractWorker { - private static final Logger logger = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final ReportDao dao; private final Integer nbOfHoursApprovalTimeout; - public ReportApprovalWorker(ReportDao dao, AnetConfiguration config) { - super("Report Approval Worker waking up to check for reports to be approved"); + public ReportApprovalWorker(AnetConfiguration config, ReportDao dao) { + super(config, "Report Approval Worker waking up to check for reports to be approved"); this.dao = dao; this.nbOfHoursApprovalTimeout = (Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursApprovalTimeout"); diff --git a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java index 1b523d6b05..634614f78d 100644 --- a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java @@ -1,6 +1,5 @@ package mil.dds.anet.threads; -import java.lang.invoke.MethodHandles; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Collections; @@ -15,19 +14,14 @@ import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.ReportDao; import mil.dds.anet.utils.AnetAuditLogger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ReportPublicationWorker extends AbstractWorker { - private static final Logger logger = - LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final ReportDao dao; private final Integer nbOfHoursQuarantineApproved; - public ReportPublicationWorker(ReportDao dao, AnetConfiguration config) { - super("Report Publication Worker waking up to check for reports to be published"); + public ReportPublicationWorker(AnetConfiguration config, ReportDao dao) { + super(config, "Report Publication Worker waking up to check for reports to be published"); this.dao = dao; this.nbOfHoursQuarantineApproved = (Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursQuarantineApproved"); diff --git a/src/test/java/mil/dds/anet/test/integration/db/FutureEngagementWorkerTest.java b/src/test/java/mil/dds/anet/test/integration/db/FutureEngagementWorkerTest.java index 609ccf6014..b6bd673709 100644 --- a/src/test/java/mil/dds/anet/test/integration/db/FutureEngagementWorkerTest.java +++ b/src/test/java/mil/dds/anet/test/integration/db/FutureEngagementWorkerTest.java @@ -67,8 +67,9 @@ public static void setUpClass() throws Exception { "@" + ((List) app.getConfiguration().getDictionaryEntry("domainNames")).get(0); final AnetObjectEngine engine = AnetObjectEngine.getInstance(); - emailWorker = new AnetEmailWorker(engine.getEmailDao(), app.getConfiguration()); - futureEngagementWorker = new FutureEngagementWorker(engine.getReportDao()); + emailWorker = new AnetEmailWorker(app.getConfiguration(), engine.getEmailDao()); + futureEngagementWorker = + new FutureEngagementWorker(app.getConfiguration(), engine.getReportDao()); emailServer = new FakeSmtpServer(app.getConfiguration().getSmtp()); // Flush all reports from previous tests diff --git a/src/test/java/mil/dds/anet/test/integration/db/ReportPublicationWorkerTest.java b/src/test/java/mil/dds/anet/test/integration/db/ReportPublicationWorkerTest.java index d0c5f39973..f497e2e155 100644 --- a/src/test/java/mil/dds/anet/test/integration/db/ReportPublicationWorkerTest.java +++ b/src/test/java/mil/dds/anet/test/integration/db/ReportPublicationWorkerTest.java @@ -70,8 +70,8 @@ public static void setUpClass() throws Exception { "@" + ((List) configuration.getDictionaryEntry("domainNames")).get(0); final AnetObjectEngine engine = AnetObjectEngine.getInstance(); - emailWorker = new AnetEmailWorker(engine.getEmailDao(), configuration); - reportPublicationWorker = new ReportPublicationWorker(engine.getReportDao(), configuration); + emailWorker = new AnetEmailWorker(configuration, engine.getEmailDao()); + reportPublicationWorker = new ReportPublicationWorker(configuration, engine.getReportDao()); emailServer = new FakeSmtpServer(configuration.getSmtp()); // Flush all reports from previous tests diff --git a/src/test/java/mil/dds/anet/test/integration/emails/AnetEmailWorkerTest.java b/src/test/java/mil/dds/anet/test/integration/emails/AnetEmailWorkerTest.java index de2bf41815..36188d1091 100644 --- a/src/test/java/mil/dds/anet/test/integration/emails/AnetEmailWorkerTest.java +++ b/src/test/java/mil/dds/anet/test/integration/emails/AnetEmailWorkerTest.java @@ -41,7 +41,7 @@ public static void setUp() throws Exception { final DropwizardAppExtension app = TestApp.app; emailDao = mock(EmailDao.class, Mockito.RETURNS_DEEP_STUBS); - emailWorker = new AnetEmailWorker(emailDao, app.getConfiguration()); + emailWorker = new AnetEmailWorker(app.getConfiguration(), emailDao); whitelistedEmail = "@" + ((List) app.getConfiguration().getDictionaryEntry("domainNames")).get(0); diff --git a/src/test/java/mil/dds/anet/test/resources/TaskApprovalTest.java b/src/test/java/mil/dds/anet/test/resources/TaskApprovalTest.java index d46d567d98..a269b606a9 100644 --- a/src/test/java/mil/dds/anet/test/resources/TaskApprovalTest.java +++ b/src/test/java/mil/dds/anet/test/resources/TaskApprovalTest.java @@ -93,7 +93,7 @@ public static void setUpEmailServer() throws Exception { final List activeDomainNames = (List) dict.get("activeDomainNames"); activeDomainNames.add("example.com"); config.setDictionary(dict); - emailWorker = new AnetEmailWorker(AnetObjectEngine.getInstance().getEmailDao(), config); + emailWorker = new AnetEmailWorker(config, AnetObjectEngine.getInstance().getEmailDao()); emailServer = new FakeSmtpServer(config.getSmtp()); } From 05a3eb5e7fb5f909ad2caf8b4ff1a2e2d9ede73e Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 09:21:08 +0100 Subject: [PATCH 02/10] NCI-Agency/anet#3231: Prevent NPE when getting a non-existing key path --- src/main/java/mil/dds/anet/config/AnetConfiguration.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/mil/dds/anet/config/AnetConfiguration.java b/src/main/java/mil/dds/anet/config/AnetConfiguration.java index 4465dda9a3..8a4bc065bd 100644 --- a/src/main/java/mil/dds/anet/config/AnetConfiguration.java +++ b/src/main/java/mil/dds/anet/config/AnetConfiguration.java @@ -256,6 +256,9 @@ public Object getDictionaryEntry(String keyPath) { Object elem = dictionary; for (final String key : keyPath.split("\\.")) { elem = ((Map) elem).get(key); + if (elem == null) { + break; + } } return elem; } From e05fc6920c9548049ab3a3a6063c99f3a7111e92 Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 09:21:59 +0100 Subject: [PATCH 03/10] NCI-Agency/anet#3231: Enhance method to get either all active top- or sub-level tasks --- src/main/java/mil/dds/anet/database/TaskDao.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/mil/dds/anet/database/TaskDao.java b/src/main/java/mil/dds/anet/database/TaskDao.java index 0e365df986..188c1df79d 100644 --- a/src/main/java/mil/dds/anet/database/TaskDao.java +++ b/src/main/java/mil/dds/anet/database/TaskDao.java @@ -197,9 +197,12 @@ public CompletableFuture> getTaskedOrganizationsForTask( } @InTransaction - public List getTopLevelTasks() { - return getDbHandle() - .createQuery("/* getTopTasks */ SELECT * FROM tasks WHERE \"customFieldRef1Uuid\" IS NULL") + public List getActiveTasks(boolean topLevel) { + final String sql = String.format( + "/* get%1$sTasks */ SELECT * FROM tasks WHERE status = :active" + + " AND \"customFieldRef1Uuid\" %2$s NULL", + topLevel ? "TopLevel" : "SubLevel", topLevel ? "IS" : "IS NOT"); + return getDbHandle().createQuery(sql).bind("active", DaoUtils.getEnumId(Task.Status.ACTIVE)) .map(new TaskMapper()).list(); } From 246ebe6e869b23108bc07ad4d1cc93a85f7abf4c Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 09:22:51 +0100 Subject: [PATCH 04/10] NCI-Agency/anet#3231: Add method to safely parse JSON --- src/main/java/mil/dds/anet/utils/Utils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/mil/dds/anet/utils/Utils.java b/src/main/java/mil/dds/anet/utils/Utils.java index 3acb2d2f4f..bdf57c8713 100644 --- a/src/main/java/mil/dds/anet/utils/Utils.java +++ b/src/main/java/mil/dds/anet/utils/Utils.java @@ -259,6 +259,11 @@ private static void internalSanitizeJsonForHtml(JsonNode jsonNode) { } } + public static JsonNode parseJsonSafe(String inputJson) throws JsonProcessingException { + final String sanitizedJson = Utils.sanitizeJson(inputJson); + return sanitizedJson == null ? null : mapper.readTree(sanitizedJson); + } + public static String trimStringReturnNull(String input) { if (input == null) { return null; From d7550aa0bf673778944a06e710eb99c45a2f63c8 Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 11:19:27 +0100 Subject: [PATCH 05/10] NCI-Agency/anet#3268: Defer getting entries from the dictionary With the exception of the AccountDeactivationWorker schedule, changes in the dictionary are now properly reflected after a reload. --- .../java/mil/dds/anet/AnetApplication.java | 9 +- .../dds/anet/resources/ReportResource.java | 53 ++++--- .../mil/dds/anet/resources/TaskResource.java | 15 +- .../threads/AccountDeactivationWorker.java | 44 +++--- .../mil/dds/anet/threads/AnetEmailWorker.java | 146 +++++++++--------- .../anet/threads/ReportApprovalWorker.java | 7 +- .../anet/threads/ReportPublicationWorker.java | 6 +- 7 files changed, 146 insertions(+), 134 deletions(-) diff --git a/src/main/java/mil/dds/anet/AnetApplication.java b/src/main/java/mil/dds/anet/AnetApplication.java index fd71e0a09d..3ed7c78f6f 100644 --- a/src/main/java/mil/dds/anet/AnetApplication.java +++ b/src/main/java/mil/dds/anet/AnetApplication.java @@ -278,9 +278,10 @@ public void run(AnetConfiguration configuration, Environment environment) } private void runAccountDeactivationWorker(final AnetConfiguration configuration, - final ScheduledExecutorService scheduler, final AnetObjectEngine engine) - throws IllegalArgumentException { - // Check whether the application is configured to auto-check for account deactivation + final ScheduledExecutorService scheduler, final AnetObjectEngine engine) { + // Check whether the application is configured to auto-check for account deactivation. + // NOTE: if you change this, reloading the dictionary from the admin interface is *not* + // sufficient, you will have to restart ANET for this change to be reflected if (configuration.getDictionaryEntry("automaticallyInactivateUsers") != null) { // Check for any accounts which are scheduled to be deactivated as they reach the end-of-tour // date. @@ -289,7 +290,7 @@ private void runAccountDeactivationWorker(final AnetConfiguration configuration, final AccountDeactivationWorker deactivationWarningWorker = new AccountDeactivationWorker( configuration, engine.getPersonDao(), accountDeactivationWarningInterval); - // Run the email deactivation worker at the set interval. In development run it every minute. + // Run the email deactivation worker at the set interval scheduler.scheduleAtFixedRate(deactivationWarningWorker, accountDeactivationWarningInterval, accountDeactivationWarningInterval, TimeUnit.SECONDS); diff --git a/src/main/java/mil/dds/anet/resources/ReportResource.java b/src/main/java/mil/dds/anet/resources/ReportResource.java index 0d33a70d86..e41a87f5ee 100644 --- a/src/main/java/mil/dds/anet/resources/ReportResource.java +++ b/src/main/java/mil/dds/anet/resources/ReportResource.java @@ -78,28 +78,10 @@ public class ReportResource { private final AnetObjectEngine engine; private final AnetConfiguration config; - private final RollupGraphComparator rollupGraphComparator; - private final DateTimeFormatter dtf; - private final boolean engagementsIncludeTimeAndDuration; - private final DateTimeFormatter edtf; - public ReportResource(AnetObjectEngine engine, AnetConfiguration config) { this.engine = engine; this.dao = engine.getReportDao(); this.config = config; - this.dtf = DateTimeFormatter - .ofPattern((String) this.config.getDictionaryEntry("dateFormats.email.date")) - .withZone(DaoUtils.getDefaultZoneId()); - engagementsIncludeTimeAndDuration = Boolean.TRUE - .equals((Boolean) this.config.getDictionaryEntry("engagementsIncludeTimeAndDuration")); - final String edtfPattern = (String) this.config - .getDictionaryEntry(engagementsIncludeTimeAndDuration ? "dateFormats.email.withTime" - : "dateFormats.email.date"); - this.edtf = DateTimeFormatter.ofPattern(edtfPattern).withZone(DaoUtils.getDefaultZoneId()); - @SuppressWarnings("unchecked") - List pinnedOrgNames = (List) this.config.getDictionaryEntry("pinned_ORGs"); - this.rollupGraphComparator = new RollupGraphComparator(pinnedOrgNames); - } @GraphQLQuery(name = "report") @@ -744,7 +726,7 @@ public List getDailyRollupGraph(@GraphQLArgument(name = "startDate" dailyRollupGraph = dao.getDailyRollupGraph(startDate, endDate, orgType, nonReportingOrgs); } - Collections.sort(dailyRollupGraph, rollupGraphComparator); + Collections.sort(dailyRollupGraph, getRollupGraphComparator()); return dailyRollupGraph; } @@ -784,10 +766,7 @@ public String showRollupEmail(@GraphQLArgument(name = "startDate") Long start, action.setAdvisorOrganizationUuid(advisorOrgUuid); action.setPrincipalOrganizationUuid(principalOrgUuid); - @SuppressWarnings("unchecked") - final Map fields = (Map) config.getDictionaryEntry("fields"); - - Map context = new HashMap(); + final Map context = new HashMap(); context.put("context", engine.getContext()); context.put("serverUrl", config.getServerUrl()); context.put(AdminSettingKeys.SECURITY_BANNER_TEXT.name(), @@ -795,10 +774,7 @@ public String showRollupEmail(@GraphQLArgument(name = "startDate") Long start, context.put(AdminSettingKeys.SECURITY_BANNER_COLOR.name(), engine.getAdminSetting(AdminSettingKeys.SECURITY_BANNER_COLOR)); context.put(DailyRollupEmail.SHOW_REPORT_TEXT_FLAG, showReportText); - context.put("dateFormatter", dtf); - context.put("engagementsIncludeTimeAndDuration", engagementsIncludeTimeAndDuration); - context.put("engagementDateFormatter", edtf); - context.put("fields", fields); + addConfigToContext(context); try { Configuration freemarkerConfig = new Configuration(FREEMARKER_VERSION); @@ -982,4 +958,27 @@ private void updateAssessment(Note newAssessment, Note oldAssessment) { noteDao.update(newAssessment); } } + + private void addConfigToContext(Map context) { + context.put("dateFormatter", + DateTimeFormatter.ofPattern((String) config.getDictionaryEntry("dateFormats.email.date")) + .withZone(DaoUtils.getDefaultZoneId())); + context.put("engagementsIncludeTimeAndDuration", Boolean.TRUE + .equals((Boolean) config.getDictionaryEntry("engagementsIncludeTimeAndDuration"))); + final String edtfPattern = (String) config.getDictionaryEntry(Boolean.TRUE + .equals((Boolean) config.getDictionaryEntry("engagementsIncludeTimeAndDuration")) + ? "dateFormats.email.withTime" + : "dateFormats.email.date"); + context.put("engagementDateFormatter", + DateTimeFormatter.ofPattern(edtfPattern).withZone(DaoUtils.getDefaultZoneId())); + @SuppressWarnings("unchecked") + final Map fields = (Map) config.getDictionaryEntry("fields"); + context.put("fields", fields); + } + + private RollupGraphComparator getRollupGraphComparator() { + @SuppressWarnings("unchecked") + final List pinnedOrgNames = (List) config.getDictionaryEntry("pinned_ORGs"); + return new RollupGraphComparator(pinnedOrgNames); + } } diff --git a/src/main/java/mil/dds/anet/resources/TaskResource.java b/src/main/java/mil/dds/anet/resources/TaskResource.java index e3c38f9343..c070f6f98c 100644 --- a/src/main/java/mil/dds/anet/resources/TaskResource.java +++ b/src/main/java/mil/dds/anet/resources/TaskResource.java @@ -31,13 +31,12 @@ public class TaskResource { private final AnetObjectEngine engine; private final TaskDao dao; - private final String duplicateTaskShortName; + private final AnetConfiguration config; public TaskResource(AnetObjectEngine engine, AnetConfiguration config) { this.engine = engine; this.dao = engine.getTaskDao(); - final String taskShortLabel = (String) config.getDictionaryEntry("fields.task.shortLabel"); - duplicateTaskShortName = String.format("Duplicate %s number", taskShortLabel); + this.config = config; } @GraphQLQuery(name = "task") @@ -59,7 +58,7 @@ public Task createTask(@GraphQLRootContext Map context, try { created = dao.insert(t); } catch (UnableToExecuteStatementException e) { - throw ResponseUtils.handleSqlException(e, duplicateTaskShortName); + throw createDuplicateException(e); } if (t.getPlanningApprovalSteps() != null) { // Create the planning approval steps @@ -156,7 +155,7 @@ public Integer updateTask(@GraphQLRootContext Map context, // GraphQL mutations *have* to return something, so we return the number of updated rows return numRows; } catch (UnableToExecuteStatementException e) { - throw ResponseUtils.handleSqlException(e, duplicateTaskShortName); + throw createDuplicateException(e); } } @@ -166,4 +165,10 @@ public AnetBeanList search(@GraphQLRootContext Map context query.setUser(DaoUtils.getUserFromContext(context)); return dao.search(query); } + + private WebApplicationException createDuplicateException(UnableToExecuteStatementException e) { + final String taskShortLabel = (String) config.getDictionaryEntry("fields.task.shortLabel"); + return ResponseUtils.handleSqlException(e, + String.format("Duplicate %s number", taskShortLabel)); + } } diff --git a/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java b/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java index 167aa584fc..d841a804b0 100644 --- a/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java +++ b/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java @@ -22,10 +22,6 @@ public class AccountDeactivationWorker extends AbstractWorker { private final PersonDao dao; - - private final List daysTillEndOfTourWarnings; - private final List ignoredDomainNames; - private final int warningIntervalInSecs; public AccountDeactivationWorker(AnetConfiguration config, PersonDao dao, @@ -33,25 +29,14 @@ public AccountDeactivationWorker(AnetConfiguration config, PersonDao dao, super(config, "Deactivation Warning Worker waking up to check for Future Account Deactivations"); this.dao = dao; - - @SuppressWarnings("unchecked") - final List daysTillWarning = (List) config - .getDictionaryEntry("automaticallyInactivateUsers.emailRemindersDaysPrior"); - this.daysTillEndOfTourWarnings = - daysTillWarning.stream().filter(i -> i > 0).collect(Collectors.toList()); - - @SuppressWarnings("unchecked") - List domainNamesToIgnore = - (List) config.getDictionaryEntry("automaticallyInactivateUsers.ignoredDomainNames"); - this.ignoredDomainNames = domainNamesToIgnore == null ? domainNamesToIgnore - : domainNamesToIgnore.stream().map(x -> x.trim()).collect(Collectors.toList()); - this.warningIntervalInSecs = warningIntervalInSecs; } @Override protected void runInternal(Instant now, JobHistory jobHistory) { // Make sure the mechanism will be triggered, so account deactivation checking can take place + final List ignoredDomainNames = getDomainNamesToIgnore(); + final List daysTillEndOfTourWarnings = getDaysTillEndOfTourWarnings(); final List warningDays = (daysTillEndOfTourWarnings == null || daysTillEndOfTourWarnings.isEmpty()) ? new ArrayList<>(0) @@ -79,15 +64,32 @@ protected void runInternal(Instant now, JobHistory jobHistory) { final Integer warning = warningDays.get(i); final Integer nextWarning = i == warningDays.size() - 1 ? null : warningDays.get(i + 1); checkDeactivationStatus(p, warning, nextWarning, now, - jobHistory == null ? null : jobHistory.getLastRun()); + jobHistory == null ? null : jobHistory.getLastRun(), ignoredDomainNames, + warningIntervalInSecs); } }); } + private List getDaysTillEndOfTourWarnings() { + @SuppressWarnings("unchecked") + final List daysTillWarning = (List) config + .getDictionaryEntry("automaticallyInactivateUsers.emailRemindersDaysPrior"); + return daysTillWarning.stream().filter(i -> i > 0).collect(Collectors.toList()); + } + + private List getDomainNamesToIgnore() { + @SuppressWarnings("unchecked") + final List domainNamesToIgnore = + (List) config.getDictionaryEntry("automaticallyInactivateUsers.ignoredDomainNames"); + return domainNamesToIgnore == null ? null + : domainNamesToIgnore.stream().map(x -> x.trim()).collect(Collectors.toList()); + } + private void checkDeactivationStatus(final Person person, final Integer daysBeforeWarning, - final Integer nextWarning, final Instant now, final Instant lastRun) { + final Integer nextWarning, final Instant now, final Instant lastRun, + final List ignoredDomainNames, final Integer warningIntervalInSecs) { if (person.getStatus() == Person.Status.INACTIVE - || Utils.isDomainUserNameIgnored(person.getDomainUsername(), this.ignoredDomainNames)) { + || Utils.isDomainUserNameIgnored(person.getDomainUsername(), ignoredDomainNames)) { // Skip inactive ANET users or users from ignored domains return; } @@ -100,7 +102,7 @@ private void checkDeactivationStatus(final Person person, final Integer daysBefo final Instant warningDate = now.plus(daysBeforeWarning, ChronoUnit.DAYS); final Instant prevWarningDate = - (lastRun == null) ? warningDate.minus(this.warningIntervalInSecs, ChronoUnit.SECONDS) + (lastRun == null) ? warningDate.minus(warningIntervalInSecs, ChronoUnit.SECONDS) : lastRun.plus(daysBeforeWarning, ChronoUnit.DAYS); if (person.getEndOfTourDate().isBefore(warningDate) && person.getEndOfTourDate().isAfter(prevWarningDate)) { diff --git a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java index ecdb1d1a7d..970b47b47a 100644 --- a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java +++ b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java @@ -46,69 +46,27 @@ public class AnetEmailWorker extends AbstractWorker { private static AnetEmailWorker instance; - private EmailDao dao; - private ObjectMapper mapper; - private Properties props; - private Authenticator auth; - private String fromAddr; - private String serverUrl; - private final Map fields; - private Configuration freemarkerConfig; - private final String supportEmailAddr; - private final DateTimeFormatter dtf; - private final DateTimeFormatter dttf; - private final boolean engagementsIncludeTimeAndDuration; - private final DateTimeFormatter edtf; - private final Integer nbOfHoursForStaleEmails; - private final boolean disabled; - private final List activeDomainNames; - - @SuppressWarnings("unchecked") + private final EmailDao dao; + private final ObjectMapper mapper; + private final String fromAddr; + private final String serverUrl; + private final SmtpConfiguration smtpConfig; + private final Properties smtpProps; + private final Authenticator smtpAuth; + private final Configuration freemarkerConfig; + public AnetEmailWorker(AnetConfiguration config, EmailDao dao) { super(config, "AnetEmailWorker waking up to send emails!"); this.dao = dao; this.mapper = MapperUtils.getDefaultMapper(); - this.fromAddr = config.getEmailFromAddr(); - this.serverUrl = config.getServerUrl(); - this.supportEmailAddr = (String) config.getDictionaryEntry("SUPPORT_EMAIL_ADDR"); - this.dtf = - DateTimeFormatter.ofPattern((String) config.getDictionaryEntry("dateFormats.email.date")) - .withZone(DaoUtils.getDefaultZoneId()); - this.dttf = DateTimeFormatter - .ofPattern((String) config.getDictionaryEntry("dateFormats.email.withTime")) - .withZone(DaoUtils.getDefaultZoneId()); - engagementsIncludeTimeAndDuration = Boolean.TRUE - .equals((Boolean) config.getDictionaryEntry("engagementsIncludeTimeAndDuration")); - final String edtfPattern = (String) config - .getDictionaryEntry(engagementsIncludeTimeAndDuration ? "dateFormats.email.withTime" - : "dateFormats.email.date"); - this.edtf = DateTimeFormatter.ofPattern(edtfPattern).withZone(DaoUtils.getDefaultZoneId()); - this.fields = (Map) config.getDictionaryEntry("fields"); - this.activeDomainNames = ((List) config.getDictionaryEntry("activeDomainNames")) - .stream().map(String::toLowerCase).collect(Collectors.toList()); setInstance(this); - SmtpConfiguration smtpConfig = config.getSmtp(); - props = new Properties(); - props.put("mail.smtp.ssl.trust", smtpConfig.getSslTrust()); - props.put("mail.smtp.starttls.enable", smtpConfig.getStartTls().toString()); - props.put("mail.smtp.host", smtpConfig.getHostname()); - props.put("mail.smtp.port", smtpConfig.getPort().toString()); - auth = null; - this.nbOfHoursForStaleEmails = smtpConfig.getNbOfHoursForStaleEmails(); - - if (smtpConfig.getUsername() != null && smtpConfig.getUsername().trim().length() > 0) { - props.put("mail.smtp.auth", "true"); - auth = new javax.mail.Authenticator() { - @Override - protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(smtpConfig.getUsername(), smtpConfig.getPassword()); - } - }; - } - - disabled = smtpConfig.isDisabled(); + this.fromAddr = config.getEmailFromAddr(); + this.serverUrl = config.getServerUrl(); + this.smtpConfig = config.getSmtp(); + this.smtpProps = getSmtpProps(smtpConfig); + this.smtpAuth = getSmtpAuth(smtpConfig); freemarkerConfig = new Configuration(FREEMARKER_VERSION); // auto-escape HTML in our .ftlh templates @@ -126,6 +84,11 @@ public static void setInstance(AnetEmailWorker instance) { @Override protected void runInternal(Instant now, JobHistory jobHistory) { + @SuppressWarnings("unchecked") + final List activeDomainNames = + ((List) config.getDictionaryEntry("activeDomainNames")).stream() + .map(String::toLowerCase).collect(Collectors.toList()); + // check the database for any emails we need to send. final List emails = dao.getAll(); @@ -135,13 +98,13 @@ protected void runInternal(Instant now, JobHistory jobHistory) { Map context = null; try { - context = buildContext(email); + context = buildContext(email, smtpConfig); if (context != null) { - logger.info("{} Sending email to {} re: {}", disabled ? "[Disabled] " : "", + logger.info("{} Sending email to {} re: {}", smtpConfig.isDisabled() ? "[Disabled] " : "", email.getToAddresses(), email.getAction().getSubject(context)); - if (!disabled) { - sendEmail(email, context); + if (!smtpConfig.isDisabled()) { + sendEmail(email, context, smtpProps, smtpAuth, activeDomainNames); } } processedEmails.add(email.getId()); @@ -149,8 +112,9 @@ protected void runInternal(Instant now, JobHistory jobHistory) { logger.error("Error sending email", t); // Process stale emails - if (this.nbOfHoursForStaleEmails != null) { - final Instant staleTime = now.minus(nbOfHoursForStaleEmails, ChronoUnit.HOURS); + if (smtpConfig.getNbOfHoursForStaleEmails() != null) { + final Instant staleTime = + now.minus(smtpConfig.getNbOfHoursForStaleEmails(), ChronoUnit.HOURS); if (email.getCreatedAt().isBefore(staleTime)) { String message = "Purging stale email to "; try { @@ -169,7 +133,8 @@ protected void runInternal(Instant now, JobHistory jobHistory) { dao.deletePendingEmails(processedEmails); } - private Map buildContext(final AnetEmail email) { + private Map buildContext(final AnetEmail email, + final SmtpConfiguration smtpConfig) { AnetObjectEngine engine = AnetObjectEngine.getInstance(); Map context = new HashMap(); context.put("context", engine.getContext()); @@ -178,17 +143,32 @@ private Map buildContext(final AnetEmail email) { engine.getAdminSetting(AdminSettingKeys.SECURITY_BANNER_TEXT)); context.put(AdminSettingKeys.SECURITY_BANNER_COLOR.name(), engine.getAdminSetting(AdminSettingKeys.SECURITY_BANNER_COLOR)); - context.put("SUPPORT_EMAIL_ADDR", supportEmailAddr); - context.put("dateFormatter", dtf); - context.put("dateTimeFormatter", dttf); + context.put("SUPPORT_EMAIL_ADDR", config.getDictionaryEntry("SUPPORT_EMAIL_ADDR")); + context.put("dateFormatter", + DateTimeFormatter.ofPattern((String) config.getDictionaryEntry("dateFormats.email.date")) + .withZone(DaoUtils.getDefaultZoneId())); + context.put("dateTimeFormatter", + DateTimeFormatter + .ofPattern((String) config.getDictionaryEntry("dateFormats.email.withTime")) + .withZone(DaoUtils.getDefaultZoneId())); + final boolean engagementsIncludeTimeAndDuration = Boolean.TRUE + .equals((Boolean) config.getDictionaryEntry("engagementsIncludeTimeAndDuration")); context.put("engagementsIncludeTimeAndDuration", engagementsIncludeTimeAndDuration); - context.put("engagementDateFormatter", edtf); + final String edtfPattern = (String) config + .getDictionaryEntry(engagementsIncludeTimeAndDuration ? "dateFormats.email.withTime" + : "dateFormats.email.date"); + context.put("engagementDateFormatter", + DateTimeFormatter.ofPattern(edtfPattern).withZone(DaoUtils.getDefaultZoneId())); + @SuppressWarnings("unchecked") + final Map fields = (Map) config.getDictionaryEntry("fields"); context.put("fields", fields); return email.getAction().buildContext(context); } - private void sendEmail(final AnetEmail email, final Map context) + private void sendEmail(final AnetEmail email, final Map context, + final Properties smtpProps, final Authenticator smtpAuth, + final List activeDomainNames) throws MessagingException, IOException, TemplateException { // Remove any null email addresses email.getToAddresses().removeIf(s -> Objects.equals(s, null)); @@ -206,7 +186,7 @@ private void sendEmail(final AnetEmail email, final Map context) // scan:ignore — false positive, we know which template we are processing temp.process(context, writer); - final Session session = Session.getInstance(props, auth); + final Session session = Session.getInstance(smtpProps, smtpAuth); final Email mail = EmailBuilder.startingBlank().from(new InternetAddress(fromAddr)) .toMultiple(email.getToAddresses()).withSubject(email.getAction().getSubject(context)) .withHTMLText(writer.toString()).buildEmail(); @@ -239,4 +219,32 @@ private synchronized void internal_sendEmailAsync(AnetEmail email) { throw new WebApplicationException(jsonError); } } + + private Properties getSmtpProps(SmtpConfiguration smtpConfig) { + final Properties props = new Properties(); + props.put("mail.smtp.ssl.trust", smtpConfig.getSslTrust()); + props.put("mail.smtp.starttls.enable", smtpConfig.getStartTls().toString()); + props.put("mail.smtp.host", smtpConfig.getHostname()); + props.put("mail.smtp.port", smtpConfig.getPort().toString()); + if (hasUsername(smtpConfig)) { + props.put("mail.smtp.auth", "true"); + } + return props; + } + + private Authenticator getSmtpAuth(SmtpConfiguration smtpConfig) { + if (hasUsername(smtpConfig)) { + return new javax.mail.Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(smtpConfig.getUsername(), smtpConfig.getPassword()); + } + }; + } + return null; + } + + private boolean hasUsername(SmtpConfiguration smtpConfig) { + return smtpConfig.getUsername() != null && smtpConfig.getUsername().trim().length() > 0; + } } diff --git a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java index fd07a2d55d..39405a246b 100644 --- a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java @@ -20,18 +20,17 @@ public class ReportApprovalWorker extends AbstractWorker { private final ReportDao dao; - private final Integer nbOfHoursApprovalTimeout; public ReportApprovalWorker(AnetConfiguration config, ReportDao dao) { super(config, "Report Approval Worker waking up to check for reports to be approved"); this.dao = dao; - this.nbOfHoursApprovalTimeout = - (Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursApprovalTimeout"); } @Override protected void runInternal(Instant now, JobHistory jobHistory) { - final Instant approvalTimeout = now.minus(nbOfHoursApprovalTimeout, ChronoUnit.HOURS); + final Instant approvalTimeout = + now.minus((Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursApprovalTimeout"), + ChronoUnit.HOURS); // Get a list of all PENDING_APPROVAL reports final ReportSearchQuery query = new ReportSearchQuery(); query.setPageSize(0); diff --git a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java index 634614f78d..86f65f2ec4 100644 --- a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java @@ -18,19 +18,17 @@ public class ReportPublicationWorker extends AbstractWorker { private final ReportDao dao; - private final Integer nbOfHoursQuarantineApproved; public ReportPublicationWorker(AnetConfiguration config, ReportDao dao) { super(config, "Report Publication Worker waking up to check for reports to be published"); this.dao = dao; - this.nbOfHoursQuarantineApproved = - (Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursQuarantineApproved"); } @Override protected void runInternal(Instant now, JobHistory jobHistory) { final Instant quarantineApproval = - now.minus(this.nbOfHoursQuarantineApproved, ChronoUnit.HOURS); + now.minus((Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursQuarantineApproved"), + ChronoUnit.HOURS); // Get a list of all APPROVED reports final ReportSearchQuery query = new ReportSearchQuery(); query.setPageSize(0); From dbf4874c207a773c2f1a4feae54898ba3ebea6ac Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 11:27:45 +0100 Subject: [PATCH 06/10] Extract common code to create Freemarker config --- .../dds/anet/resources/ReportResource.java | 21 +++---------------- .../mil/dds/anet/threads/AnetEmailWorker.java | 18 ++-------------- src/main/java/mil/dds/anet/utils/Utils.java | 17 +++++++++++++++ 3 files changed, 22 insertions(+), 34 deletions(-) diff --git a/src/main/java/mil/dds/anet/resources/ReportResource.java b/src/main/java/mil/dds/anet/resources/ReportResource.java index e41a87f5ee..7a34a07d9b 100644 --- a/src/main/java/mil/dds/anet/resources/ReportResource.java +++ b/src/main/java/mil/dds/anet/resources/ReportResource.java @@ -1,9 +1,5 @@ package mil.dds.anet.resources; -import static mil.dds.anet.AnetApplication.FREEMARKER_VERSION; - -import freemarker.template.Configuration; -import freemarker.template.DefaultObjectWrapperBuilder; import freemarker.template.Template; import io.leangen.graphql.annotations.GraphQLArgument; import io.leangen.graphql.annotations.GraphQLEnvironment; @@ -12,7 +8,6 @@ import io.leangen.graphql.annotations.GraphQLRootContext; import java.io.StringWriter; import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; import java.time.DayOfWeek; import java.time.Instant; import java.time.format.DateTimeFormatter; @@ -777,21 +772,11 @@ public String showRollupEmail(@GraphQLArgument(name = "startDate") Long start, addConfigToContext(context); try { - Configuration freemarkerConfig = new Configuration(FREEMARKER_VERSION); - // auto-escape HTML in our .ftlh templates - freemarkerConfig.setRecognizeStandardFileExtensions(true); - freemarkerConfig - .setObjectWrapper(new DefaultObjectWrapperBuilder(FREEMARKER_VERSION).build()); - freemarkerConfig.loadBuiltInEncodingMap(); - freemarkerConfig.setDefaultEncoding(StandardCharsets.UTF_8.name()); - freemarkerConfig.setClassForTemplateLoading(this.getClass(), "/"); - freemarkerConfig.setAPIBuiltinEnabled(true); - - Template temp = freemarkerConfig.getTemplate(action.getTemplateName()); - StringWriter writer = new StringWriter(); + final Template temp = + Utils.getFreemarkerConfig(this.getClass()).getTemplate(action.getTemplateName()); + final StringWriter writer = new StringWriter(); // scan:ignore — false positive, we know which template we are processing temp.process(action.buildContext(context), writer); - return writer.toString(); } catch (Exception e) { throw new WebApplicationException(e); diff --git a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java index 970b47b47a..c37729ba05 100644 --- a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java +++ b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java @@ -1,16 +1,11 @@ package mil.dds.anet.threads; -import static mil.dds.anet.AnetApplication.FREEMARKER_VERSION; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import freemarker.template.Configuration; -import freemarker.template.DefaultObjectWrapperBuilder; import freemarker.template.Template; import freemarker.template.TemplateException; import java.io.IOException; import java.io.StringWriter; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; @@ -53,7 +48,6 @@ public class AnetEmailWorker extends AbstractWorker { private final SmtpConfiguration smtpConfig; private final Properties smtpProps; private final Authenticator smtpAuth; - private final Configuration freemarkerConfig; public AnetEmailWorker(AnetConfiguration config, EmailDao dao) { super(config, "AnetEmailWorker waking up to send emails!"); @@ -67,15 +61,6 @@ public AnetEmailWorker(AnetConfiguration config, EmailDao dao) { this.smtpConfig = config.getSmtp(); this.smtpProps = getSmtpProps(smtpConfig); this.smtpAuth = getSmtpAuth(smtpConfig); - - freemarkerConfig = new Configuration(FREEMARKER_VERSION); - // auto-escape HTML in our .ftlh templates - freemarkerConfig.setRecognizeStandardFileExtensions(true); - freemarkerConfig.setObjectWrapper(new DefaultObjectWrapperBuilder(FREEMARKER_VERSION).build()); - freemarkerConfig.loadBuiltInEncodingMap(); - freemarkerConfig.setDefaultEncoding(StandardCharsets.UTF_8.name()); - freemarkerConfig.setClassForTemplateLoading(this.getClass(), "/"); - freemarkerConfig.setAPIBuiltinEnabled(true); } public static void setInstance(AnetEmailWorker instance) { @@ -182,7 +167,8 @@ private void sendEmail(final AnetEmail email, final Map context, } final StringWriter writer = new StringWriter(); - final Template temp = freemarkerConfig.getTemplate(email.getAction().getTemplateName()); + final Template temp = + Utils.getFreemarkerConfig(this.getClass()).getTemplate(email.getAction().getTemplateName()); // scan:ignore — false positive, we know which template we are processing temp.process(context, writer); diff --git a/src/main/java/mil/dds/anet/utils/Utils.java b/src/main/java/mil/dds/anet/utils/Utils.java index bdf57c8713..0c8cd67fdf 100644 --- a/src/main/java/mil/dds/anet/utils/Utils.java +++ b/src/main/java/mil/dds/anet/utils/Utils.java @@ -1,16 +1,21 @@ package mil.dds.anet.utils; +import static mil.dds.anet.AnetApplication.FREEMARKER_VERSION; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.json.JsonSanitizer; +import freemarker.template.Configuration; +import freemarker.template.DefaultObjectWrapperBuilder; import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -592,4 +597,16 @@ public static void validateApprovalStep(ApprovalStep step) { } } + public static Configuration getFreemarkerConfig(Class clazz) { + final Configuration freemarkerConfig = new Configuration(FREEMARKER_VERSION); + // auto-escape HTML in our .ftlh templates + freemarkerConfig.setRecognizeStandardFileExtensions(true); + freemarkerConfig.setObjectWrapper(new DefaultObjectWrapperBuilder(FREEMARKER_VERSION).build()); + freemarkerConfig.loadBuiltInEncodingMap(); + freemarkerConfig.setDefaultEncoding(StandardCharsets.UTF_8.name()); + freemarkerConfig.setClassForTemplateLoading(clazz, "/"); + freemarkerConfig.setAPIBuiltinEnabled(true); + return freemarkerConfig; + } + } From a105402561a54d7a0e0384dbc750419fba306fcd Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 13:56:30 +0100 Subject: [PATCH 07/10] NCI-Agency/anet#3231: Move metricRegistry to AnetObjectEngine --- src/main/java/mil/dds/anet/AnetObjectEngine.java | 7 ++++++- src/main/java/mil/dds/anet/database/PersonDao.java | 10 +--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/mil/dds/anet/AnetObjectEngine.java b/src/main/java/mil/dds/anet/AnetObjectEngine.java index 2d023c89ca..056ae340fc 100644 --- a/src/main/java/mil/dds/anet/AnetObjectEngine.java +++ b/src/main/java/mil/dds/anet/AnetObjectEngine.java @@ -69,6 +69,7 @@ public class AnetObjectEngine { private final AuthorizationGroupDao authorizationGroupDao; private final NoteDao noteDao; private final JobHistoryDao jobHistoryDao; + private final MetricRegistry metricRegistry; private ThreadLocal> context; ISearcher searcher; @@ -82,7 +83,6 @@ public AnetObjectEngine(String dbUrl, Application application, MetricRegistry this.dbUrl = dbUrl; injector = InjectorLookup.getInjector(application).get(); personDao = injector.getInstance(PersonDao.class); - personDao.setMetricRegistry(metricRegistry); taskDao = injector.getInstance(TaskDao.class); locationDao = injector.getInstance(LocationDao.class); orgDao = injector.getInstance(OrganizationDao.class); @@ -99,6 +99,7 @@ public AnetObjectEngine(String dbUrl, Application application, MetricRegistry authorizationGroupDao = injector.getInstance(AuthorizationGroupDao.class); noteDao = injector.getInstance(NoteDao.class); jobHistoryDao = injector.getInstance(JobHistoryDao.class); + this.metricRegistry = metricRegistry; searcher = Searcher.getSearcher(DaoUtils.getDbType(dbUrl), injector); instance = this; } @@ -179,6 +180,10 @@ public EmailDao getEmailDao() { return emailDao; } + public MetricRegistry getMetricRegistry() { + return metricRegistry; + } + public ISearcher getSearcher() { return searcher; } diff --git a/src/main/java/mil/dds/anet/database/PersonDao.java b/src/main/java/mil/dds/anet/database/PersonDao.java index 17ca3440aa..a00bd29ef3 100644 --- a/src/main/java/mil/dds/anet/database/PersonDao.java +++ b/src/main/java/mil/dds/anet/database/PersonDao.java @@ -62,7 +62,6 @@ public class PersonDao extends AnetBaseDao { private static final int ACTIVITY_LOG_LIMIT = 100; private Cache domainUsersCache; - private MetricRegistry metricRegistry; public PersonDao() { try { @@ -79,14 +78,6 @@ public PersonDao() { } } - public MetricRegistry getMetricRegistry() { - return metricRegistry; - } - - public void setMetricRegistry(MetricRegistry metricRegistry) { - this.metricRegistry = metricRegistry; - } - public Cache getDomainUsersCache() { return domainUsersCache; } @@ -263,6 +254,7 @@ private Person getFromCache(String domainUsername) { return null; } final Person person = domainUsersCache.get(domainUsername); + final MetricRegistry metricRegistry = AnetObjectEngine.getInstance().getMetricRegistry(); if (metricRegistry != null) { metricRegistry.counter(MetricRegistry.name(DOMAIN_USERS_CACHE, "LoadCount")).inc(); if (person == null) { From b3cd57ee8b7031adb5ed6fb53392e2ccf6800950 Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Tue, 27 Oct 2020 13:12:22 +0100 Subject: [PATCH 08/10] NCI-Agency/anet#3231: Introduce a batching & caching context for workers --- .../mil/dds/anet/threads/AbstractWorker.java | 68 ++++++++++++++++++- .../threads/AccountDeactivationWorker.java | 3 +- .../mil/dds/anet/threads/AnetEmailWorker.java | 48 ++++++------- .../anet/threads/FutureEngagementWorker.java | 3 +- .../MaterializedViewRefreshWorker.java | 3 +- .../anet/threads/ReportApprovalWorker.java | 4 +- .../anet/threads/ReportPublicationWorker.java | 4 +- 7 files changed, 97 insertions(+), 36 deletions(-) diff --git a/src/main/java/mil/dds/anet/threads/AbstractWorker.java b/src/main/java/mil/dds/anet/threads/AbstractWorker.java index 6a48b1d5b1..285a399b17 100644 --- a/src/main/java/mil/dds/anet/threads/AbstractWorker.java +++ b/src/main/java/mil/dds/anet/threads/AbstractWorker.java @@ -2,10 +2,15 @@ import java.lang.invoke.MethodHandles; import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; import mil.dds.anet.AnetObjectEngine; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.JobHistoryDao; +import mil.dds.anet.utils.BatchingUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,19 +29,78 @@ public AbstractWorker(AnetConfiguration config, String startMessage) { this.jobHistoryDao = AnetObjectEngine.getInstance().getJobHistoryDao(); } - protected abstract void runInternal(Instant now, JobHistory jobHistory); + protected abstract void runInternal(Instant now, JobHistory jobHistory, + Map context); @Override public final void run() { final String className = this.getClass().getSimpleName(); logger.debug("Starting {}: {}", className, startMessage); + final RunStatus runStatus = new RunStatus(); + BatchingUtils batchingUtils = null; try { - jobHistoryDao.runInTransaction(className, (now, jobHistory) -> runInternal(now, jobHistory)); + batchingUtils = startDispatcher(runStatus); + final Map context = new HashMap<>(); + context.put("dataLoaderRegistry", batchingUtils.getDataLoaderRegistry()); + jobHistoryDao.runInTransaction(className, + (now, jobHistory) -> runInternal(now, jobHistory, context)); } catch (Throwable e) { // Cannot let this thread die. Otherwise ANET will stop checking. logger.error("Exception in run()", e); + } finally { + runStatus.setDone(true); + if (batchingUtils != null) { + batchingUtils.updateStats(AnetObjectEngine.getInstance().getMetricRegistry(), + batchingUtils.getDataLoaderRegistry()); + batchingUtils.shutdown(); + } } logger.debug("Ending {}", className); } + private static class RunStatus { + private boolean done = false; + + public boolean isDone() { + return done; + } + + public void setDone(boolean done) { + this.done = done; + } + + @Override + public String toString() { + return "RunStatus [done=" + done + "]"; + } + } + + private BatchingUtils startDispatcher(final RunStatus runStatus) { + final BatchingUtils batchingUtils = + new BatchingUtils(AnetObjectEngine.getInstance(), true, true); + final Runnable dispatcher = () -> { + while (!runStatus.isDone()) { + // Wait a while, giving other threads the chance to do some work + try { + Thread.yield(); + Thread.sleep(50); + } catch (InterruptedException ignored) { + // just retry + } + + // Dispatch all our data loaders until the request is done; + // we have data loaders at various depths (one dependent on another), + // e.g. in {@link Report#loadWorkflow} + final CompletableFuture[] dispatchersWithWork = batchingUtils.getDataLoaderRegistry() + .getDataLoaders().stream().filter(dl -> dl.dispatchDepth() > 0) + .map(dl -> (CompletableFuture) dl.dispatch()).toArray(CompletableFuture[]::new); + if (dispatchersWithWork.length > 0) { + CompletableFuture.allOf(dispatchersWithWork).join(); + } + } + }; + Executors.newSingleThreadExecutor().execute(dispatcher); + return batchingUtils; + } + } diff --git a/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java b/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java index d841a804b0..aa3fc87029 100644 --- a/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java +++ b/src/main/java/mil/dds/anet/threads/AccountDeactivationWorker.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import mil.dds.anet.AnetObjectEngine; import mil.dds.anet.beans.AnetEmail; @@ -33,7 +34,7 @@ public AccountDeactivationWorker(AnetConfiguration config, PersonDao dao, } @Override - protected void runInternal(Instant now, JobHistory jobHistory) { + protected void runInternal(Instant now, JobHistory jobHistory, Map context) { // Make sure the mechanism will be triggered, so account deactivation checking can take place final List ignoredDomainNames = getDomainNamesToIgnore(); final List daysTillEndOfTourWarnings = getDaysTillEndOfTourWarnings(); diff --git a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java index c37729ba05..09b8fb25ca 100644 --- a/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java +++ b/src/main/java/mil/dds/anet/threads/AnetEmailWorker.java @@ -68,7 +68,7 @@ public static void setInstance(AnetEmailWorker instance) { } @Override - protected void runInternal(Instant now, JobHistory jobHistory) { + protected void runInternal(Instant now, JobHistory jobHistory, Map context) { @SuppressWarnings("unchecked") final List activeDomainNames = ((List) config.getDictionaryEntry("activeDomainNames")).stream() @@ -80,16 +80,16 @@ protected void runInternal(Instant now, JobHistory jobHistory) { // Send the emails! final List processedEmails = new LinkedList(); for (final AnetEmail email : emails) { - Map context = null; + Map emailContext = null; try { - context = buildContext(email, smtpConfig); - if (context != null) { + emailContext = buildContext(context, email); + if (emailContext != null) { logger.info("{} Sending email to {} re: {}", smtpConfig.isDisabled() ? "[Disabled] " : "", - email.getToAddresses(), email.getAction().getSubject(context)); + email.getToAddresses(), email.getAction().getSubject(emailContext)); if (!smtpConfig.isDisabled()) { - sendEmail(email, context, smtpProps, smtpAuth, activeDomainNames); + sendEmail(email, emailContext, smtpProps, smtpAuth, activeDomainNames); } } processedEmails.add(email.getId()); @@ -104,7 +104,7 @@ protected void runInternal(Instant now, JobHistory jobHistory) { String message = "Purging stale email to "; try { message += email.getToAddresses(); - message += email.getAction().getSubject(context); + message += email.getAction().getSubject(emailContext); } finally { logger.info(message); processedEmails.add(email.getId()); @@ -118,40 +118,40 @@ protected void runInternal(Instant now, JobHistory jobHistory) { dao.deletePendingEmails(processedEmails); } - private Map buildContext(final AnetEmail email, - final SmtpConfiguration smtpConfig) { + private Map buildContext(final Map context, + final AnetEmail email) { AnetObjectEngine engine = AnetObjectEngine.getInstance(); - Map context = new HashMap(); - context.put("context", engine.getContext()); - context.put("serverUrl", serverUrl); - context.put(AdminSettingKeys.SECURITY_BANNER_TEXT.name(), + Map emailContext = new HashMap(); + emailContext.put("context", context); + emailContext.put("serverUrl", serverUrl); + emailContext.put(AdminSettingKeys.SECURITY_BANNER_TEXT.name(), engine.getAdminSetting(AdminSettingKeys.SECURITY_BANNER_TEXT)); - context.put(AdminSettingKeys.SECURITY_BANNER_COLOR.name(), + emailContext.put(AdminSettingKeys.SECURITY_BANNER_COLOR.name(), engine.getAdminSetting(AdminSettingKeys.SECURITY_BANNER_COLOR)); - context.put("SUPPORT_EMAIL_ADDR", config.getDictionaryEntry("SUPPORT_EMAIL_ADDR")); - context.put("dateFormatter", + emailContext.put("SUPPORT_EMAIL_ADDR", config.getDictionaryEntry("SUPPORT_EMAIL_ADDR")); + emailContext.put("dateFormatter", DateTimeFormatter.ofPattern((String) config.getDictionaryEntry("dateFormats.email.date")) .withZone(DaoUtils.getDefaultZoneId())); - context.put("dateTimeFormatter", + emailContext.put("dateTimeFormatter", DateTimeFormatter .ofPattern((String) config.getDictionaryEntry("dateFormats.email.withTime")) .withZone(DaoUtils.getDefaultZoneId())); final boolean engagementsIncludeTimeAndDuration = Boolean.TRUE .equals((Boolean) config.getDictionaryEntry("engagementsIncludeTimeAndDuration")); - context.put("engagementsIncludeTimeAndDuration", engagementsIncludeTimeAndDuration); + emailContext.put("engagementsIncludeTimeAndDuration", engagementsIncludeTimeAndDuration); final String edtfPattern = (String) config .getDictionaryEntry(engagementsIncludeTimeAndDuration ? "dateFormats.email.withTime" : "dateFormats.email.date"); - context.put("engagementDateFormatter", + emailContext.put("engagementDateFormatter", DateTimeFormatter.ofPattern(edtfPattern).withZone(DaoUtils.getDefaultZoneId())); @SuppressWarnings("unchecked") final Map fields = (Map) config.getDictionaryEntry("fields"); - context.put("fields", fields); + emailContext.put("fields", fields); - return email.getAction().buildContext(context); + return email.getAction().buildContext(emailContext); } - private void sendEmail(final AnetEmail email, final Map context, + private void sendEmail(final AnetEmail email, final Map emailContext, final Properties smtpProps, final Authenticator smtpAuth, final List activeDomainNames) throws MessagingException, IOException, TemplateException { @@ -170,11 +170,11 @@ private void sendEmail(final AnetEmail email, final Map context, final Template temp = Utils.getFreemarkerConfig(this.getClass()).getTemplate(email.getAction().getTemplateName()); // scan:ignore — false positive, we know which template we are processing - temp.process(context, writer); + temp.process(emailContext, writer); final Session session = Session.getInstance(smtpProps, smtpAuth); final Email mail = EmailBuilder.startingBlank().from(new InternetAddress(fromAddr)) - .toMultiple(email.getToAddresses()).withSubject(email.getAction().getSubject(context)) + .toMultiple(email.getToAddresses()).withSubject(email.getAction().getSubject(emailContext)) .withHTMLText(writer.toString()).buildEmail(); try { diff --git a/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java b/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java index ce6d077875..e538f42819 100644 --- a/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java +++ b/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java @@ -22,7 +22,7 @@ public FutureEngagementWorker(AnetConfiguration config, ReportDao dao) { } @Override - protected void runInternal(Instant now, JobHistory jobHistory) { + protected void runInternal(Instant now, JobHistory jobHistory, Map context) { // Get a list of all reports related to upcoming engagements which have just // become past engagements and need to change their report status to draft. // When a report is for an engagement which just moved from future to past @@ -35,7 +35,6 @@ protected void runInternal(Instant now, JobHistory jobHistory) { // update to draft state and send emails to the authors to let them know we updated their // report. - final Map context = AnetObjectEngine.getInstance().getContext(); for (Report r : reports) { try { AnetEmail email = new AnetEmail(); diff --git a/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java b/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java index 4061a275c5..0b1d0ba113 100644 --- a/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java +++ b/src/main/java/mil/dds/anet/threads/MaterializedViewRefreshWorker.java @@ -1,6 +1,7 @@ package mil.dds.anet.threads; import java.time.Instant; +import java.util.Map; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.AdminDao; @@ -19,7 +20,7 @@ public MaterializedViewRefreshWorker(AnetConfiguration config, AdminDao dao) { } @Override - protected void runInternal(Instant now, JobHistory jobHistory) { + protected void runInternal(Instant now, JobHistory jobHistory, Map context) { for (final String materializedView : materializedViews) { try { dao.updateMaterializedView(materializedView); diff --git a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java index 39405a246b..e5f067487c 100644 --- a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java @@ -5,7 +5,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import mil.dds.anet.AnetObjectEngine; import mil.dds.anet.beans.ApprovalStep; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.beans.Report; @@ -27,7 +26,7 @@ public ReportApprovalWorker(AnetConfiguration config, ReportDao dao) { } @Override - protected void runInternal(Instant now, JobHistory jobHistory) { + protected void runInternal(Instant now, JobHistory jobHistory, Map context) { final Instant approvalTimeout = now.minus((Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursApprovalTimeout"), ChronoUnit.HOURS); @@ -37,7 +36,6 @@ protected void runInternal(Instant now, JobHistory jobHistory) { query.setState(Collections.singletonList(ReportState.PENDING_APPROVAL)); query.setSystemSearch(true); final List reports = dao.search(query).getList(); - final Map context = AnetObjectEngine.getInstance().getContext(); for (final Report r : reports) { final List workflow = r.loadWorkflow(context).join(); if (workflow.isEmpty()) { diff --git a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java index 86f65f2ec4..589feadf83 100644 --- a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java @@ -5,7 +5,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import mil.dds.anet.AnetObjectEngine; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.beans.Report; import mil.dds.anet.beans.Report.ReportState; @@ -25,7 +24,7 @@ public ReportPublicationWorker(AnetConfiguration config, ReportDao dao) { } @Override - protected void runInternal(Instant now, JobHistory jobHistory) { + protected void runInternal(Instant now, JobHistory jobHistory, Map context) { final Instant quarantineApproval = now.minus((Integer) config.getDictionaryEntry("reportWorkflow.nbOfHoursQuarantineApproved"), ChronoUnit.HOURS); @@ -35,7 +34,6 @@ protected void runInternal(Instant now, JobHistory jobHistory) { query.setState(Collections.singletonList(ReportState.APPROVED)); query.setSystemSearch(true); final List reports = dao.search(query).getList(); - final Map context = AnetObjectEngine.getInstance().getContext(); for (final Report r : reports) { final List workflow = r.loadWorkflow(context).join(); if (workflow.isEmpty()) { From 1f98e7687c5b5d17f674ec74bb2c90b821560b01 Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Wed, 28 Oct 2020 10:13:55 +0100 Subject: [PATCH 09/10] NCI-Agency/anet#3231: Make relevant parts of some workers asynchronous This gives the context the opportunity to batch queries. --- .../anet/threads/FutureEngagementWorker.java | 35 ++++++----- .../anet/threads/ReportApprovalWorker.java | 63 ++++++++++--------- .../anet/threads/ReportPublicationWorker.java | 45 +++++++------ 3 files changed, 80 insertions(+), 63 deletions(-) diff --git a/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java b/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java index e538f42819..54d494e6b5 100644 --- a/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java +++ b/src/main/java/mil/dds/anet/threads/FutureEngagementWorker.java @@ -3,11 +3,13 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import mil.dds.anet.AnetObjectEngine; import mil.dds.anet.beans.AnetEmail; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.beans.Report; +import mil.dds.anet.beans.ReportPerson; import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.ReportDao; import mil.dds.anet.emails.FutureEngagementUpdated; @@ -35,20 +37,25 @@ protected void runInternal(Instant now, JobHistory jobHistory, Map rp.getEmailAddress()) - .collect(Collectors.toList())); - AnetEmailWorker.sendEmailAsync(email); - dao.updateToDraftState(r); - } catch (Exception e) { - logger.error("Exception when updating", e); - } - } + final CompletableFuture[] allFutures = reports.stream().map(r -> { + final AnetEmail email = new AnetEmail(); + final FutureEngagementUpdated action = new FutureEngagementUpdated(); + action.setReport(r); + email.setAction(action); + return r.loadAuthors(context).thenApply(authors -> { + try { + email.setToAddresses( + authors.stream().map(ReportPerson::getEmailAddress).collect(Collectors.toList())); + AnetEmailWorker.sendEmailAsync(email); + dao.updateToDraftState(r); + } catch (Exception e) { + logger.error("Exception when updating", e); + } + return true; + }); + }).toArray(CompletableFuture[]::new); + // Wait for all our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); } } diff --git a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java index e5f067487c..5c11f92306 100644 --- a/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportApprovalWorker.java @@ -5,6 +5,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import mil.dds.anet.beans.ApprovalStep; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.beans.Report; @@ -36,40 +37,44 @@ protected void runInternal(Instant now, JobHistory jobHistory, Map reports = dao.search(query).getList(); - for (final Report r : reports) { - final List workflow = r.loadWorkflow(context).join(); - if (workflow.isEmpty()) { - logger.error("Couldn't process report approval for report {}, it has no workflow", - r.getUuid()); - } else { - for (int i = workflow.size() - 1; i >= 0; i--) { - final ReportAction reportAction = workflow.get(i); - if (reportAction.getCreatedAt() == null && i > 1) { - // Check previous action - final ReportAction previousAction = workflow.get(i - 1); - if (previousAction.getCreatedAt() != null - && previousAction.getCreatedAt().isBefore(approvalTimeout)) { - // Approve the report - try { - final ApprovalStep approvalStep = reportAction.getStep(); - final int numRows = dao.approve(r, null, approvalStep); - if (numRows == 0) { - logger.error("Couldn't process report approval for report {} step {}", - r.getUuid(), DaoUtils.getUuid(approvalStep)); - } else { - AnetAuditLogger.log( - "report {} step {} automatically approved by the ReportApprovalWorker", - r.getUuid(), DaoUtils.getUuid(approvalStep)); + final CompletableFuture[] allFutures = reports.stream().map(r -> { + return r.loadWorkflow(context).thenApply(workflow -> { + if (workflow.isEmpty()) { + logger.error("Couldn't process report approval for report {}, it has no workflow", + r.getUuid()); + } else { + for (int i = workflow.size() - 1; i >= 0; i--) { + final ReportAction reportAction = workflow.get(i); + if (reportAction.getCreatedAt() == null && i > 1) { + // Check previous action + final ReportAction previousAction = workflow.get(i - 1); + if (previousAction.getCreatedAt() != null + && previousAction.getCreatedAt().isBefore(approvalTimeout)) { + // Approve the report + try { + final ApprovalStep approvalStep = reportAction.getStep(); + final int numRows = dao.approve(r, null, approvalStep); + if (numRows == 0) { + logger.error("Couldn't process report approval for report {} step {}", + r.getUuid(), DaoUtils.getUuid(approvalStep)); + } else { + AnetAuditLogger.log( + "report {} step {} automatically approved by the ReportApprovalWorker", + r.getUuid(), DaoUtils.getUuid(approvalStep)); + } + } catch (Exception e) { + logger.error("Exception when approving report", e); } - } catch (Exception e) { - logger.error("Exception when approving report", e); + break; } - break; } } } - } - } + return true; + }); + }).toArray(CompletableFuture[]::new); + // Wait for all our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); } } diff --git a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java index 589feadf83..b789e84410 100644 --- a/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java +++ b/src/main/java/mil/dds/anet/threads/ReportPublicationWorker.java @@ -5,10 +5,10 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import mil.dds.anet.beans.JobHistory; import mil.dds.anet.beans.Report; import mil.dds.anet.beans.Report.ReportState; -import mil.dds.anet.beans.ReportAction; import mil.dds.anet.beans.search.ReportSearchQuery; import mil.dds.anet.config.AnetConfiguration; import mil.dds.anet.database.ReportDao; @@ -34,28 +34,33 @@ protected void runInternal(Instant now, JobHistory jobHistory, Map reports = dao.search(query).getList(); - for (final Report r : reports) { - final List workflow = r.loadWorkflow(context).join(); - if (workflow.isEmpty()) { - logger.error("Couldn't process report publication for report {}, it has no workflow", - r.getUuid()); - } else { - if (workflow.get(workflow.size() - 1).getCreatedAt().isBefore(quarantineApproval)) { - // Publish the report - try { - final int numRows = dao.publish(r, null); - if (numRows == 0) { - logger.error("Couldn't process report publication for report {}", r.getUuid()); - } else { - AnetAuditLogger.log( - "report {} automatically published by the ReportPublicationWorker", r.getUuid()); + final CompletableFuture[] allFutures = reports.stream().map(r -> { + return r.loadWorkflow(context).thenApply(workflow -> { + if (workflow.isEmpty()) { + logger.error("Couldn't process report publication for report {}, it has no workflow", + r.getUuid()); + } else { + if (workflow.get(workflow.size() - 1).getCreatedAt().isBefore(quarantineApproval)) { + // Publish the report + try { + final int numRows = dao.publish(r, null); + if (numRows == 0) { + logger.error("Couldn't process report publication for report {}", r.getUuid()); + } else { + AnetAuditLogger.log( + "report {} automatically published by the ReportPublicationWorker", + r.getUuid()); + } + } catch (Exception e) { + logger.error("Exception when publishing report", e); } - } catch (Exception e) { - logger.error("Exception when publishing report", e); } } - } - } + return true; + }); + }).toArray(CompletableFuture[]::new); + // Wait for all our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); } } From 49f3983e9086fa4c66009296ece4e6ba61816b3d Mon Sep 17 00:00:00 2001 From: Gertjan van Oosten Date: Wed, 18 Nov 2020 15:24:40 +0100 Subject: [PATCH 10/10] NCI-Agency/anet#3231: Add PendingAssessmentsNotificationWorker --- insertBaseData-mssql.sql | 22 +- mssql2pg.pl | 4 +- .../java/mil/dds/anet/AnetApplication.java | 8 + .../java/mil/dds/anet/beans/JobHistory.java | 4 + .../beans/search/PositionSearchQuery.java | 16 +- .../java/mil/dds/anet/database/TaskDao.java | 10 - .../PendingAssessmentsNotificationEmail.java | 102 +++ .../anet/search/AbstractPositionSearcher.java | 9 + .../PendingAssessmentsNotificationWorker.java | 610 ++++++++++++++++++ .../pendingAssessmentsNotification.ftlh | 65 ++ ...dingAssessmentsNotificationWorkerTest.java | 280 ++++++++ 11 files changed, 1116 insertions(+), 14 deletions(-) create mode 100644 src/main/java/mil/dds/anet/emails/PendingAssessmentsNotificationEmail.java create mode 100644 src/main/java/mil/dds/anet/threads/PendingAssessmentsNotificationWorker.java create mode 100644 src/main/resources/emails/pendingAssessmentsNotification.ftlh create mode 100644 src/test/java/mil/dds/anet/test/integration/db/PendingAssessmentsNotificationWorkerTest.java diff --git a/insertBaseData-mssql.sql b/insertBaseData-mssql.sql index 966cfcf5c5..aae24fb8f7 100644 --- a/insertBaseData-mssql.sql +++ b/insertBaseData-mssql.sql @@ -324,7 +324,7 @@ INSERT INTO tasks (uuid, shortName, longName, category, createdAt, updatedAt, cu VALUES (N'953e0b0b-25e6-44b6-bc77-ef98251d046a', '1.2.A', 'Milestone the First in EF 1.2', 'Milestone', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, N'fe6b6b2f-d2a1-4ce1-9aa7-05361812a4d0', '{ "assessments":[{"questions":{ "question1": { "type": "special_field", "widget": "likertScale", "label": "Test Question 1", "helpText": "Please provide assessment for something important", "levels": [ { "color": "red", "endValue": 2, "label": "test" }, { "color": "#FFBF00", "endValue": 8, "label": "mid" }, { "color": "green", "endValue": 10, "label": "high" } ], "aggregation": { "widget": "likertScale" } }, "question2": { "type": "number", "label": "Test Question 2", "aggregation": { "widget": "numberAggregation" } }, "question3": { "type": "number", "label": "Test Question 3", "aggregation": { "widget": "numberAggregation" } } },"relatedObjectType":"report"}] }'), (N'9d3da7f4-8266-47af-b518-995f587250c9', '1.2.B', 'Milestone the Second in EF 1.2', 'Milestone', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, N'fe6b6b2f-d2a1-4ce1-9aa7-05361812a4d0', '{ "assessments":[{"questions":{ "frenchFlag": { "type": "special_field", "widget": "likertScale", "label": "French Flag assessment", "helpText": "Please tell us which is the best color in the French flag", "levels": [ { "color": "blue", "endValue": 3.3, "label": "blue" }, { "color": "white", "endValue": 6.6, "label": "white" }, { "color": "red", "endValue": 10, "label": "red" } ] }, "levels": { "type": "enumset", "label": "Achieved levels", "choices": { "lvl1": { "label": "Level 1" }, "lvl2": { "label": "Level 2" }, "lvl3": { "label": "Level 3" } } }, "description": { "type": "special_field", "label": "Detail levels", "widget": "richTextEditor" } },"relatedObjectType":"report"}] }'), - (N'6bbb1be9-4655-48d7-83f2-bc474781544a', '1.2.C', 'Milestone the Third in EF 1.2', 'Milestone', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, N'fe6b6b2f-d2a1-4ce1-9aa7-05361812a4d0', '{ "assessments":[{"questions":{ "question1": { "type": "special_field", "widget": "likertScale", "label": "Monthly assessment Question 1", "helpText": "Please provide assessment for something important", "levels": [ { "color": "red", "endValue": 2, "label": "test" }, { "color": "#FFBF00", "endValue": 8, "label": "mid" }, { "color": "green", "endValue": 10, "label": "high" } ], "aggregation": { "widget": "likertScale" } }, "question2": { "type": "number", "label": "Monthly assessment Question 2", "aggregation": { "widget": "numberAggregation" } }, "question3": { "type": "number", "label": "Monthly assessment Question 3", "aggregation": { "widget": "numberAggregation" } } },"recurrence":"monthly"},{"questions":{ "question1": { "type": "special_field", "widget": "likertScale", "label": "Weekly assessment Question 1", "helpText": "Please provide assessment for something important", "levels": [ { "color": "red", "endValue": 2, "label": "test" }, { "color": "#FFBF00", "endValue": 8, "label": "mid" }, { "color": "green", "endValue": 10, "label": "high" } ], "aggregation": { "widget": "likertScale" } }, "question2": { "type": "number", "label": "Weekly assessment Question 2", "aggregation": { "widget": "numberAggregation" } }, "question3": { "type": "number", "label": "Weekly assessment Question 3", "aggregation": { "widget": "numberAggregation" } } },"recurrence":"weekly"},{"questions":{ "question1": { "type": "special_field", "widget": "likertScale", "label": "Instant assessment Question 1", "helpText": "Please provide assessment for something important", "levels": [ { "color": "red", "endValue": 2, "label": "test" }, { "color": "#FFBF00", "endValue": 8, "label": "mid" }, { "color": "green", "endValue": 10, "label": "high" } ], "aggregation": { "widget": "likertScale" } }, "question2": { "type": "number", "label": "Instant assessment Question 2", "aggregation": { "widget": "numberAggregation" } }, "question3": { "type": "number", "label": "Instant assessment Question 3", "aggregation": { "widget": "numberAggregation" } } },"relatedObjectType":"report"}] }'); + (N'6bbb1be9-4655-48d7-83f2-bc474781544a', '1.2.C', 'Milestone the Third in EF 1.2', 'Milestone', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, N'fe6b6b2f-d2a1-4ce1-9aa7-05361812a4d0', '{ "assessments":[{"questions":{ "question1": { "type": "special_field", "widget": "likertScale", "label": "Monthly assessment Question 1", "helpText": "Please provide assessment for something important", "levels": [ { "color": "red", "endValue": 2, "label": "test" }, { "color": "#FFBF00", "endValue": 8, "label": "mid" }, { "color": "green", "endValue": 10, "label": "high" } ], "aggregation": { "widget": "likertScale" } }, "question2": { "type": "number", "label": "Monthly assessment Question 2", "aggregation": { "widget": "numberAggregation" } }, "question3": { "type": "number", "label": "Monthly assessment Question 3", "aggregation": { "widget": "numberAggregation" } } },"recurrence":"quarterly"},{"questions":{ "question1": { "type": "special_field", "widget": "likertScale", "label": "Weekly assessment Question 1", "helpText": "Please provide assessment for something important", "levels": [ { "color": "red", "endValue": 2, "label": "test" }, { "color": "#FFBF00", "endValue": 8, "label": "mid" }, { "color": "green", "endValue": 10, "label": "high" } ], "aggregation": { "widget": "likertScale" } }, "question2": { "type": "number", "label": "Weekly assessment Question 2", "aggregation": { "widget": "numberAggregation" } }, "question3": { "type": "number", "label": "Weekly assessment Question 3", "aggregation": { "widget": "numberAggregation" } } },"recurrence":"daily"},{"questions":{ "question1": { "type": "special_field", "widget": "likertScale", "label": "Instant assessment Question 1", "helpText": "Please provide assessment for something important", "levels": [ { "color": "red", "endValue": 2, "label": "test" }, { "color": "#FFBF00", "endValue": 8, "label": "mid" }, { "color": "green", "endValue": 10, "label": "high" } ], "aggregation": { "widget": "likertScale" } }, "question2": { "type": "number", "label": "Instant assessment Question 2", "aggregation": { "widget": "numberAggregation" } }, "question3": { "type": "number", "label": "Instant assessment Question 3", "aggregation": { "widget": "numberAggregation" } } },"relatedObjectType":"report"}] }'); INSERT INTO tasks (uuid, shortName, longName, category, createdAt, updatedAt, customFieldRef1Uuid) VALUES @@ -955,6 +955,26 @@ INSERT INTO noteRelatedObjects (noteUuid, relatedObjectType, relatedObjectUuid) FROM tasks t WHERE t.shortName = '1.2.B'; +-- Add periodic assessment for a task +SET @authorUuid = (SELECT uuid FROM people WHERE name = 'ANDERSON, Andrew'); +SET @noteUuid = lower(newid()); +INSERT INTO notes (uuid, authorUuid, type, text, createdAt, updatedAt) + VALUES (@noteUuid, @authorUuid, 3, '{"status":"GREEN","issues":"
  1. one
  2. two
  3. three
","__recurrence":"monthly","__periodStart":"' + FORMAT(DATEADD(month, DATEDIFF(month, 0, CURRENT_TIMESTAMP), 0), 'yyyy-MM-dd') + '"}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); +INSERT INTO noteRelatedObjects (noteUuid, relatedObjectType, relatedObjectUuid) + SELECT @noteUuid, 'tasks', t.uuid + FROM tasks t + WHERE t.shortName = '1.1.A'; + +-- Add periodic assessment for a person +SET @authorUuid = (SELECT uuid FROM people WHERE name = 'JACKSON, Jack'); +SET @noteUuid = lower(newid()); +INSERT INTO notes (uuid, authorUuid, type, text, createdAt, updatedAt) + VALUES (@noteUuid, @authorUuid, 3, '{"test3":"3","test2":"3","test1":"3","__recurrence":"quarterly","__periodStart":"' + FORMAT(DATEADD(quarter, DATEDIFF(quarter, 0, CURRENT_TIMESTAMP), 0), 'yyyy-MM-dd') + '"}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP); +INSERT INTO noteRelatedObjects (noteUuid, relatedObjectType, relatedObjectUuid) + SELECT @noteUuid, 'people', p.uuid + FROM people p + WHERE p.name = 'ROGWELL, Roger'; + -- LEAVE THIS AS LAST STATEMENT -- Truncate all the dates (on reports etc.) to dates that could have been generated by -- Java (millisecond precision) rather than by the database itself (microsecond precision) diff --git a/mssql2pg.pl b/mssql2pg.pl index 4911c62d89..4b541ce325 100755 --- a/mssql2pg.pl +++ b/mssql2pg.pl @@ -20,7 +20,7 @@ # Eliminate the weird "[key]" column naming s/\[key\]/key/g; # Quote mixed-case column and table names -s/(?> getTaskedOrganizationsForTask( FkDataLoaderKey.TASK_TASKED_ORGANIZATIONS, taskUuid); } - @InTransaction - public List getActiveTasks(boolean topLevel) { - final String sql = String.format( - "/* get%1$sTasks */ SELECT * FROM tasks WHERE status = :active" - + " AND \"customFieldRef1Uuid\" %2$s NULL", - topLevel ? "TopLevel" : "SubLevel", topLevel ? "IS" : "IS NOT"); - return getDbHandle().createQuery(sql).bind("active", DaoUtils.getEnumId(Task.Status.ACTIVE)) - .map(new TaskMapper()).list(); - } - @Override public AnetBeanList search(TaskSearchQuery query) { return AnetObjectEngine.getInstance().getSearcher().getTaskSearcher().runSearch(query); diff --git a/src/main/java/mil/dds/anet/emails/PendingAssessmentsNotificationEmail.java b/src/main/java/mil/dds/anet/emails/PendingAssessmentsNotificationEmail.java new file mode 100644 index 0000000000..968c0988c9 --- /dev/null +++ b/src/main/java/mil/dds/anet/emails/PendingAssessmentsNotificationEmail.java @@ -0,0 +1,102 @@ +package mil.dds.anet.emails; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import mil.dds.anet.AnetObjectEngine; +import mil.dds.anet.beans.Person; +import mil.dds.anet.beans.Position; +import mil.dds.anet.beans.Task; +import mil.dds.anet.utils.IdDataLoaderKey; +import mil.dds.anet.views.UuidFetcher; + +public class PendingAssessmentsNotificationEmail implements AnetEmailAction { + + private String advisorUuid; + private List positionUuidsToAssess; + private List taskUuidsToAssess; + + @Override + public String getTemplateName() { + return "/emails/pendingAssessmentsNotification.ftlh"; + } + + @Override + public String getSubject(Map context) { + return "ANET assessments due"; + } + + @Override + public Map buildContext(Map context) { + @SuppressWarnings("unchecked") + final Map dbContext = (Map) context.get("context"); + + // Load positions and person & organization for each position + final UuidFetcher positionFetcher = new UuidFetcher(); + final List> positions = positionUuidsToAssess.stream() + .map(uuid -> positionFetcher.load(dbContext, IdDataLoaderKey.POSITIONS, uuid) + .thenCompose(pos -> pos.loadPerson(dbContext).thenCompose( + person -> pos.loadOrganization(dbContext).thenApply(organization -> pos)))) + .collect(Collectors.toList()); + + // Load tasks + final UuidFetcher taskFetcher = new UuidFetcher(); + final List> tasks = taskUuidsToAssess.stream() + .map(uuid -> taskFetcher.load(dbContext, IdDataLoaderKey.TASKS, uuid)) + .collect(Collectors.toList()); + + // Wait for our futures to complete + final List> allFutures = new ArrayList<>(); + allFutures.addAll(positions); + allFutures.addAll(tasks); + CompletableFuture.allOf(allFutures.toArray(new CompletableFuture[0])).join(); + + // Fill email context + context.put("advisor", AnetObjectEngine.getInstance().getPersonDao().getByUuid(advisorUuid)); + context.put("positions", positions.stream().map(p -> p.join()).collect(Collectors.toList())); + context.put("tasks", tasks.stream().map(p -> p.join()).collect(Collectors.toList())); + return context; + } + + public String getAdvisorUuid() { + return advisorUuid; + } + + public void setAdvisorUuid(String advisorUuid) { + this.advisorUuid = advisorUuid; + } + + public void setAdvisor(Person advisor) { + this.advisorUuid = advisor.getUuid(); + } + + public List getPositionUuidsToAssess() { + return positionUuidsToAssess; + } + + public void setPositionUuidsToAssess(List positionUuidsToAssess) { + this.positionUuidsToAssess = positionUuidsToAssess; + } + + public void setPositionsToAssess(Set positionsToAssess) { + this.positionUuidsToAssess = + positionsToAssess.stream().map(p -> p.getUuid()).collect(Collectors.toList()); + } + + public List getTaskUuidsToAssess() { + return taskUuidsToAssess; + } + + public void setTaskUuidsToAssess(List taskUuidsToAssess) { + this.taskUuidsToAssess = taskUuidsToAssess; + } + + public void setTasksToAssess(Set tasksToAssess) { + this.taskUuidsToAssess = + tasksToAssess.stream().map(t -> t.getUuid()).collect(Collectors.toList()); + } + +} diff --git a/src/main/java/mil/dds/anet/search/AbstractPositionSearcher.java b/src/main/java/mil/dds/anet/search/AbstractPositionSearcher.java index fc63248535..98e10dbbd9 100644 --- a/src/main/java/mil/dds/anet/search/AbstractPositionSearcher.java +++ b/src/main/java/mil/dds/anet/search/AbstractPositionSearcher.java @@ -75,6 +75,15 @@ protected void buildQuery(PositionSearchQuery query) { qb.addSqlArg("authorizationGroupUuid", query.getAuthorizationGroupUuid()); } + if (query.getHasCounterparts()) { + qb.addWhereClause("(" + + "positions.uuid IN (SELECT \"positionUuid_a\" FROM \"positionRelationships\"" + + " WHERE \"positionUuid_b\" IS NOT NULL AND deleted = :deleted)" + + " OR positions.uuid IN (" + "SELECT \"positionUuid_b\" FROM \"positionRelationships\"" + + " WHERE \"positionUuid_a\" IS NOT NULL AND deleted = :deleted))"); + qb.addSqlArg("deleted", false); + } + addOrderByClauses(qb, query); } diff --git a/src/main/java/mil/dds/anet/threads/PendingAssessmentsNotificationWorker.java b/src/main/java/mil/dds/anet/threads/PendingAssessmentsNotificationWorker.java new file mode 100644 index 0000000000..5f8d2b15f6 --- /dev/null +++ b/src/main/java/mil/dds/anet/threads/PendingAssessmentsNotificationWorker.java @@ -0,0 +1,610 @@ +package mil.dds.anet.threads; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.google.common.collect.ImmutableList; +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoField; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAdjuster; +import java.time.temporal.TemporalAdjusters; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import mil.dds.anet.AnetObjectEngine; +import mil.dds.anet.beans.AnetEmail; +import mil.dds.anet.beans.JobHistory; +import mil.dds.anet.beans.Note.NoteType; +import mil.dds.anet.beans.Person; +import mil.dds.anet.beans.Position; +import mil.dds.anet.beans.Position.PositionType; +import mil.dds.anet.beans.Task; +import mil.dds.anet.beans.search.PositionSearchQuery; +import mil.dds.anet.beans.search.TaskSearchQuery; +import mil.dds.anet.config.AnetConfiguration; +import mil.dds.anet.database.PositionDao; +import mil.dds.anet.database.TaskDao; +import mil.dds.anet.emails.PendingAssessmentsNotificationEmail; +import mil.dds.anet.utils.DaoUtils; +import mil.dds.anet.utils.Utils; +import mil.dds.anet.views.AbstractAnetBean; + +public class PendingAssessmentsNotificationWorker extends AbstractWorker { + + // Recurrence types that we support + public enum Recurrence { + DAILY("daily"), WEEKLY("weekly"), BIWEEKLY("biweekly"), SEMIMONTHLY("semimonthly"), + MONTHLY("monthly"), QUARTERLY("quarterly"), SEMIANNUALLY("semiannually"), ANNUALLY("annually"); + + private final String recurrence; + + private Recurrence(final String recurrence) { + this.recurrence = recurrence; + } + + @Override + public String toString() { + return recurrence; + } + + public static Recurrence valueOfRecurrence(final String recurrence) { + for (final Recurrence v : values()) { + if (v.recurrence.equalsIgnoreCase(recurrence)) { + return v; + } + } + return null; + } + } + + /** + * Given a reference date and a recurrence, compute: the assessment date of the most recent + * completed assessment period, the date a notification should be sent, and the date a reminder + * should be sent. + */ + public static class AssessmentDates { + private final ZonedDateTime assessmentDate; + private final ZonedDateTime notificationDate; + private final ZonedDateTime reminderDate; + + public AssessmentDates(final Instant referenceDate, final Recurrence recurrence) { + // Compute some period boundaries + final ZonedDateTime zonefulReferenceDate = referenceDate.atZone(DaoUtils.getDefaultZoneId()); + final ZonedDateTime bod = zonefulReferenceDate.truncatedTo(ChronoUnit.DAYS); + // Monday is the first day of the week + final TemporalAdjuster firstDayOfWeek = TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY); + final ZonedDateTime bow = + zonefulReferenceDate.with(firstDayOfWeek).truncatedTo(ChronoUnit.DAYS); + // Bi-weekly reference date is first Monday of 2021 + final ZonedDateTime biWeeklyReferenceDate = + LocalDate.of(2021, 1, 4).with(firstDayOfWeek).atStartOfDay(DaoUtils.getDefaultZoneId()) + .toInstant().atZone(DaoUtils.getDefaultZoneId()); + final ZonedDateTime bom = zonefulReferenceDate.with(TemporalAdjusters.firstDayOfMonth()) + .truncatedTo(ChronoUnit.DAYS); + final ZonedDateTime boy = zonefulReferenceDate.with(TemporalAdjusters.firstDayOfYear()) + .truncatedTo(ChronoUnit.DAYS); + final int moyLessOne = zonefulReferenceDate.get(ChronoField.MONTH_OF_YEAR) - 1; + + switch (recurrence) { + case DAILY: + notificationDate = bod; + assessmentDate = notificationDate.minus(1, ChronoUnit.DAYS); + reminderDate = null; // no reminders + break; + case WEEKLY: + notificationDate = bow; + assessmentDate = notificationDate.minus(1, ChronoUnit.WEEKS); + reminderDate = notificationDate.plus(3, ChronoUnit.DAYS); + break; + case BIWEEKLY: + notificationDate = bow.minus( + Math.abs(ChronoUnit.WEEKS.between(biWeeklyReferenceDate, bow)) % 2, ChronoUnit.WEEKS); + assessmentDate = notificationDate.minus(2, ChronoUnit.WEEKS); + reminderDate = notificationDate.plus(5, ChronoUnit.DAYS); + break; + case SEMIMONTHLY: // two per month: [1 - 14] and [15 - end-of-month] + final int daysInFirstPeriod = 14; + if (zonefulReferenceDate.get(ChronoField.DAY_OF_MONTH) <= daysInFirstPeriod) { + notificationDate = bom; + assessmentDate = + bom.minus(1, ChronoUnit.MONTHS).plus(daysInFirstPeriod, ChronoUnit.DAYS); + } else { + notificationDate = bom.plus(daysInFirstPeriod, ChronoUnit.DAYS); + assessmentDate = bom; + } + reminderDate = notificationDate.plus(5, ChronoUnit.DAYS); + break; + case MONTHLY: + notificationDate = bom; + assessmentDate = notificationDate.minus(1, ChronoUnit.MONTHS); + reminderDate = notificationDate.plus(1, ChronoUnit.WEEKS); + break; + case QUARTERLY: + final long monthsInQuarter = 3; + final long q = moyLessOne / monthsInQuarter; + notificationDate = boy.plus(q * monthsInQuarter, ChronoUnit.MONTHS); + assessmentDate = notificationDate.minus(monthsInQuarter, ChronoUnit.MONTHS); + reminderDate = notificationDate.plus(4, ChronoUnit.WEEKS); + break; + case SEMIANNUALLY: // two per year: [Jan 1 - Jun 30] and [Jul 1 - Dec 31] + final long monthsInHalfYear = 6; + final long sa = moyLessOne / monthsInHalfYear; + notificationDate = boy.plus(sa * monthsInHalfYear, ChronoUnit.MONTHS); + assessmentDate = notificationDate.minus(monthsInHalfYear, ChronoUnit.MONTHS); + reminderDate = notificationDate.plus(1, ChronoUnit.MONTHS); + break; + case ANNUALLY: + notificationDate = boy; + assessmentDate = notificationDate.minus(1, ChronoUnit.YEARS); + reminderDate = notificationDate.plus(1, ChronoUnit.MONTHS); + break; + default: + // Unknown recurrence + logger.error("Unknown recurrence encountered: {}", recurrence); + assessmentDate = null; + notificationDate = null; + reminderDate = null; + break; + } + } + + /** + * @return the date of the most recent completed assessment period of the given recurrence + * before the given reference date + */ + public Instant getAssessmentDate() { + return getInstant(assessmentDate); + } + + /** + * @return the notification date for the assessment period of the given recurrence and the given + * reference date; may be null meaning: don't send notifications + */ + public Instant getNotificationDate() { + return getInstant(notificationDate); + } + + /** + * @return the reminder date for the assessment period of the given recurrence and the given + * reference date; may be null meaning: don't send reminders + */ + public Instant getReminderDate() { + return getInstant(reminderDate); + } + + private Instant getInstant(final ZonedDateTime zonedDateTime) { + return zonedDateTime == null ? null : zonedDateTime.toInstant(); + } + + @Override + public String toString() { + return "AssessmentDates [assessmentDate=" + assessmentDate + ", notificationDate=" + + notificationDate + ", reminderDate=" + reminderDate + "]"; + } + } + + // Dictionary lookup keys we use + private static final String PRINCIPAL_PERSON_ASSESSMENTS = "fields.principal.person.assessments"; + private static final String TASK_SUB_LEVEL_ASSESSMENTS = "fields.task.subLevel.assessments"; + private static final String TASK_TOP_LEVEL_ASSESSMENTS = "fields.task.topLevel.assessments"; + // JSON fields in task.customFields we use + private static final String TASK_ASSESSMENTS = "assessments"; + private static final String TASK_RECURRENCE = "recurrence"; + // JSON fields in note.text we use + private static final String NOTE_RECURRENCE = "__recurrence"; + private static final String NOTE_PERIOD_START = "__periodStart"; + + private final PositionDao positionDao; + private final TaskDao taskDao; + + public PendingAssessmentsNotificationWorker(AnetConfiguration config) { + super(config, + "Pending Assessments Notification Worker waking up to check for pending periodic assessments"); + this.positionDao = AnetObjectEngine.getInstance().getPositionDao(); + this.taskDao = AnetObjectEngine.getInstance().getTaskDao(); + } + + @Override + protected void runInternal(Instant now, JobHistory jobHistory, Map context) { + // Define which recurrences may need to be notified since last run + final Set recurrenceSet = getRecurrenceSet(now, JobHistory.getLastRun(jobHistory)); + if (recurrenceSet.isEmpty()) { + logger.debug("Nothing to do, now new recurrences since last run"); + return; + } + + // Look up periodic assessment definitions for people in the dictionary + final Set globalPositionAssessmentRecurrence = + getGlobalAssessmentRecurrence(recurrenceSet, PRINCIPAL_PERSON_ASSESSMENTS); + logger.trace("globalPositionAssessmentRecurrence={}", globalPositionAssessmentRecurrence); + + // Look up periodic assessment definitions for all tasks + final Map> taskAssessmentRecurrence = new HashMap<>(); + addTaskDefinitions(recurrenceSet, taskAssessmentRecurrence, true); + addTaskDefinitions(recurrenceSet, taskAssessmentRecurrence, false); + logger.trace("taskAssessmentRecurrence={}", taskAssessmentRecurrence); + + // Prepare maps of positions and tasks linked to active advisor positions + final Map objectsToAssessByPosition = new HashMap<>(); + final Map> allPositionsToAssess = preparePositionAssessmentMap( + context, globalPositionAssessmentRecurrence, objectsToAssessByPosition); + logger.trace("the following positions need to be checked for missing assessments: {}", + allPositionsToAssess); + final Map> allTasksToAssess = + prepareTaskAssessmentMap(context, taskAssessmentRecurrence, objectsToAssessByPosition); + logger.trace("the following tasks need to be checked for missing assessments: {}", + allTasksToAssess); + + // First load person for each position, and filter out the inactive ones + filterPositionsToAssessOnPerson(context, allPositionsToAssess); + // Process the existing assessments for positions to assess + processExistingAssessments(context, now, recurrenceSet, allPositionsToAssess); + // Process the existing assessments for tasks to assess + processExistingAssessments(context, now, recurrenceSet, allTasksToAssess); + // Now filter out the ones that don't need assessments + filterObjectsToAssess(objectsToAssessByPosition, allPositionsToAssess, allTasksToAssess); + // Load the people who should receive a notification email + loadPeopleToBeNotified(context, objectsToAssessByPosition, allPositionsToAssess, + allTasksToAssess); + } + + private Set getRecurrenceSet(final Instant now, final Instant lastRun) { + final Set recurrenceSet = + Stream.of(Recurrence.values()).collect(Collectors.toSet()); + for (final Iterator iter = recurrenceSet.iterator(); iter.hasNext();) { + final Recurrence recurrence = iter.next(); + final AssessmentDates assessmentDates = new AssessmentDates(now, recurrence); + // Note that if someone gets assigned a new counterpart or a new task, or the recurrence of + // assessment definitions is changed, this means they may not be notified until the *next* + // period. + if (!shouldAssess(now, lastRun, assessmentDates)) { + logger.debug("recurrence {} does not need checking since last run {}", recurrence, lastRun); + iter.remove(); + } + } + return recurrenceSet; + } + + private boolean shouldAssess(final Instant now, final Instant lastRun, + final AssessmentDates assessmentDates) { + return assessmentDates.getAssessmentDate() != null // no assessment + && (shouldAssess(now, lastRun, assessmentDates.getNotificationDate()) + || shouldAssess(now, lastRun, assessmentDates.getReminderDate())); + } + + private boolean shouldAssess(final Instant now, final Instant lastRun, final Instant date) { + return date != null && (lastRun == null || date.isAfter(lastRun) && !date.isAfter(now)); + } + + private Set getGlobalAssessmentRecurrence(final Set recurrenceSet, + final String keyPath) { + final Set globalPersonAssessmentRecurrence = new HashSet<>(); + @SuppressWarnings("unchecked") + final List> personAssessmentDefinitions = + (List>) config.getDictionaryEntry(keyPath); + if (personAssessmentDefinitions != null) { + personAssessmentDefinitions.stream().forEach(pad -> { + final Recurrence recurrence = + Recurrence.valueOfRecurrence((String) pad.get(TASK_RECURRENCE)); + if (shouldAddRecurrence(recurrenceSet, recurrence)) { + globalPersonAssessmentRecurrence.add(recurrence); + } + }); + } + return globalPersonAssessmentRecurrence; + } + + private boolean shouldAddRecurrence(final Set recurrenceSet, + final Recurrence recurrence) { + return recurrenceSet.contains(recurrence); + } + + private void addTaskDefinitions(final Set recurrenceSet, + final Map> taskAssessmentRecurrence, boolean topLevel) { + // Look up periodic assessment definitions for all tasks in the dictionary + final Set globalTaskAssessmentRecurrence = getGlobalAssessmentRecurrence( + recurrenceSet, topLevel ? TASK_TOP_LEVEL_ASSESSMENTS : TASK_SUB_LEVEL_ASSESSMENTS); + + // Look up periodic assessment definitions for each tasks in customFields + final List tasks = getActiveTasks(topLevel); + tasks.stream().forEach(t -> { + if (!globalTaskAssessmentRecurrence.isEmpty()) { + // Add all global recurrence definitions for this task + taskAssessmentRecurrence.computeIfAbsent(t, + task -> new HashSet<>(globalTaskAssessmentRecurrence)); + } + try { + final JsonNode taskCustomFields = Utils.parseJsonSafe(t.getCustomFields()); + if (taskCustomFields != null) { + final JsonNode taskAssessmentsDefinition = taskCustomFields.get(TASK_ASSESSMENTS); + if (taskAssessmentsDefinition != null && taskAssessmentsDefinition.isArray()) { + final ArrayNode arrayNode = (ArrayNode) taskAssessmentsDefinition; + for (int i = 0; i < arrayNode.size(); i++) { + final JsonNode recurrenceDefinition = arrayNode.get(i).get(TASK_RECURRENCE); + if (recurrenceDefinition != null) { + final Recurrence recurrence = + Recurrence.valueOfRecurrence(recurrenceDefinition.asText()); + if (shouldAddRecurrence(recurrenceSet, recurrence)) { + // Add task-specific recurrence definition + taskAssessmentRecurrence.compute(t, (task, currentValue) -> { + if (currentValue == null) { + return new HashSet<>(Collections.singleton(recurrence)); + } else { + currentValue.add(recurrence); + return currentValue; + } + }); + } + } + } + } + } + } catch (JsonProcessingException ignored) { + // Invalid JSON, log and skip it + logger.error("Task {} has invalid JSON in customFields: {}", t, t.getCustomFields()); + } + }); + } + + private Map> preparePositionAssessmentMap( + final Map context, final Set globalPositionAssessmentRecurrence, + final Map objectsToAssessByPosition) { + final Map> allPositionsToAssess = new HashMap<>(); + final CompletableFuture[] allFutures = getActiveAdvisorPositions(true).stream() + .map(p -> getPositionsToAssess(context, p, globalPositionAssessmentRecurrence) + .thenApply(positionsToAssess -> { + if (!positionsToAssess.isEmpty()) { + positionsToAssess.stream().forEach(pta -> allPositionsToAssess.put(pta, + new HashSet<>(globalPositionAssessmentRecurrence))); + objectsToAssessByPosition.put(p, new ObjectsToAssess(positionsToAssess, null)); + } + return null; + })) + .toArray(CompletableFuture[]::new); + // Wait for our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); + return allPositionsToAssess; + } + + private Map> prepareTaskAssessmentMap(final Map context, + final Map> taskAssessmentRecurrence, + final Map objectsToAssessByPosition) { + final List activeAdvisors = getActiveAdvisorPositions(false); + final Map> allTasksToAssess = new HashMap<>(); + final CompletableFuture[] allFutures = + taskAssessmentRecurrence.entrySet().stream().map(e -> { + final Task taskToAssess = e.getKey(); + final Set recurrenceSet = e.getValue(); + return taskToAssess.loadResponsiblePositions(context).thenApply(positions -> { + // Only active advisors can assess + final Set positionsToAssess = positions.stream() + .filter(pos -> activeAdvisors.contains(pos)).collect(Collectors.toSet()); + if (!positionsToAssess.isEmpty()) { + allTasksToAssess.put(taskToAssess, recurrenceSet); + positionsToAssess.stream().forEach(pta -> { + objectsToAssessByPosition.compute(pta, (pos, currentValue) -> { + if (currentValue == null) { + return new ObjectsToAssess(null, Collections.singleton(taskToAssess)); + } else { + currentValue.getTasksToAssess().add(taskToAssess); + return currentValue; + } + }); + }); + } + return null; + }); + }).toArray(CompletableFuture[]::new); + // Wait for our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); + return allTasksToAssess; + } + + private List getActiveAdvisorPositions(boolean withCounterparts) { + // Get all active, filled advisor positions, possibly with counterparts + final PositionSearchQuery psq = new PositionSearchQuery(); + psq.setPageSize(0); + psq.setStatus(Position.Status.ACTIVE); + psq.setIsFilled(Boolean.TRUE); + if (withCounterparts) { + psq.setHasCounterparts(Boolean.TRUE); + } + psq.setType(ImmutableList.of(PositionType.ADMINISTRATOR, PositionType.SUPER_USER, + PositionType.ADVISOR)); + return positionDao.search(psq).getList(); + } + + private List getActiveTasks(boolean topLevel) { + // Get all active tasks with a non-empty customFields + final TaskSearchQuery tsq = new TaskSearchQuery(); + tsq.setPageSize(0); + tsq.setStatus(Position.Status.ACTIVE); + tsq.setHasCustomFieldRef1(!topLevel); + return taskDao.search(tsq).getList(); + } + + private CompletableFuture> getPositionsToAssess(final Map context, + final Position position, final Set globalPersonAssessmentRecurrence) { + if (position == null || globalPersonAssessmentRecurrence.isEmpty()) { + return CompletableFuture.completedFuture(Collections.emptySet()); + } else { + return position.loadAssociatedPositions(context).thenApply(ap -> ap.stream() + .filter(pp -> Position.Status.ACTIVE.equals(pp.getStatus())).collect(Collectors.toSet())); + } + } + + private void filterPositionsToAssessOnPerson(final Map context, + final Map> allPositionsToAssess) { + // Load person for each position + final CompletableFuture[] allFutures = allPositionsToAssess.keySet().stream() + .map(p -> p.loadPerson(context)).toArray(CompletableFuture[]::new); + // Wait for our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); + // Remove inactive people + for (final Iterator aptai = allPositionsToAssess.keySet().iterator(); aptai + .hasNext();) { + final Position p = aptai.next(); + if (p.getPerson() == null || !Person.Status.ACTIVE.equals(p.getPerson().getStatus())) { + aptai.remove(); + } + } + } + + private void processExistingAssessments(final Map context, final Instant now, + final Set recurrenceSet, + final Map> objectsToAssess) { + final CompletableFuture[] allFutures = objectsToAssess.entrySet().stream().map(e -> { + final AbstractAnetBean entryKey = e.getKey(); + final Set periods = e.getValue(); + // For positions the current person holding it gets assessed + final AbstractAnetBean ota = + entryKey instanceof Position ? ((Position) entryKey).getPerson() : entryKey; + return ota.loadNotes(context).thenApply(notes -> { + final Map assessmentsByRecurrence = new HashMap<>(); + notes.stream().filter(note -> NoteType.ASSESSMENT.equals(note.getType())).forEach(note -> { + try { + final JsonNode noteJson = Utils.parseJsonSafe(note.getText()); + final JsonNode recurrence = noteJson.get(NOTE_RECURRENCE); + final JsonNode periodStart = noteJson.get(NOTE_PERIOD_START); + if (periodStart != null && recurrence != null && shouldAddRecurrence(recurrenceSet, + Recurrence.valueOfRecurrence(recurrence.asText()))) { + // __periodStart is stored in the database as a zone-agnostic date string yyyy-mm-dd + final LocalDate periodStartDate = + DateTimeFormatter.ISO_LOCAL_DATE.parse(periodStart.asText(), LocalDate::from); + final Instant periodStartInstant = + periodStartDate.atStartOfDay(DaoUtils.getDefaultZoneId()).toInstant(); + assessmentsByRecurrence.compute(Recurrence.valueOfRecurrence(recurrence.asText()), + (r, currentValue) -> currentValue == null ? periodStartInstant + : periodStartInstant.isAfter(currentValue) ? periodStartInstant + : currentValue); + } + } catch (JsonProcessingException ignored) { + // Invalid JSON, skip it + } + }); + assessmentsByRecurrence.entrySet().stream().forEach(entry -> { + final Recurrence recurrence = entry.getKey(); + final Instant lastAssessment = entry.getValue(); + final AssessmentDates assessmentDates = new AssessmentDates(now, recurrence); + if (assessmentDates.getAssessmentDate() == null + || !lastAssessment.isBefore(assessmentDates.getAssessmentDate())) { + // Assessment already done + logger.trace("{} assessment for {} already done on {}", recurrence, ota, + lastAssessment); + periods.remove(recurrence); + } + }); + return null; + }); + }).toArray(CompletableFuture[]::new); + // Wait for our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); + } + + private void filterObjectsToAssess(final Map objectsToAssessByPosition, + final Map> allPositionsToAssess, + final Map> allTasksToAssess) { + for (final Iterator> otabpi = + objectsToAssessByPosition.entrySet().iterator(); otabpi.hasNext();) { + final Entry otabp = otabpi.next(); + final ObjectsToAssess ota = otabp.getValue(); + final Set positionsToAssess = ota.getPositionsToAssess(); + for (final Iterator ptai = positionsToAssess.iterator(); ptai.hasNext();) { + final Position pta = ptai.next(); + if (!allPositionsToAssess.containsKey(pta) || allPositionsToAssess.get(pta).isEmpty()) { + // Position/person does not need assessment + logger.trace("person {} does not need assessments", pta.getPerson()); + ptai.remove(); + } + } + + final Set tasksToAssess = ota.getTasksToAssess(); + for (final Iterator ttai = tasksToAssess.iterator(); ttai.hasNext();) { + final Task tta = ttai.next(); + if (!allTasksToAssess.containsKey(tta) || allTasksToAssess.get(tta).isEmpty()) { + // Task does not need assessment + logger.trace("task {} does not need assessments", tta); + ttai.remove(); + } + } + + if (positionsToAssess.isEmpty() && tasksToAssess.isEmpty()) { + // Nothing to assess by this position + logger.trace("position {} has no pending assessments", otabp.getKey()); + otabpi.remove(); + } + } + } + + private void loadPeopleToBeNotified(final Map context, + final Map objectsToAssessByPosition, + final Map> allPositionsToAssess, + final Map> allTasksToAssess) { + final CompletableFuture[] allFutures = + objectsToAssessByPosition.entrySet().stream().map(otabp -> { + final Position pos = otabp.getKey(); + // Get the person to be notified + return pos.loadPerson(context).thenApply(advisor -> { + final ObjectsToAssess ota = otabp.getValue(); + final Set positionsToAssess = ota.getPositionsToAssess(); + final Set tasksToAssess = ota.getTasksToAssess(); + logger.info("advisor {} should do assessments:", advisor); + positionsToAssess.stream() + .forEach(pta -> logger.info(" - {} for position {} held by person {}", + allPositionsToAssess.get(pta), pta, pta.getPerson())); + tasksToAssess.stream() + .forEach(tta -> logger.info(" - {} for task {}", allTasksToAssess.get(tta), tta)); + sendEmail(advisor, positionsToAssess, tasksToAssess); + return null; + }); + }).toArray(CompletableFuture[]::new); + // Wait for our futures to complete before returning + CompletableFuture.allOf(allFutures).join(); + } + + private void sendEmail(Person advisor, final Set positionsToAssess, + final Set tasksToAssess) { + final AnetEmail email = new AnetEmail(); + final PendingAssessmentsNotificationEmail action = new PendingAssessmentsNotificationEmail(); + action.setAdvisor(advisor); + action.setPositionsToAssess(positionsToAssess); + action.setTasksToAssess(tasksToAssess); + email.setAction(action); + email.setToAddresses(Collections.singletonList(advisor.getEmailAddress())); + AnetEmailWorker.sendEmailAsync(email); + } + + private static class ObjectsToAssess { + private final Set positionsToAssess; + private final Set tasksToAssess; + + public ObjectsToAssess(Set positionsToAssess, Set tasksToAssess) { + this.positionsToAssess = + new HashSet<>(positionsToAssess == null ? Collections.emptySet() : positionsToAssess); + this.tasksToAssess = + new HashSet<>(tasksToAssess == null ? Collections.emptySet() : tasksToAssess); + } + + public Set getPositionsToAssess() { + return positionsToAssess; + } + + public Set getTasksToAssess() { + return tasksToAssess; + } + } + +} diff --git a/src/main/resources/emails/pendingAssessmentsNotification.ftlh b/src/main/resources/emails/pendingAssessmentsNotification.ftlh new file mode 100644 index 0000000000..4eccbde0a4 --- /dev/null +++ b/src/main/resources/emails/pendingAssessmentsNotification.ftlh @@ -0,0 +1,65 @@ + + + + +

Classification: ${SECURITY_BANNER_TEXT}

+

Dear ${advisor.name},

+ +<#if tasks?has_content> +
+

The following ${fields.task.longLabel} for which you are responsible, have their periodic assessment due:

+ <#list tasks as task> + + +
+ + +<#if positions?has_content> + + + +

Please create your periodic assessment(s).

+ +

ANET Support Team +<#if SUPPORT_EMAIL_ADDR??> +
${SUPPORT_EMAIL_ADDR} +

+ + + diff --git a/src/test/java/mil/dds/anet/test/integration/db/PendingAssessmentsNotificationWorkerTest.java b/src/test/java/mil/dds/anet/test/integration/db/PendingAssessmentsNotificationWorkerTest.java new file mode 100644 index 0000000000..c6124b704a --- /dev/null +++ b/src/test/java/mil/dds/anet/test/integration/db/PendingAssessmentsNotificationWorkerTest.java @@ -0,0 +1,280 @@ +package mil.dds.anet.test.integration.db; + +import static mil.dds.anet.threads.PendingAssessmentsNotificationWorker.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import com.google.common.collect.ImmutableSet; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.lang.invoke.MethodHandles; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import mil.dds.anet.AnetObjectEngine; +import mil.dds.anet.beans.AnetEmail; +import mil.dds.anet.beans.JobHistory; +import mil.dds.anet.config.AnetConfiguration; +import mil.dds.anet.database.EmailDao; +import mil.dds.anet.database.JobHistoryDao; +import mil.dds.anet.database.PositionDao; +import mil.dds.anet.database.TaskDao; +import mil.dds.anet.emails.PendingAssessmentsNotificationEmail; +import mil.dds.anet.test.integration.config.AnetTestConfiguration; +import mil.dds.anet.test.integration.utils.FakeSmtpServer; +import mil.dds.anet.test.integration.utils.TestApp; +import mil.dds.anet.threads.AnetEmailWorker; +import mil.dds.anet.threads.PendingAssessmentsNotificationWorker; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ExtendWith(TestApp.class) +public class PendingAssessmentsNotificationWorkerTest { + + protected static final Logger logger = + LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static PendingAssessmentsNotificationWorker pendingAssessmentsNotificationWorker; + private static AnetEmailWorker emailWorker; + private static FakeSmtpServer emailServer; + + private static boolean executeEmailServerTests; + + @BeforeAll + public static void setUpClass() throws Exception { + final DropwizardAppExtension app = TestApp.app; + if (app.getConfiguration().getSmtp().isDisabled()) { + fail("'ANET_SMTP_DISABLE' system environment variable must have value 'false' to run test."); + } + + executeEmailServerTests = Boolean.parseBoolean( + AnetTestConfiguration.getConfiguration().get("emailServerTestsExecute").toString()); + + pendingAssessmentsNotificationWorker = + new PendingAssessmentsNotificationWorker(app.getConfiguration()); + emailWorker = + new AnetEmailWorker(app.getConfiguration(), AnetObjectEngine.getInstance().getEmailDao()); + emailServer = new FakeSmtpServer(app.getConfiguration().getSmtp()); + + // Flush all assessment notifications + pendingAssessmentsNotificationWorker.run(); + // Clear the email server before starting testing + flushEmail(); + } + + @AfterAll + public static void tearDownClass() throws Exception { + // Clear the email server after testing + flushEmail(); + + emailWorker = null; + AnetEmailWorker.setInstance(null); + } + + @Test + public void testGetAssessmentDates() { + final Object[][] testData = new Object[][] { + // each item has: { test date, recurrence, expected assessment date, + // expected notification date, expected reminder date } + // daily tests + {"2004-02-28T23:59:59.999Z", Recurrence.DAILY, "2004-02-27T00:00:00.000Z", + "2004-02-28T00:00:00.000Z", null}, + {"2004-02-29T00:00:00.000Z", Recurrence.DAILY, "2004-02-28T00:00:00.000Z", + "2004-02-29T00:00:00.000Z", null}, + {"2004-02-29T23:59:59.999Z", Recurrence.DAILY, "2004-02-28T00:00:00.000Z", + "2004-02-29T00:00:00.000Z", null}, + {"2004-03-01T00:00:00.000Z", Recurrence.DAILY, "2004-02-29T00:00:00.000Z", + "2004-03-01T00:00:00.000Z", null}, + // weekly tests + {"2004-02-28T23:59:59.999Z", Recurrence.WEEKLY, "2004-02-16T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-26T00:00:00.000Z"}, + {"2004-02-29T00:00:00.000Z", Recurrence.WEEKLY, "2004-02-16T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-26T00:00:00.000Z"}, + {"2004-02-29T23:59:59.999Z", Recurrence.WEEKLY, "2004-02-16T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-26T00:00:00.000Z"}, + {"2004-03-01T00:00:00.000Z", Recurrence.WEEKLY, "2004-02-23T00:00:00.000Z", + "2004-03-01T00:00:00.000Z", "2004-03-04T00:00:00.000Z"}, + // biweekly tests + {"2004-02-28T23:59:59.999Z", Recurrence.BIWEEKLY, "2004-02-09T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-28T00:00:00.000Z"}, + {"2004-02-29T00:00:00.000Z", Recurrence.BIWEEKLY, "2004-02-09T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-28T00:00:00.000Z"}, + {"2004-02-29T23:59:59.999Z", Recurrence.BIWEEKLY, "2004-02-09T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-28T00:00:00.000Z"}, + {"2004-03-01T00:00:00.000Z", Recurrence.BIWEEKLY, "2004-02-09T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-28T00:00:00.000Z"}, + {"2004-03-07T23:59:59.999Z", Recurrence.BIWEEKLY, "2004-02-09T00:00:00.000Z", + "2004-02-23T00:00:00.000Z", "2004-02-28T00:00:00.000Z"}, + {"2004-03-08T00:00:00.000Z", Recurrence.BIWEEKLY, "2004-02-23T00:00:00.000Z", + "2004-03-08T00:00:00.000Z", "2004-03-13T00:00:00.000Z"}, + {"2021-01-03T23:59:59.999Z", Recurrence.BIWEEKLY, "2020-12-07T00:00:00.000Z", + "2020-12-21T00:00:00.000Z", "2020-12-26T00:00:00.000Z"}, + {"2021-01-04T00:00:00.000Z", Recurrence.BIWEEKLY, "2020-12-21T00:00:00.000Z", + "2021-01-04T00:00:00.000Z", "2021-01-09T00:00:00.000Z"}, + {"2021-01-18T00:00:00.000Z", Recurrence.BIWEEKLY, "2021-01-04T00:00:00.000Z", + "2021-01-18T00:00:00.000Z", "2021-01-23T00:00:00.000Z"}, + {"2021-01-24T23:59:59.999Z", Recurrence.BIWEEKLY, "2021-01-04T00:00:00.000Z", + "2021-01-18T00:00:00.000Z", "2021-01-23T00:00:00.000Z"}, + {"2021-01-25T00:00:00.000Z", Recurrence.BIWEEKLY, "2021-01-04T00:00:00.000Z", + "2021-01-18T00:00:00.000Z", "2021-01-23T00:00:00.000Z"}, + {"2021-01-31T23:59:59.999Z", Recurrence.BIWEEKLY, "2021-01-04T00:00:00.000Z", + "2021-01-18T00:00:00.000Z", "2021-01-23T00:00:00.000Z"}, + {"2021-02-01T00:00:00.000Z", Recurrence.BIWEEKLY, "2021-01-18T00:00:00.000Z", + "2021-02-01T00:00:00.000Z", "2021-02-06T00:00:00.000Z"}, + // semimonthly tests + {"2004-02-28T23:59:59.999Z", Recurrence.SEMIMONTHLY, "2004-02-01T00:00:00.000Z", + "2004-02-15T00:00:00.000Z", "2004-02-20T00:00:00.000Z"}, + {"2004-02-29T00:00:00.000Z", Recurrence.SEMIMONTHLY, "2004-02-01T00:00:00.000Z", + "2004-02-15T00:00:00.000Z", "2004-02-20T00:00:00.000Z"}, + {"2004-02-29T23:59:59.999Z", Recurrence.SEMIMONTHLY, "2004-02-01T00:00:00.000Z", + "2004-02-15T00:00:00.000Z", "2004-02-20T00:00:00.000Z"}, + {"2004-03-01T00:00:00.000Z", Recurrence.SEMIMONTHLY, "2004-02-15T00:00:00.000Z", + "2004-03-01T00:00:00.000Z", "2004-03-06T00:00:00.000Z"}, + // monthly tests + {"2004-02-28T23:59:59.999Z", Recurrence.MONTHLY, "2004-01-01T00:00:00.000Z", + "2004-02-01T00:00:00.000Z", "2004-02-08T00:00:00.000Z"}, + {"2004-02-29T00:00:00.000Z", Recurrence.MONTHLY, "2004-01-01T00:00:00.000Z", + "2004-02-01T00:00:00.000Z", "2004-02-08T00:00:00.000Z"}, + {"2004-02-29T23:59:59.999Z", Recurrence.MONTHLY, "2004-01-01T00:00:00.000Z", + "2004-02-01T00:00:00.000Z", "2004-02-08T00:00:00.000Z"}, + {"2004-03-01T00:00:00.000Z", Recurrence.MONTHLY, "2004-02-01T00:00:00.000Z", + "2004-03-01T00:00:00.000Z", "2004-03-08T00:00:00.000Z"}, + // quarterly tests + {"2004-02-29T00:00:00.000Z", Recurrence.QUARTERLY, "2003-10-01T00:00:00.000Z", + "2004-01-01T00:00:00.000Z", "2004-01-29T00:00:00.000Z"}, + {"2004-03-31T23:59:59.999Z", Recurrence.QUARTERLY, "2003-10-01T00:00:00.000Z", + "2004-01-01T00:00:00.000Z", "2004-01-29T00:00:00.000Z"}, + {"2004-04-01T00:00:00.000Z", Recurrence.QUARTERLY, "2004-01-01T00:00:00.000Z", + "2004-04-01T00:00:00.000Z", "2004-04-29T00:00:00.000Z"}, + // semiannually tests + {"2004-02-29T00:00:00.000Z", Recurrence.SEMIANNUALLY, "2003-07-01T00:00:00.000Z", + "2004-01-01T00:00:00.000Z", "2004-02-01T00:00:00.000Z"}, + {"2004-06-30T23:59:59.999Z", Recurrence.SEMIANNUALLY, "2003-07-01T00:00:00.000Z", + "2004-01-01T00:00:00.000Z", "2004-02-01T00:00:00.000Z"}, + {"2004-07-01T00:00:00.000Z", Recurrence.SEMIANNUALLY, "2004-01-01T00:00:00.000Z", + "2004-07-01T00:00:00.000Z", "2004-08-01T00:00:00.000Z"}, + // annually tests + {"2003-12-31T23:59:59.999Z", Recurrence.ANNUALLY, "2002-01-01T00:00:00.000Z", + "2003-01-01T00:00:00.000Z", "2003-02-01T00:00:00.000Z"}, + {"2004-01-01T00:00:00.000Z", Recurrence.ANNUALLY, "2003-01-01T00:00:00.000Z", + "2004-01-01T00:00:00.000Z", "2004-02-01T00:00:00.000Z"}, + {"2004-02-29T00:00:00.000Z", Recurrence.ANNUALLY, "2003-01-01T00:00:00.000Z", + "2004-01-01T00:00:00.000Z", "2004-02-01T00:00:00.000Z"}, + // end + }; + for (final Object[] testItem : testData) { + final AssessmentDates assessmentDates = + new AssessmentDates(toInstant(testItem[0]), (Recurrence) testItem[1]); + logger.debug("checking {} against {}", testItem, assessmentDates); + assertThat(assessmentDates.getAssessmentDate()).isEqualTo(toInstant(testItem[2])); + assertThat(assessmentDates.getNotificationDate()).isEqualTo(toInstant(testItem[3])); + assertThat(assessmentDates.getReminderDate()).isEqualTo(toInstant(testItem[4])); + } + } + + private Instant toInstant(final Object testDate) { + return testDate == null ? null : Instant.parse((String) testDate); + } + + @Test + public void testNoAssessments() { + // There should be no new pending assessments since the run in setUpClass() + testPendingAssessmentsNotificationWorker(0); + } + + @Test + public void testInitialDataAssessments() throws Exception { + final JobHistoryDao jobHistoryDao = AnetObjectEngine.getInstance().getJobHistoryDao(); + final JobHistory jobHistory = + jobHistoryDao.getByJobName(pendingAssessmentsNotificationWorker.getClass().getSimpleName()); + if (jobHistory != null) { + // Wipe out last run + jobHistory.setLastRun(Instant.EPOCH); + jobHistoryDao.update(jobHistory); + } + + // From our initial data we should get 6 pending assessments + final List emails = testPendingAssessmentsNotificationWorker(6); + // Check the actual emails + for (final AnetEmail email : emails) { + assertThat(email.getToAddresses()).hasSize(1); + final String to = email.getToAddresses().get(0); + final PendingAssessmentsNotificationEmail action = + (PendingAssessmentsNotificationEmail) email.getAction(); + switch (to.split("@")[0]) { + case "hunter+jack": + // Jack should assess task 2.B + assertAssessments(action, Collections.emptySet(), ImmutableSet.of("2.B")); + break; + case "hunter+erin": + // Erin should assess position Planning Captain + assertAssessments(action, ImmutableSet.of("Planning Captain"), Collections.emptySet()); + break; + case "hunter+henry": + // Henry should assess task 2.A + assertAssessments(action, Collections.emptySet(), ImmutableSet.of("2.A")); + break; + case "hunter+liz": + // Elizabeth should assess position Cost Adder - MoD and task 1.1.A + assertAssessments(action, ImmutableSet.of("Cost Adder - MoD"), ImmutableSet.of("1.1.A")); + break; + case "hunter+bob": + // Bob should assess task 1.1 + assertAssessments(action, Collections.emptySet(), ImmutableSet.of("1.1")); + break; + case "hunter+andrew": + // Andrew should assess tasks 1.1.A, 1.2.A and 1.2.B + assertAssessments(action, Collections.emptySet(), + ImmutableSet.of("1.1.A", "1.2.A", "1.2.B")); + break; + default: + fail("Unknown to address: " + to); + } + } + + // There should be no new pending assessments since the previous run + flushEmail(); + testPendingAssessmentsNotificationWorker(0); + } + + private void assertAssessments(final PendingAssessmentsNotificationEmail action, + final Set expectedPositions, final Set expectedTasks) { + assertThat(action.getPositionUuidsToAssess()).hasSize(expectedPositions.size()); + final PositionDao positionDao = AnetObjectEngine.getInstance().getPositionDao(); + final Set actualPositions = action.getPositionUuidsToAssess().stream() + .map(positionUuid -> positionDao.getByUuid(positionUuid).getName()) + .collect(Collectors.toSet()); + assertThat(actualPositions).isEqualTo(expectedPositions); + + assertThat(action.getTaskUuidsToAssess()).hasSize(expectedTasks.size()); + final TaskDao taskDao = AnetObjectEngine.getInstance().getTaskDao(); + final Set actualTasks = action.getTaskUuidsToAssess().stream() + .map(taskUuid -> taskDao.getByUuid(taskUuid).getShortName()).collect(Collectors.toSet()); + assertThat(actualTasks).isEqualTo(expectedTasks); + } + + private List testPendingAssessmentsNotificationWorker(final int expectedCount) { + final EmailDao emailDao = AnetObjectEngine.getInstance().getEmailDao(); + final int prevEmailSize = emailDao.getAll().size(); + pendingAssessmentsNotificationWorker.run(); + final List emails = emailDao.getAll(); + assertThat(emails).hasSize(prevEmailSize + expectedCount); + return emails.subList(prevEmailSize, emails.size()); + } + + private static void flushEmail() throws Exception { + assumeTrue(executeEmailServerTests, "Email server tests configured to be skipped."); + + // Flush all messages + emailWorker.run(); + // Clear the email server + emailServer.clearEmailServer(); + } + +}