diff --git a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/resources/data/postgres/audit-schema.sql b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/resources/data/postgres/audit-schema.sql index aa3b4b7c1..e5476dda7 100644 --- a/JeMPI_Apps/JeMPI_Bootstrapper/src/main/resources/data/postgres/audit-schema.sql +++ b/JeMPI_Apps/JeMPI_Bootstrapper/src/main/resources/data/postgres/audit-schema.sql @@ -2,10 +2,9 @@ CREATE TABLE IF NOT EXISTS audit_trail ( id UUID NOT NULL DEFAULT gen_random_uuid(), insertedAt TIMESTAMP NOT NULL DEFAULT now(), createdAt TIMESTAMP NOT NULL, - interactionID VARCHAR(64), - goldenID VARCHAR(64), - event VARCHAR(256), + eventType VARCHAR(256), + eventData JSONB, CONSTRAINT PKEY_AUDIT_TRAIL PRIMARY KEY (id) ); -CREATE INDEX IF NOT EXISTS idx_gid ON audit_trail(goldenID); -CREATE INDEX IF NOT EXISTS idx_iid ON audit_trail(interactionID); +CREATE INDEX IF NOT EXISTS idx_eventdata ON audit_trail USING GIN (eventData jsonb_path_ops); +CREATE INDEX IF NOT EXISTS idx_eventtype ON audit_trail(eventType); \ No newline at end of file diff --git a/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/PsqlAuditTrail.java b/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/PsqlAuditTrail.java index 556eb3eb5..7f7508e68 100644 --- a/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/PsqlAuditTrail.java +++ b/JeMPI_Apps/JeMPI_Controller/src/main/java/org/jembi/jempi/controller/PsqlAuditTrail.java @@ -6,6 +6,7 @@ import org.jembi.jempi.shared.models.AuditEvent; import java.sql.SQLException; +import java.sql.Types; import java.util.Locale; import static org.jembi.jempi.shared.models.GlobalConstants.PSQL_TABLE_AUDIT_TRAIL; @@ -53,17 +54,17 @@ CONSTRAINT PKEY_AUDIT_TRAIL PRIMARY KEY (id) */ } - void addAuditEvent(final AuditEvent event) { + void addAuditEvent(final AuditEvent auditEvent) { psqlClient.connect(AppConfig.POSTGRESQL_AUDIT_DB); + try (var preparedStatement = psqlClient.prepareStatement(String.format(Locale.ROOT, """ - INSERT INTO %s (createdAt, interactionID, goldenID, event) - VALUES (?, ?, ?, ?); + INSERT INTO %s (createdAt, eventType, eventData) + VALUES (?, ?, ?::json); """, PSQL_TABLE_AUDIT_TRAIL) - .stripIndent())) { - preparedStatement.setTimestamp(1, event.createdAt()); - preparedStatement.setString(2, event.interactionID()); - preparedStatement.setString(3, event.goldenID()); - preparedStatement.setString(4, event.event()); + .stripIndent())) { + preparedStatement.setTimestamp(1, auditEvent.createdAt()); + preparedStatement.setString(2, auditEvent.eventType().name()); + preparedStatement.setObject(3, auditEvent.eventData(), Types.OTHER); preparedStatement.executeUpdate(); } catch (SQLException e) { LOGGER.error(e.getLocalizedMessage(), e); diff --git a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java index 5095587ca..82be94adf 100644 --- a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java +++ b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/BackEnd.java @@ -505,7 +505,7 @@ public record GetGoldenRecordAuditTrailRequest( String uid) implements Event { } - public record GetGoldenRecordAuditTrailResponse(List auditTrail) { + public record GetGoldenRecordAuditTrailResponse(List auditTrail) { } public record GetInteractionAuditTrailRequest( @@ -519,7 +519,7 @@ public record SQLDashboardDataRequest( ActorRef replyTo) implements Event { } - public record GetInteractionAuditTrailResponse(List auditTrail) { + public record GetInteractionAuditTrailResponse(List auditTrail) { } diff --git a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PsqlAuditTrail.java b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PsqlAuditTrail.java index cdc530246..81ec636a2 100644 --- a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PsqlAuditTrail.java +++ b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PsqlAuditTrail.java @@ -2,13 +2,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jembi.jempi.shared.models.AuditEvent; +import org.jembi.jempi.shared.models.ApiModels; +import org.jembi.jempi.shared.models.GlobalConstants; +import org.jembi.jempi.shared.models.LinkingAuditEventData; +import org.jembi.jempi.shared.utils.AuditTrailBridge; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; +import java.util.*; import static org.jembi.jempi.shared.models.GlobalConstants.PSQL_TABLE_AUDIT_TRAIL; @@ -25,24 +26,32 @@ final class PsqlAuditTrail { psqlClient = new PsqlClient(pgServer, pgPort, pgDatabase, pgUser, pgPassword); } - List goldenRecordAuditTrail(final String uid) { + List goldenRecordAuditTrail(final String uid) { psqlClient.connect(); - final var list = new ArrayList(); + final var list = new ArrayList(); try (PreparedStatement preparedStatement = psqlClient.prepareStatement(String.format(Locale.ROOT, - """ - SELECT * FROM %s where goldenID = ?; - """, - PSQL_TABLE_AUDIT_TRAIL) - .stripIndent())) { - preparedStatement.setString(1, uid); + "SELECT * FROM %s WHERE eventType = ? AND eventData ->> 'goldenID' = ?", + PSQL_TABLE_AUDIT_TRAIL))) { + preparedStatement.setString(1, GlobalConstants.AuditEventType.LINKING_EVENT.name()); + preparedStatement.setString(2, uid); ResultSet rs = preparedStatement.executeQuery(); while (rs.next()) { - final var insertedAt = rs.getTimestamp(2); - final var createdAt = rs.getTimestamp(3); - final var interactionID = rs.getString(4); - final var goldenID = rs.getString(5); - final var event = rs.getString(6); - list.add(new AuditEvent(createdAt, insertedAt, interactionID, goldenID, event)); + final var insertTime = rs.getString(2); + final var createdTime = rs.getString(3); + final var eventType = rs.getString(4); + final var eventData = rs.getString(5); + if (Objects.equals(eventType, GlobalConstants.AuditEventType.LINKING_EVENT.name())) { + LinkingAuditEventData deserializeEventData = AuditTrailBridge.getDeserializeEventData(eventData, LinkingAuditEventData.class); + list.add(new ApiModels.ApiAuditTrail.LinkingAuditEntry( + insertTime, + createdTime, + deserializeEventData.interaction_id(), + deserializeEventData.goldenID(), + deserializeEventData.message(), + deserializeEventData.score(), + deserializeEventData.linkingRule().name() + )); + } } } catch (Exception e) { LOGGER.error(e); @@ -50,24 +59,34 @@ List goldenRecordAuditTrail(final String uid) { return list; } - List interactionRecordAuditTrail(final String uid) { + List interactionRecordAuditTrail(final String uid) { psqlClient.connect(); - final var list = new ArrayList(); - try (PreparedStatement preparedStatement = psqlClient.prepareStatement(String.format(Locale.ROOT, - """ - SELECT * FROM %s where interactionID = ?; - """, - PSQL_TABLE_AUDIT_TRAIL) - .stripIndent())) { - preparedStatement.setString(1, uid); + final var list = new ArrayList(); + try (PreparedStatement preparedStatement = psqlClient.prepareStatement(String.format( + Locale.ROOT, + "SELECT * FROM %s WHERE eventType = ? AND eventData ->> 'interaction_id' = ?", + PSQL_TABLE_AUDIT_TRAIL))) { + preparedStatement.setString(1, GlobalConstants.AuditEventType.LINKING_EVENT.name()); + preparedStatement.setString(2, uid); ResultSet rs = preparedStatement.executeQuery(); while (rs.next()) { - final var insertedAt = rs.getTimestamp(2); - final var createdAt = rs.getTimestamp(3); - final var interactionID = rs.getString(4); - final var goldenID = rs.getString(5); - final var event = rs.getString(6); - list.add(new AuditEvent(createdAt, insertedAt, interactionID, goldenID, event)); + final var insertTime = rs.getString(2); + final var createdTime = rs.getString(3); + final var eventType = rs.getString(4); + final var eventData = rs.getString(5); + + if (Objects.equals(eventType, GlobalConstants.AuditEventType.LINKING_EVENT.name())) { + LinkingAuditEventData deserializeEventData = AuditTrailBridge.getDeserializeEventData(eventData, LinkingAuditEventData.class); + list.add(new ApiModels.ApiAuditTrail.LinkingAuditEntry( + insertTime, + createdTime, + deserializeEventData.interaction_id(), + deserializeEventData.goldenID(), + deserializeEventData.message(), + deserializeEventData.score(), + deserializeEventData.linkingRule().name() + )); + } } } catch (Exception e) { LOGGER.error(e); diff --git a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/Routes.java b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/Routes.java index 8c7f1e50e..debc8126b 100644 --- a/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/Routes.java +++ b/JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/Routes.java @@ -111,16 +111,12 @@ public static Route getGoldenRecordAuditTrail( final ActorRef backEnd) { return parameter("gid", uid -> onComplete(Ask.getGoldenRecordAuditTrail(actorSystem, backEnd, uid), - result -> { - if (!result.isSuccess()) { - LOGGER.warn("IM_A_TEAPOT"); - } - return result.isSuccess() - ? complete(StatusCodes.OK, - ApiModels.ApiAuditTrail.fromAuditTrail(result.get().auditTrail()), - JSON_MARSHALLER) - : complete(ApiModels.getHttpErrorResponse(GlobalConstants.IM_A_TEA_POT)); - })); + result -> result.isSuccess() + ? complete(StatusCodes.OK, + result.get().auditTrail(), + JSON_MARSHALLER) + : complete(ApiModels.getHttpErrorResponse(StatusCodes.IM_A_TEAPOT)))); + } public static Route getInteractionAuditTrail( @@ -128,16 +124,11 @@ public static Route getInteractionAuditTrail( final ActorRef backEnd) { return parameter("iid", uid -> onComplete(Ask.getInteractionAuditTrail(actorSystem, backEnd, uid), - result -> { - if (!result.isSuccess()) { - LOGGER.warn("IM_A_TEAPOT"); - } - return result.isSuccess() - ? complete(StatusCodes.OK, - ApiModels.ApiAuditTrail.fromAuditTrail(result.get().auditTrail()), - JSON_MARSHALLER) - : complete(ApiModels.getHttpErrorResponse(GlobalConstants.IM_A_TEA_POT)); - })); + result -> result.isSuccess() + ? complete(StatusCodes.OK, + result.get().auditTrail(), + JSON_MARSHALLER) + : complete(ApiModels.getHttpErrorResponse(StatusCodes.IM_A_TEAPOT)))); } public static Route patchIidNewGidLink( diff --git a/JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/LibMPI.java b/JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/LibMPI.java index 11a488f1e..5d36a9951 100644 --- a/JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/LibMPI.java +++ b/JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/LibMPI.java @@ -12,8 +12,7 @@ import org.jembi.jempi.shared.kafka.MyKafkaProducer; import org.jembi.jempi.shared.models.*; import org.jembi.jempi.shared.serdes.JsonPojoSerializer; - -import java.sql.Timestamp; +import org.jembi.jempi.shared.utils.AuditTrailBridge; import java.time.LocalDateTime; import java.util.List; import java.util.Locale; @@ -24,6 +23,7 @@ public final class LibMPI { private final LibMPIClientInterface client; private final MyKafkaProducer topicAuditEvents; private final HooksRunner hooksRunner; + private final AuditTrailBridge auditTrailUtil; public LibMPI( final Level level, @@ -38,6 +38,7 @@ public LibMPI( new JsonPojoSerializer<>(), kafkaClientId); client = new LibDgraph(level, host, port); + auditTrailUtil = new AuditTrailBridge(topicAuditEvents); hooksRunner = new HooksRunner(client); } @@ -54,25 +55,19 @@ public LibMPI( new JsonPojoSerializer<>(), kafkaClientId); client = new LibPostgresql(URL, USR, PSW); + auditTrailUtil = new AuditTrailBridge(topicAuditEvents); hooksRunner = new HooksRunner(client); } private void sendAuditEvent( - final String interactionID, - final String goldenID, - final String event) { - topicAuditEvents.produceAsync(goldenID, - new AuditEvent(new Timestamp(System.currentTimeMillis()), - null, - interactionID, - goldenID, - event), - ((metadata, exception) -> { - if (exception != null) { - LOGGER.error(exception.getLocalizedMessage(), exception); - } - })); + final String interactionID, + final String goldenID, + final String message, + final float score, + final LinkingRule linkingRule) { + LinkingAuditEventData linkingEvent = new LinkingAuditEventData(message, interactionID, goldenID, score, linkingRule); + auditTrailUtil.sendAuditEvent(GlobalConstants.AuditEventType.LINKING_EVENT, linkingEvent); } /* @@ -239,9 +234,9 @@ public boolean setScore( final float newScore) { final var result = client.setScore(interactionID, goldenID, newScore); if (result) { - sendAuditEvent(interactionID, goldenID, String.format(Locale.ROOT, "score: %.5f -> %.5f", oldScore, newScore)); + sendAuditEvent(interactionID, goldenID, String.format(Locale.ROOT, "score: %.5f -> %.5f", oldScore, newScore), newScore, LinkingRule.UNMATCHED); } else { - sendAuditEvent(interactionID, goldenID, String.format(Locale.ROOT, "set score error: %.5f -> %.5f", oldScore, newScore)); + sendAuditEvent(interactionID, goldenID, String.format(Locale.ROOT, "set score error: %.5f -> %.5f", oldScore, newScore), newScore, LinkingRule.UNMATCHED); } return result; @@ -263,11 +258,21 @@ public boolean updateGoldenRecordField( final String newValue) { final var result = client.updateGoldenRecordField(goldenId, fieldName, newValue); if (result) { - sendAuditEvent(interactionId, goldenId, String.format(Locale.ROOT, "%s: '%s' -> '%s'", fieldName, oldValue, newValue)); + sendAuditEvent(interactionId, goldenId, String.format(Locale.ROOT, + "%s: '%s' -> '%s'", + fieldName, + oldValue, + newValue), + -1.0F, + LinkingRule.UNMATCHED); } else { - sendAuditEvent(interactionId, - goldenId, - String.format(Locale.ROOT, "%s: error updating '%s' -> '%s'", fieldName, oldValue, newValue)); + sendAuditEvent(interactionId, goldenId, String.format(Locale.ROOT, + "%s: error updating '%s' -> '%s'", + fieldName, + oldValue, + newValue), + -1.0F, + LinkingRule.UNMATCHED); } return result; } @@ -284,11 +289,15 @@ public Either linkToNewGoldenRecord( "Interaction -> new GoldenID: old(%s) new(%s) [%f]", currentGoldenId, result.get().goldenUID(), - score)); + score), score, LinkingRule.UNMATCHED); } else { sendAuditEvent(interactionId, - currentGoldenId, - String.format(Locale.ROOT, "Interaction -> update GoldenID error: old(%s) [%f]", currentGoldenId, score)); + currentGoldenId, + String.format(Locale.ROOT, + "Interaction -> update GoldenID error: old(%s) [%f]", + currentGoldenId, score), + score, + LinkingRule.UNMATCHED); } return result; } @@ -306,7 +315,7 @@ public Either updateLink( "Interaction -> update GoldenID: old(%s) new(%s) [%f]", goldenID, newGoldenID, - score)); + score), score, LinkingRule.UNMATCHED); } else { sendAuditEvent(interactionID, newGoldenID, @@ -314,7 +323,7 @@ public Either updateLink( "Interaction -> update GoldenID error: old(%s) new(%s) [%f]", goldenID, newGoldenID, - score)); + score), score, LinkingRule.UNMATCHED); } return result; } @@ -323,7 +332,8 @@ public LinkInfo createInteractionAndLinkToExistingGoldenRecord( final Interaction interaction, final LibMPIClientInterface.GoldenIdScore goldenIdScore, final boolean deterministicValidation, - final float probabilisticValidation) { + final float probabilisticValidation, + final LinkingRule linkingRule) { final var result = client.createInteractionAndLinkToExistingGoldenRecord(interaction, goldenIdScore); if (result != null) { sendAuditEvent(result.interactionUID(), @@ -333,13 +343,13 @@ public LinkInfo createInteractionAndLinkToExistingGoldenRecord( + "Probabilistic(%.3f)", result.score(), deterministicValidation, - probabilisticValidation)); + probabilisticValidation), result.score(), linkingRule); } else { sendAuditEvent(interaction.interactionId(), goldenIdScore.goldenId(), String.format(Locale.ROOT, "Interaction -> error linking to existing GoldenRecord (%.5f)", - goldenIdScore.score())); + goldenIdScore.score()), goldenIdScore.score(), linkingRule); } return result; @@ -351,12 +361,16 @@ public LinkInfo createInteractionAndLinkToClonedGoldenRecord( final var result = client.createInteractionAndLinkToClonedGoldenRecord(interaction, score); if (result != null) { sendAuditEvent(result.interactionUID(), - result.goldenUID(), - String.format(Locale.ROOT, "Interaction -> New GoldenRecord (%f)", score)); + result.goldenUID(), + String.format(Locale.ROOT, + "Interaction -> New GoldenRecord (%f)", score), + score, LinkingRule.UNMATCHED); } else { sendAuditEvent(interaction.interactionId(), - null, - String.format(Locale.ROOT, "Interaction -> error linking to new GoldenRecord (%f)", score)); + null, + String.format(Locale.ROOT, + "Interaction -> error linking to new GoldenRecord (%f)", score), + score, LinkingRule.UNMATCHED); } return result; } diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/ApiModels.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/ApiModels.java index 978325c6a..ef3555bea 100644 --- a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/ApiModels.java +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/ApiModels.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; @@ -306,25 +305,18 @@ public record ApiNumberOfRecords( @JsonInclude(JsonInclude.Include.NON_NULL) public record ApiAuditTrail( - List entries) { - public static ApiAuditTrail fromAuditTrail(final List trail) { - final var apiDateFormat = new SimpleDateFormat(DATE_PATTERN); - return new ApiAuditTrail(trail.stream() - .map(x -> new AuditEntry(apiDateFormat.format(x.insertedAt()), - apiDateFormat.format(x.createdAt()), - x.interactionID(), - x.goldenID(), - x.event())) - .toList()); - } + List entries) { @JsonInclude(JsonInclude.Include.NON_NULL) - public record AuditEntry( - @JsonProperty("inserted_at") String insertedAt, - @JsonProperty("created_at") String createdAt, - @JsonProperty("interaction_id") String interactionId, - @JsonProperty("golden_id") String goldenId, - @JsonProperty("entry") String entry) { + public record LinkingAuditEntry( + @JsonProperty("inserted_at") String insertedAt, + @JsonProperty("created_at") String createdAt, + @JsonProperty("interaction_id") String interactionId, + @JsonProperty("golden_id") String goldenId, + @JsonProperty("entry") String entry, + @JsonProperty("score") Float score, + @JsonProperty("linking_rule") String linkingRule + ) { } } diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/AuditEvent.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/AuditEvent.java index 7faf964f6..65b7e2a38 100644 --- a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/AuditEvent.java +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/AuditEvent.java @@ -1,14 +1,11 @@ package org.jembi.jempi.shared.models; -import com.fasterxml.jackson.annotation.JsonInclude; - import java.sql.Timestamp; -@JsonInclude(JsonInclude.Include.NON_NULL) public record AuditEvent( Timestamp createdAt, Timestamp insertedAt, - String interactionID, - String goldenID, - String event) { + GlobalConstants.AuditEventType eventType, + String eventData +) { } diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java index fae967877..7a3e7b911 100644 --- a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/GlobalConstants.java @@ -77,8 +77,13 @@ public final class GlobalConstants { public static final String DEFAULT_LINKER_GLOBAL_STORE_NAME = "linker"; - public static final StatusCode IM_A_TEA_POT = StatusCodes.IM_A_TEAPOT; + + public enum AuditEventType { + LINKING_EVENT, + UNKNOWN_EVENT + } + private GlobalConstants() { } diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/LinkingAuditEventData.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/LinkingAuditEventData.java new file mode 100644 index 000000000..53a7190f5 --- /dev/null +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/LinkingAuditEventData.java @@ -0,0 +1,11 @@ +package org.jembi.jempi.shared.models; + + +public record LinkingAuditEventData( + String message, + String interaction_id, + String goldenID, + float score, + LinkingRule linkingRule +) { +} diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/LinkingRule.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/LinkingRule.java new file mode 100644 index 000000000..376034363 --- /dev/null +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/models/LinkingRule.java @@ -0,0 +1,7 @@ +package org.jembi.jempi.shared.models; + +public enum LinkingRule { + DETERMINISTIC, + PROBABILISTIC, + UNMATCHED +} diff --git a/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/utils/AuditTrailBridge.java b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/utils/AuditTrailBridge.java new file mode 100644 index 000000000..c6daafb9e --- /dev/null +++ b/JeMPI_Apps/JeMPI_LibShared/src/main/java/org/jembi/jempi/shared/utils/AuditTrailBridge.java @@ -0,0 +1,57 @@ +package org.jembi.jempi.shared.utils; + +import java.sql.Timestamp; +import java.util.UUID; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.jembi.jempi.shared.kafka.MyKafkaProducer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jembi.jempi.shared.models.AuditEvent; +import org.jembi.jempi.shared.models.GlobalConstants; + +import static org.jembi.jempi.shared.utils.AppUtils.OBJECT_MAPPER; + + +public final class AuditTrailBridge { + private MyKafkaProducer topicAuditEvents = null; + private static final Logger LOGGER = LogManager.getLogger(AuditTrailBridge.class); + + public AuditTrailBridge(final MyKafkaProducer topicAuditEvents) { + this.topicAuditEvents = topicAuditEvents; + } + + public void sendAuditEvent( + final GlobalConstants.AuditEventType eventType, + final T eventData) { + + + var auditEvent = new AuditEvent( + new Timestamp(System.currentTimeMillis()), + null, + eventType, + getSerializedEventData(eventData) + ); + LOGGER.info("Creating Audit Event {} ", auditEvent.toString()); + + topicAuditEvents.produceAsync(UUID.randomUUID().toString(), + auditEvent, + (metadata, exception) -> { + if (exception != null) { + LOGGER.error(exception.getMessage(), exception); + } + }); + } + + public static String getSerializedEventData(final T eventData) { + try { + return eventData != null ? OBJECT_MAPPER.writeValueAsString(eventData) : null; + } catch (JsonProcessingException e) { + LOGGER.error("Failed to serialize event data", e); + return null; + } + } + public static T getDeserializeEventData(final String eventData, final Class valueType) throws JsonProcessingException { + return OBJECT_MAPPER.readValue(eventData, valueType); + } +} diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java index 8c9be2157..6f4f4025d 100644 --- a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java +++ b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java @@ -160,8 +160,13 @@ public static Either> linkInteraction( final var workCandidate = candidates.parallelStream() .unordered() .map(candidate -> new WorkCandidate(candidate, - LinkerUtils.calcNormalizedScore(candidate.demographicData(), - interaction.demographicData()))) + LinkerUtils.calcNormalizedScore( + candidate.demographicData(), + interaction.demographicData()), + LinkerUtils.determineLinkingRule( + candidate.demographicData(), + interaction.demographicData()) + )) .sorted((o1, o2) -> Float.compare(o2.score(), o1.score())) .collect(Collectors.toCollection(ArrayList::new)) .getFirst(); @@ -201,7 +206,11 @@ public static Either> linkInteraction( .map(candidate -> new WorkCandidate(candidate, LinkerUtils.calcNormalizedScore( candidate.demographicData(), - interaction.demographicData()))) + interaction.demographicData()), + LinkerUtils.determineLinkingRule( + candidate.demographicData(), + interaction.demographicData()) + )) .sorted((o1, o2) -> Float.compare(o2.score(), o1.score())) .collect(Collectors.toCollection(ArrayList::new)); @@ -273,7 +282,8 @@ public static Either> linkInteraction( linkInfo = libMPI.createInteractionAndLinkToExistingGoldenRecord(interaction, linkToGoldenId, validated1, - validated2); + validated2, + firstCandidate.linkingRule()); if (linkToGoldenId.score() <= matchThreshold + 0.1) { sendNotification(Notification.NotificationType.ABOVE_THRESHOLD, @@ -352,7 +362,8 @@ private static Serializer linkStatsMetaSerializer() { public record WorkCandidate( GoldenRecord goldenRecord, - float score) { + float score, + LinkingRule linkingRule) { } } diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerUtils.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerUtils.java index 606458700..c1b7c7949 100644 --- a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerUtils.java +++ b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerUtils.java @@ -3,6 +3,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jembi.jempi.shared.models.CustomDemographicData; +import org.jembi.jempi.shared.models.LinkingRule; public final class LinkerUtils { @@ -19,4 +20,13 @@ public static float calcNormalizedScore( } return CustomLinkerProbabilistic.linkProbabilisticScore(goldenRecord, interaction); } + + public static LinkingRule determineLinkingRule( + final CustomDemographicData goldenRecord, + final CustomDemographicData interaction) { + if (CustomLinkerDeterministic.linkDeterministicMatch(goldenRecord, interaction)) { + return LinkingRule.DETERMINISTIC; + } + return LinkingRule.PROBABILISTIC; + } } diff --git a/JeMPI_Apps/JeMPI_UI/src/components/recordDetails/RecordDetails.tsx b/JeMPI_Apps/JeMPI_UI/src/components/recordDetails/RecordDetails.tsx index 60bee4b90..978a11b19 100644 --- a/JeMPI_Apps/JeMPI_UI/src/components/recordDetails/RecordDetails.tsx +++ b/JeMPI_Apps/JeMPI_UI/src/components/recordDetails/RecordDetails.tsx @@ -343,4 +343,4 @@ const RecordDetails = () => { ) } -export default RecordDetails +export default RecordDetails \ No newline at end of file diff --git a/JeMPI_Apps/JeMPI_UI/src/services/ApiClient.ts b/JeMPI_Apps/JeMPI_UI/src/services/ApiClient.ts index 4233f3674..64d48b95e 100644 --- a/JeMPI_Apps/JeMPI_UI/src/services/ApiClient.ts +++ b/JeMPI_Apps/JeMPI_UI/src/services/ApiClient.ts @@ -1,5 +1,5 @@ import { AxiosInstance, AxiosRequestConfig } from 'axios' -import { AuditTrailEntries } from '../types/AuditTrail' +import { AuditTrail } from '../types/AuditTrail' import { FieldChangeReq, Fields } from '../types/Fields' import { ApiSearchResponse, @@ -345,8 +345,8 @@ export class ApiClient { async getGoldenRecordAuditTrail(gid: string) { const { - data: { entries } - } = await this.client.get( + data + } = await this.client.get>( ROUTES.GET_GOLDEN_RECORD_AUDIT_TRAIL, { params: { @@ -354,13 +354,13 @@ export class ApiClient { } } ) - return entries + return data } async getInteractionAuditTrail(iid: string) { const { - data: { entries } - } = await this.client.get( + data + } = await this.client.get>( ROUTES.GET_INTERACTION_AUDIT_TRAIL, { params: { @@ -368,7 +368,7 @@ export class ApiClient { } } ) - return entries + return data } async validateOAuth(oauthParams: OAuthParams) { diff --git a/JeMPI_Apps/JeMPI_UI/src/services/mockData.ts b/JeMPI_Apps/JeMPI_UI/src/services/mockData.ts index 540a2943e..af46e4150 100644 --- a/JeMPI_Apps/JeMPI_UI/src/services/mockData.ts +++ b/JeMPI_Apps/JeMPI_UI/src/services/mockData.ts @@ -187,21 +187,27 @@ const auditTrail: AuditTrail[] = [ created_at: '2023-09-05 14:46:51.000988', interaction_id: '0x1627', golden_id: '0x1628', - entry: 'Interaction -> New GoldenRecord (1.000000)' + entry: 'Interaction -> New GoldenRecord (1.000000)', + score: 0, + linking_rule: "DETERMINISTIC" }, { inserted_at: '2023-09-05 14:46:51.000991', created_at: '2023-09-05 14:46:51.000988', interaction_id: '0x1627', golden_id: '0x1628', - entry: 'Interaction -> New GoldenRecord (1.000000)' + entry: 'Interaction -> New GoldenRecord (1.000000)', + score: 0, + linking_rule: "DETERMINISTIC" }, { inserted_at: '2023-09-05 14:46:51.000991', created_at: '2023-09-05 14:46:51.000988', interaction_id: '0x1627', golden_id: '0x1628', - entry: 'Interaction -> New GoldenRecord (1.000000)' + entry: 'Interaction -> New GoldenRecord (1.000000)', + score: 0, + linking_rule: "DETERMINISTIC" } ] diff --git a/JeMPI_Apps/JeMPI_UI/src/types/AuditTrail.ts b/JeMPI_Apps/JeMPI_UI/src/types/AuditTrail.ts index 953c5ced6..412b67c5b 100644 --- a/JeMPI_Apps/JeMPI_UI/src/types/AuditTrail.ts +++ b/JeMPI_Apps/JeMPI_UI/src/types/AuditTrail.ts @@ -4,8 +4,10 @@ export interface AuditTrail { interaction_id: string golden_id: string entry: string + score: number + linking_rule: string } export interface AuditTrailEntries { entries: Array -} +} \ No newline at end of file diff --git a/JeMPI_Apps/JeMPI_UI/src/utils/constants.ts b/JeMPI_Apps/JeMPI_UI/src/utils/constants.ts index 12b08fefd..d4910fd1c 100644 --- a/JeMPI_Apps/JeMPI_UI/src/utils/constants.ts +++ b/JeMPI_Apps/JeMPI_UI/src/utils/constants.ts @@ -92,5 +92,13 @@ export const AUDIT_TRAIL_COLUMNS: GridColDef[] = [ disableColumnMenu: true, headerClassName: 'super-app-theme--header', flex: 1 + }, + { + field: 'linking_rule', + headerName: 'Matching Type', + sortable: false, + disableColumnMenu: true, + headerClassName: 'super-app-theme--header', + flex: 1 } ]