Skip to content

Commit

Permalink
Merge pull request #327 from jembi/CU-86c02t3u3_Linker-Default-Min-an…
Browse files Browse the repository at this point in the history 
…d-Max-Threshold

Cu 86c02t3u3 linker default min and max threshold
  • Loading branch information
@MatthewErispe
MatthewErispe authored Sep 26, 2024
2 parents 8927d25 + 888fc26 commit c35289b
Show file tree
Hide file tree
Showing 9 changed files with 51 additions and 64 deletions.
2 changes: 1 addition & 1 deletion JeMPI_Apps/JeMPI_Linker/checkstyle/suppression.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
/>

<suppress
checks="(MethodLength)"
checks="(MethodLength|ParameterNumber)"
files="LinkerDWH.java"
/>

Expand Down
2 changes: 2 additions & 0 deletions JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ public final class AppConfig {
public static final String API_IP = CONFIG.getString("API_IP");
public static final String API_HTTP_PORT = CONFIG.getString("API_HTTP_PORT");
public static final Float LINKER_MATCH_THRESHOLD = (float) CONFIG.getDouble("LINKER_MATCH_THRESHOLD");
public static final Float LINKER_MIN_THRESHOLD = (float) CONFIG.getDouble("LINKER_MIN_THRESHOLD");
public static final Float LINKER_MAX_THRESHOLD = (float) CONFIG.getDouble("LINKER_MAX_THRESHOLD");
public static final Float LINKER_MATCH_THRESHOLD_MARGIN = (float) CONFIG.getDouble("LINKER_MATCH_THRESHOLD_MARGIN");
public static final Level GET_LOG_LEVEL = Level.toLevel(CONFIG.getString("LOG4J2_LEVEL"));

Expand Down
66 changes: 20 additions & 46 deletions JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/BackEnd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,9 @@ private Behavior<Request> syncLinkInteractionHandler(final SyncLinkInteractionRe
request.link.matchThreshold() == null
? AppConfig.LINKER_MATCH_THRESHOLD
: request.link.matchThreshold(),
request.link.externalLinkRange().low(),
request.link.externalLinkRange().high(),
AppConfig.LINKER_MATCH_THRESHOLD_MARGIN,
request.link.stan());
request.replyTo.tell(new SyncLinkInteractionResponse(request.link.stan(),
listLinkInfo.isRight()
Expand All @@ -209,14 +212,25 @@ private Behavior<Request> asyncLinkInteractionHandler(final AsyncLinkInteraction
return Behaviors.same();
});
}
final var uploadConfig = req.batchInteraction.sessionMetadata().commonMetaData().uploadConfig();
final var linkInfo =
LinkerDWH.linkInteraction(libMPI,
req.batchInteraction.interaction(),
null,
req.batchInteraction.sessionMetadata().commonMetaData().uploadConfig() != null
? req.batchInteraction.sessionMetadata().commonMetaData().uploadConfig().linkThreshold().floatValue()
: AppConfig.LINKER_MATCH_THRESHOLD,
req.batchInteraction.stan());
req.batchInteraction.interaction(),
null,
uploadConfig != null
? uploadConfig.linkThreshold().floatValue()
: AppConfig.LINKER_MATCH_THRESHOLD,
uploadConfig != null
? uploadConfig.minThreshold().floatValue()
: AppConfig.LINKER_MIN_THRESHOLD,
uploadConfig != null
? uploadConfig.maxThreshold().floatValue()
: AppConfig.LINKER_MAX_THRESHOLD,
uploadConfig != null
? uploadConfig.marginWindowSize().floatValue()
: AppConfig.LINKER_MATCH_THRESHOLD_MARGIN,
req.batchInteraction.stan());
req.batchInteraction.stan();
if (linkInfo.isRight()) {
req.replyTo.tell(new AsyncLinkInteractionResponse(linkInfo.get()));
} else {
Expand All @@ -235,46 +249,6 @@ private Behavior<Request> asyncLinkInteractionHandler(final AsyncLinkInteraction
});
}

/*
private Behavior<Request> syncLinkInteractionToGidHandler(final SyncLinkInteractionToGidRequest request) {
final LinkInfo linkInfo;
final var interaction = request.link.interaction();
final var gid = request.link.gid();
try {
// Check if we have new M&U values
LinkerProbabilistic.checkUpdatedMU();
libMPI.startTransaction();
if (StringUtils.isBlank(gid)) {
linkInfo = libMPI.createInteractionAndLinkToClonedGoldenRecord(interaction, 1.0F);
} else {
final var goldenRecord = libMPI.findGoldenRecord(gid);
if (goldenRecord == null) {
LOGGER.error("Golden Record for GID {} is null", gid);
linkInfo = null;
} else {
final var validated1 = CustomLinkerDeterministic.validateDeterministicMatch(goldenRecord.demographicData(),
interaction.demographicData());
final var validated2 = CustomLinkerProbabilistic.validateProbabilisticScore(goldenRecord.demographicData(),
interaction.demographicData());
linkInfo = libMPI.createInteractionAndLinkToExistingGoldenRecord(interaction,
new LibMPIClientInterface.GoldenIdScore(gid,
3.0F),
validated1, validated2);
if (Boolean.TRUE.equals(goldenRecord.customUniqueGoldenRecordData().auxAutoUpdateEnabled())) {
CustomLinkerBackEnd.updateGoldenRecordFields(libMPI, 0.0F, linkInfo.interactionUID(), gid);
}
}
}
} finally {
libMPI.closeTransaction();
}
request.replyTo.tell(new SyncLinkInteractionToGidResponse(request.link.stan(), linkInfo));
return Behaviors.same();
}
*/

private Behavior<Request> workTimeHandler(final WorkTimeRequest request) {
LOGGER.info("WORK TIME");
return Behaviors.same();
Expand Down
3 changes: 3 additions & 0 deletions JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerCR.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ private static Either<MpiGeneralError, LinkInfo> crLinkBySourceIdUpdate(
demographicData),
null,
AppConfig.LINKER_MATCH_THRESHOLD,
AppConfig.LINKER_MIN_THRESHOLD,
AppConfig.LINKER_MAX_THRESHOLD,
AppConfig.LINKER_MATCH_THRESHOLD_MARGIN,
"STAN");

return Either.right(linkInfo.get());
Expand Down
26 changes: 13 additions & 13 deletions JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,9 @@ static Either<List<ExternalLinkCandidate>, LinkInfo> linkInteraction(
final Interaction interaction,
final ExternalLinkRange externalLinkRange,
final float matchThreshold_,
final float minThreshold_,
final float maxThreshold_,
final float marginWindowSize_,
final String envelopStan) {

LinkStatsMeta.ConfusionMatrix confusionMatrix;
Expand All @@ -283,9 +286,6 @@ static Either<List<ExternalLinkCandidate>, LinkInfo> linkInteraction(
} else {
LinkInfo linkInfo = null;
final List<ExternalLinkCandidate> externalLinkCandidateList = new ArrayList<>();
final var matchThreshold = externalLinkRange != null
? externalLinkRange.high()
: matchThreshold_;
LinkerProbabilistic.checkUpdatedLinkMU();
final var candidateGoldenRecords = libMPI.findLinkCandidates(interaction.demographicData());
LOGGER.debug("{} : {}", envelopStan, candidateGoldenRecords.size());
Expand Down Expand Up @@ -313,17 +313,17 @@ static Either<List<ExternalLinkCandidate>, LinkInfo> linkInteraction(
.parallel()
.mapToObj(i -> {
final var workCandidate = allCandidateScores.get(i);
return FieldTallies.map(i == 0 && workCandidate.score >= matchThreshold,
return FieldTallies.map(i == 0 && workCandidate.score >= matchThreshold_,
interaction.demographicData(),
workCandidate.goldenRecord.demographicData());
})
.reduce(CUSTOM_FIELD_TALLIES_SUM_IDENTITY, FieldTallies::sum);
final var score = allCandidateScores.getFirst().score;
if (score >= matchThreshold + 0.1) {
if (score >= maxThreshold_) {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(1.0, 0.0, 0.0, 0.0);
} else if (score >= matchThreshold) {
} else if (score >= matchThreshold_) {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(0.80, 0.20, 0.0, 0.0);
} else if (score >= matchThreshold - 0.1) {
} else if (score >= minThreshold_) {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(0.0, 0.0, 0.20, 0.80);
} else {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(0.0, 0.0, 1.0, 0.0);
Expand All @@ -340,12 +340,12 @@ static Either<List<ExternalLinkCandidate>, LinkInfo> linkInteraction(
final var belowThresholdNotifications = new ArrayList<Notification.MatchData>();
final var aboveThresholdNotifications = new ArrayList<Notification.MatchData>();
final var candidatesAboveMatchThreshold = allCandidateScores.stream().peek(v -> {
if (v.score() > matchThreshold - 0.1 && v.score() < matchThreshold) {
if (v.score() > minThreshold_ && v.score() < matchThreshold_) {
belowThresholdNotifications.add(new Notification.MatchData(v.goldenRecord().goldenId(), v.score()));
} else if (v.score() >= matchThreshold && v.score() < matchThreshold + 0.1) {
} else if (v.score() >= matchThreshold_ && v.score() < maxThreshold_) {
aboveThresholdNotifications.add(new Notification.MatchData(v.goldenRecord().goldenId(), v.score()));
}
}).filter(v -> v.score() >= matchThreshold).collect(Collectors.toCollection(ArrayList::new));
}).filter(v -> v.score() >= matchThreshold_).collect(Collectors.toCollection(ArrayList::new));
if (candidatesAboveMatchThreshold.isEmpty()) {
if (candidatesInExternalLinkRange.isEmpty()) {
linkInfo = libMPI.createInteractionAndLinkToClonedGoldenRecord(interaction, 1.0F);
Expand Down Expand Up @@ -376,7 +376,7 @@ static Either<List<ExternalLinkCandidate>, LinkInfo> linkInteraction(
validated1,
validated2,
firstCandidate.linkingRule());
if (linkToGoldenId.score() <= matchThreshold + 0.1) {
if (linkToGoldenId.score() <= maxThreshold_) {
sendNotification(Notification.NotificationType.ABOVE_THRESHOLD,
linkInfo.interactionUID(),
patientName(interaction),
Expand All @@ -388,15 +388,15 @@ static Either<List<ExternalLinkCandidate>, LinkInfo> linkInteraction(
}
if (Boolean.TRUE.equals(firstCandidate.goldenRecord.auxGoldenRecordData().auxAutoUpdateEnabled())) {
updateGoldenRecordFields(libMPI,
matchThreshold,
matchThreshold_,
linkInfo.interactionUID(),
linkInfo.goldenUID());
}
final var marginCandidates = new ArrayList<Notification.MatchData>();
if (candidatesInExternalLinkRange.isEmpty() && candidatesAboveMatchThreshold.size() > 1) {
for (var i = 1; i < candidatesAboveMatchThreshold.size(); i++) {
final var candidate = candidatesAboveMatchThreshold.get(i);
if (firstCandidate.score - candidate.score <= 0.1) {
if (firstCandidate.score - candidate.score <= marginWindowSize_) {
marginCandidates.add(new Notification.MatchData(candidate.goldenRecord.goldenId(), candidate.score));
} else {
break;
Expand Down
4 changes: 3 additions & 1 deletion devops/linux/docker/conf/stack/docker-stack-high-0.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
LINKER_MATCH_THRESHOLD: 0.65
LINKER_MATCH_THRESHOLD: 0.7
LINKER_MIN_THRESHOLD: 0.65
LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend
Expand Down
4 changes: 3 additions & 1 deletion devops/linux/docker/conf/stack/docker-stack-high-1.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,7 +611,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
LINKER_MATCH_THRESHOLD: 0.65
LINKER_MATCH_THRESHOLD: 0.7
LINKER_MIN_THRESHOLD: 0.65
LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend
Expand Down
4 changes: 3 additions & 1 deletion devops/linux/docker/conf/stack/docker-stack-low-0.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
LINKER_MATCH_THRESHOLD: 0.65
LINKER_MATCH_THRESHOLD: 0.7
LINKER_MIN_THRESHOLD: 0.65
LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend
Expand Down
4 changes: 3 additions & 1 deletion devops/linux/docker/conf/stack/docker-stack-low-1.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
LINKER_MATCH_THRESHOLD: 0.65
LINKER_MATCH_THRESHOLD: 0.7
LINKER_MIN_THRESHOLD: 0.65
LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend
Expand Down

0 comments on commit c35289b

Please sign in to comment.