From 6b26b0ef50901917e800d90585598b986d774734 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 31 Dec 2023 14:09:36 +0100 Subject: [PATCH 1/2] #4418 - Improve CAS loading during training - Pull CAS loading out of the training task class --- .../recommendation/tasks/LazyCasLoader.java | 158 ++++++++++++++++++ .../recommendation/tasks/TrainingTask.java | 124 ++------------ 2 files changed, 171 insertions(+), 111 deletions(-) create mode 100644 inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/LazyCasLoader.java diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/LazyCasLoader.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/LazyCasLoader.java new file mode 100644 index 00000000000..4004245c4ec --- /dev/null +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/LazyCasLoader.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.tasks; + +import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode.SHARED_READ_ONLY_ACCESS; +import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasUpgradeMode.AUTO_CAS_UPGRADE; +import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState.NEW; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import org.apache.commons.lang3.concurrent.ConcurrentException; +import org.apache.uima.cas.CAS; +import org.apache.uima.cas.Type; +import org.apache.uima.fit.util.CasUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState; +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.inception.documents.api.DocumentService; +import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; + +public class LazyCasLoader +{ + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final DocumentService documentService; + private final Project project; + private final String dataOwner; + + private List candidates; + + public LazyCasLoader(DocumentService aDocumentService, Project aProject, String aDataOwner) + { + documentService = aDocumentService; + project = aProject; + dataOwner = aDataOwner; + } + + public List getRelevantCasses(Recommender aRecommender) throws ConcurrentException + { + return get().stream() // + .filter(e -> !aRecommender.getStatesIgnoredForTraining().contains(e.state)) // + .map(e -> e.getCas()) // + .filter(Objects::nonNull) // + .filter(cas -> containsTargetTypeAndFeature(aRecommender, cas)) // + .toList(); + } + + private Collection get() + { + if (candidates == null) { + candidates = readCasses(); + } + return candidates; + } + + private List readCasses() + { + var casses = new ArrayList(); + + var allDocuments = documentService.listAllDocuments(project, dataOwner); + for (var entry : allDocuments.entrySet()) { + var sourceDocument = entry.getKey(); + var annotationDocument = entry.getValue(); + var state = annotationDocument != null ? annotationDocument.getState() : NEW; + + casses.add(new TrainingDocument(sourceDocument, dataOwner, state)); + } + + return casses; + } + + public int size() + { + return get().size(); + } + + private boolean containsTargetTypeAndFeature(Recommender aRecommender, CAS aCas) + { + Type type; + try { + type = CasUtil.getType(aCas, aRecommender.getLayer().getName()); + } + catch (IllegalArgumentException e) { + // If the CAS does not contain the target type at all, then it cannot contain any + // annotations of that type. + return false; + } + + if (type.getFeatureByBaseName(aRecommender.getFeature().getName()) == null) { + // If the CAS does not contain the target feature, then there won't be any training + // data. + return false; + } + + return !aCas.select(type).isEmpty(); + } + + private class TrainingDocument + { + private final SourceDocument document; + private final String user; + private final AnnotationDocumentState state; + + private boolean attemptedLoading = false; + private CAS _cas; + + TrainingDocument(SourceDocument aDocument, String aAnnotator, + AnnotationDocumentState aState) + { + document = aDocument; + user = aAnnotator; + state = aState; + } + + private CAS getCas() + { + if (attemptedLoading) { + return _cas; + } + + attemptedLoading = true; + try { + // During training, we should not have to modify the CASes... right? Fingers + // crossed. + _cas = documentService.readAnnotationCas(document, user, AUTO_CAS_UPGRADE, + SHARED_READ_ONLY_ACCESS); + } + catch (IOException e) { + LOG.error("Unable to load CAS to train recommender", e); + } + + return _cas; + } + } +} diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java index 8c1413f0510..81d2577d920 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java @@ -17,32 +17,22 @@ */ package de.tudarmstadt.ukp.inception.recommendation.tasks; -import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode.SHARED_READ_ONLY_ACCESS; -import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasUpgradeMode.AUTO_CAS_UPGRADE; import static de.tudarmstadt.ukp.inception.recommendation.api.recommender.TrainingCapability.TRAINING_NOT_SUPPORTED; import static de.tudarmstadt.ukp.inception.recommendation.api.recommender.TrainingCapability.TRAINING_REQUIRED; import static java.lang.System.currentTimeMillis; -import static java.util.stream.Collectors.toList; import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import java.util.Objects; import javax.persistence.NoResultException; import org.apache.commons.lang3.concurrent.ConcurrentException; -import org.apache.commons.lang3.concurrent.LazyInitializer; import org.apache.uima.cas.CAS; -import org.apache.uima.cas.Type; -import org.apache.uima.fit.util.CasUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState; import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; import de.tudarmstadt.ukp.clarin.webanno.model.Project; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; @@ -126,14 +116,7 @@ private void executeTraining() // Read the CASes only when they are accessed the first time. This allows us to skip // reading the CASes in case that no layer / recommender is available or if no // recommender requires evaluation. - var casLoader = new LazyInitializer>() - { - @Override - protected List initialize() - { - return readCasses(getProject(), dataOwner); - } - }; + var casLoader = new LazyCasLoader(documentService, getProject(), dataOwner); boolean seenSuccessfulTraining = false; boolean seenNonTrainingRecommender = false; @@ -204,13 +187,7 @@ protected List initialize() continue; } - var trainingCasses = casLoader.get().stream() // - .filter(e -> !recommender.getStatesIgnoredForTraining() - .contains(e.state)) // - .map(e -> e.getCas()) // - .filter(Objects::nonNull) - .filter(cas -> containsTargetTypeAndFeature(recommender, cas)) // - .collect(toList()); + var trainingCasses = casLoader.getRelevantCasses(recommender); // If no data for training is available, but the engine requires training, // do not mark as ready @@ -234,7 +211,7 @@ protected List initialize() long duration = currentTimeMillis() - startTime; if (!engine.isReadyForPrediction(ctx)) { - logTrainingFailure(sessionOwner, recommender, duration, casLoader.get(), + logTrainingFailure(sessionOwner, recommender, duration, casLoader, trainingCasses); continue; } @@ -282,42 +259,6 @@ private void commitContext(User user, Recommender recommender, RecommenderContex recommenderService.putContext(user, recommender, ctx); } - private List readCasses(Project aProject, String aUser) - { - var casses = new ArrayList(); - var allDocuments = documentService.listAllDocuments(aProject, aUser); - for (var entry : allDocuments.entrySet()) { - var sourceDocument = entry.getKey(); - var annotationDocument = entry.getValue(); - var state = annotationDocument != null ? annotationDocument.getState() - : AnnotationDocumentState.NEW; - - casses.add(new TrainingDocument(sourceDocument, aUser, state)); - } - return casses; - } - - private boolean containsTargetTypeAndFeature(Recommender aRecommender, CAS aCas) - { - Type type; - try { - type = CasUtil.getType(aCas, aRecommender.getLayer().getName()); - } - catch (IllegalArgumentException e) { - // If the CAS does not contain the target type at all, then it cannot contain any - // annotations of that type. - return false; - } - - if (type.getFeatureByBaseName(aRecommender.getFeature().getName()) == null) { - // If the CAS does not contain the target feature, then there won't be any training - // data. - return false; - } - - return !aCas.select(type).isEmpty(); - } - private void logUnsupportedRecommenderType(User user, EvaluatedRecommender evaluatedRecommender) { log.warn("[{}][{}]: No factory found - skipping recommender", user.getUsername(), @@ -382,9 +323,9 @@ private void logNoDataAvailableForTraining(User user, AnnotationLayer layer, } private void logTrainingFailure(User user, Recommender recommender, long duration, - List aAllCasses, List aTrainCasses) + LazyCasLoader aLoader, List aTrainCasses) { - int docNum = aAllCasses.size(); + int docNum = aLoader.size(); int trainDocNum = aTrainCasses.size(); log.debug("[{}][{}][{}]: Training on [{}] out of [{}] documents not successful ({} ms)", @@ -399,16 +340,16 @@ private void logTrainingFailure(User user, Recommender recommender, long duratio // recommender)); } - private void logTrainingSuccessful(User user, LazyInitializer> casses, - Recommender recommender, List cassesForTraining, long duration) + private void logTrainingSuccessful(User user, LazyCasLoader casses, Recommender recommender, + List cassesForTraining, long duration) throws ConcurrentException { log.debug("[{}][{}][{}]: Training successful on [{}] out of [{}] documents ({} ms)", getId(), user.getUsername(), recommender.getName(), cassesForTraining.size(), - casses.get().size(), duration); + casses.size(), duration); log(LogMessage.info(recommender.getName(), "Training successful on [%d] out of [%d] documents (%d ms)", - cassesForTraining.size(), casses.get().size(), duration)); + cassesForTraining.size(), casses.size(), duration)); } private void logTrainingOverallStart(User user) @@ -418,18 +359,17 @@ private void logTrainingOverallStart(User user) info("Starting training triggered by [%s]...", getTrigger()); } - private void logTrainingRecommenderStart(User user, - LazyInitializer> casses, AnnotationLayer layer, - Recommender recommender, List cassesForTraining) + private void logTrainingRecommenderStart(User user, LazyCasLoader aLoader, + AnnotationLayer layer, Recommender recommender, List cassesForTraining) throws ConcurrentException { getMonitor().addMessage(LogMessage.info(this, "%s", recommender.getName())); log.debug("[{}][{}][{}]: Training model on [{}] out of [{}] documents ...", getId(), user.getUsername(), recommender.getName(), cassesForTraining.size(), - casses.get().size()); + aLoader.size()); log(LogMessage.info(recommender.getName(), "Training model for [%s] on [%d] out of [%d] documents ...", layer.getUiName(), - cassesForTraining.size(), casses.get().size())); + cassesForTraining.size(), aLoader.size())); } private void handleError(User user, Recommender recommender, long startTime, Throwable e) @@ -447,42 +387,4 @@ private void handleError(User user, Recommender recommender, long startTime, Thr duration, e.getMessage())) .build()); } - - private class TrainingDocument - { - private final SourceDocument document; - private final String user; - private final AnnotationDocumentState state; - - private boolean attemptedLoading = false; - private CAS _cas; - - private TrainingDocument(SourceDocument aDocument, String aAnnotator, - AnnotationDocumentState aState) - { - document = aDocument; - user = aAnnotator; - state = aState; - } - - public CAS getCas() - { - if (attemptedLoading) { - return _cas; - } - - attemptedLoading = true; - try { - // During training, we should not have to modify the CASes... right? Fingers - // crossed. - _cas = documentService.readAnnotationCas(document, user, AUTO_CAS_UPGRADE, - SHARED_READ_ONLY_ACCESS); - } - catch (IOException e) { - log.error("Unable to load CAS to train recommender", e); - } - - return _cas; - } - } } From f251559bb7a79054f572c31f92a8b25610c6df0f Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 31 Dec 2023 18:18:49 +0100 Subject: [PATCH 2/2] #4418 - Improve CAS loading during training - Pull prediction code out of the recommender service and into the prediction task --- .../api/RecommendationService.java | 40 - .../api/recommender/PredictionContext.java | 12 - .../RecommenderServiceAutoConfiguration.java | 8 +- .../service/RecommendationServiceImpl.java | 576 +------------ .../recommendation/tasks/PredictionTask.java | 778 +++++++++++++++++- .../recommendation/tasks/TrainingTask.java | 396 +++++---- ...ommendationServiceImplIntegrationTest.java | 48 +- .../RecommendationServiceImplTest.java | 114 --- .../tasks/PredictionTaskTest.java | 169 ++++ 9 files changed, 1143 insertions(+), 998 deletions(-) delete mode 100644 inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java create mode 100644 inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTaskTest.java diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/RecommendationService.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/RecommendationService.java index dcd73f818c1..2a983f943ce 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/RecommendationService.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/RecommendationService.java @@ -43,7 +43,6 @@ import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactory; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommenderContext; -import de.tudarmstadt.ukp.inception.scheduling.TaskMonitor; import de.tudarmstadt.ukp.inception.schema.api.adapter.AnnotationException; import de.tudarmstadt.ukp.inception.support.logging.LogMessageGroup; @@ -250,45 +249,6 @@ void skipSuggestion(String aSessionOwner, SourceDocument aDocument, String aData AnnotationSuggestion aSuggestion, LearningRecordChangeLocation aLocation) throws AnnotationException; - /** - * Compute predictions. - * - * @param aSessionOwner - * the user to compute the predictions for. - * @param aProject - * the project to compute the predictions for. - * @param aDocuments - * the documents to compute the predictions for. - * @param aDataOwner - * the owner of the annotations. - * @return the new predictions. - */ - Predictions computePredictions(User aSessionOwner, Project aProject, - List aDocuments, String aDataOwner, TaskMonitor aMonitor); - - /** - * Compute predictions. - * - * @param aSessionOwner - * the user to compute the predictions for. - * @param aProject - * the project to compute the predictions for. - * @param aCurrentDocument - * the document to compute the predictions for. - * @param aDataOwner - * the owner of the annotations. - * @param aInherit - * any documents for which to inherit the predictions from a previous run - * @param aPredictionBegin - * begin of the prediction range (negative to predict from 0) - * @param aPredictionEnd - * end of the prediction range (negative to predict until the end of the document) - * @return the new predictions. - */ - Predictions computePredictions(User aSessionOwner, Project aProject, - SourceDocument aCurrentDocument, String aDataOwner, List aInherit, - int aPredictionBegin, int aPredictionEnd, TaskMonitor aMonitor); - /** * Determine the visibility of suggestions. * diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/recommender/PredictionContext.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/recommender/PredictionContext.java index c509be7da93..54206ef0115 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/recommender/PredictionContext.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/recommender/PredictionContext.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.Optional; -import de.tudarmstadt.ukp.clarin.webanno.security.model.User; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommenderContext.Key; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -31,7 +30,6 @@ public class PredictionContext private RecommenderContext modelContext; private List messages; private boolean closed = false; - private Optional user; public PredictionContext(RecommenderContext aCtx) { @@ -76,14 +74,4 @@ synchronized public boolean isClosed() { return closed; } - - public Optional getUser() - { - return user; - } - - public void setUser(User aUser) - { - user = Optional.ofNullable(aUser); - } } diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/config/RecommenderServiceAutoConfiguration.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/config/RecommenderServiceAutoConfiguration.java index 07619edef90..037ad75b741 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/config/RecommenderServiceAutoConfiguration.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/config/RecommenderServiceAutoConfiguration.java @@ -80,14 +80,12 @@ public RecommendationService recommendationService(PreferencesService aPreferenc SessionRegistry aSessionRegistry, UserDao aUserRepository, RecommenderFactoryRegistry aRecommenderFactoryRegistry, SchedulingService aSchedulingService, AnnotationSchemaService aAnnoService, - DocumentService aDocumentService, ProjectService aProjectService, - ApplicationEventPublisher aApplicationEventPublisher, + ProjectService aProjectService, ApplicationEventPublisher aApplicationEventPublisher, SuggestionSupportRegistry aLayerRecommendtionSupportRegistry) { return new RecommendationServiceImpl(aPreferencesService, aSessionRegistry, aUserRepository, - aRecommenderFactoryRegistry, aSchedulingService, aAnnoService, aDocumentService, - aProjectService, entityManager, aApplicationEventPublisher, - aLayerRecommendtionSupportRegistry); + aRecommenderFactoryRegistry, aSchedulingService, aAnnoService, aProjectService, + entityManager, aApplicationEventPublisher, aLayerRecommendtionSupportRegistry); } @Bean diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java index 16cb905ff85..dd29699deac 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java @@ -17,22 +17,13 @@ */ package de.tudarmstadt.ukp.inception.recommendation.service; -import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode.EXCLUSIVE_WRITE_ACCESS; -import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode.SHARED_READ_ONLY_ACCESS; -import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasUpgradeMode.AUTO_CAS_UPGRADE; import static de.tudarmstadt.ukp.inception.recommendation.api.model.AutoAcceptMode.ON_FIRST_ACCESS; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordChangeLocation.AUTO_ACCEPT; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordUserAction.ACCEPTED; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordUserAction.CORRECTED; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordUserAction.REJECTED; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordUserAction.SKIPPED; -import static de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionCapability.PREDICTION_USES_TEXT_ONLY; -import static de.tudarmstadt.ukp.inception.recommendation.api.recommender.TrainingCapability.TRAINING_NOT_SUPPORTED; -import static de.tudarmstadt.ukp.inception.rendering.model.Range.rangeCoveringDocument; -import static java.util.Collections.emptyList; import static java.util.function.Function.identity; -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import java.io.IOException; @@ -43,7 +34,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -64,11 +54,9 @@ import org.apache.commons.lang3.Validate; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.uima.UIMAException; import org.apache.uima.cas.AnnotationBaseFS; import org.apache.uima.cas.CAS; import org.apache.uima.cas.text.AnnotationFS; -import org.apache.uima.resource.ResourceInitializationException; import org.apache.wicket.MetaDataKey; import org.apache.wicket.ajax.AjaxRequestTarget; import org.apache.wicket.request.cycle.IRequestCycleListener; @@ -98,8 +86,6 @@ import de.tudarmstadt.ukp.clarin.webanno.ui.annotation.AnnotationPage; import de.tudarmstadt.ukp.inception.annotation.events.AnnotationEvent; import de.tudarmstadt.ukp.inception.annotation.events.DocumentOpenedEvent; -import de.tudarmstadt.ukp.inception.annotation.storage.CasStorageSession; -import de.tudarmstadt.ukp.inception.documents.api.DocumentService; import de.tudarmstadt.ukp.inception.documents.event.AfterCasWrittenEvent; import de.tudarmstadt.ukp.inception.documents.event.AfterDocumentCreatedEvent; import de.tudarmstadt.ukp.inception.documents.event.AfterDocumentResetEvent; @@ -111,7 +97,6 @@ import de.tudarmstadt.ukp.inception.recommendation.api.LearningRecordService; import de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService; import de.tudarmstadt.ukp.inception.recommendation.api.RecommenderFactoryRegistry; -import de.tudarmstadt.ukp.inception.recommendation.api.RecommenderTypeSystemUtils; import de.tudarmstadt.ukp.inception.recommendation.api.SuggestionSupport; import de.tudarmstadt.ukp.inception.recommendation.api.SuggestionSupportRegistry; import de.tudarmstadt.ukp.inception.recommendation.api.model.AnnotationSuggestion; @@ -126,33 +111,23 @@ import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender_; import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion; -import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionDocumentGroup; import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.ExtractionContext; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactory; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommenderContext; import de.tudarmstadt.ukp.inception.recommendation.config.RecommenderServiceAutoConfiguration; import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderDeletedEvent; -import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderTaskNotificationEvent; import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderUpdatedEvent; import de.tudarmstadt.ukp.inception.recommendation.model.DirtySpot; import de.tudarmstadt.ukp.inception.recommendation.tasks.NonTrainableRecommenderActivationTask; import de.tudarmstadt.ukp.inception.recommendation.tasks.PredictionTask; import de.tudarmstadt.ukp.inception.recommendation.tasks.SelectionTask; import de.tudarmstadt.ukp.inception.recommendation.tasks.TrainingTask; -import de.tudarmstadt.ukp.inception.rendering.model.Range; import de.tudarmstadt.ukp.inception.scheduling.SchedulingService; -import de.tudarmstadt.ukp.inception.scheduling.TaskMonitor; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.schema.api.adapter.AnnotationException; import de.tudarmstadt.ukp.inception.schema.api.event.LayerConfigurationChangedEvent; -import de.tudarmstadt.ukp.inception.support.StopWatch; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; import de.tudarmstadt.ukp.inception.support.logging.LogMessageGroup; -import de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil; import de.tudarmstadt.ukp.inception.support.wicket.WicketExceptionUtil; /** @@ -169,8 +144,6 @@ public class RecommendationServiceImpl private static final int TRAININGS_PER_SELECTION = 5; - private static final String PREDICTION_CAS = "predictionCas"; - private final EntityManager entityManager; private final SessionRegistry sessionRegistry; @@ -178,7 +151,6 @@ public class RecommendationServiceImpl private final RecommenderFactoryRegistry recommenderFactoryRegistry; private final SchedulingService schedulingService; private final AnnotationSchemaService schemaService; - private final DocumentService documentService; private final ProjectService projectService; private final ApplicationEventPublisher applicationEventPublisher; private final PreferencesService preferencesService; @@ -210,8 +182,8 @@ public RecommendationServiceImpl(PreferencesService aPreferencesService, SessionRegistry aSessionRegistry, UserDao aUserRepository, RecommenderFactoryRegistry aRecommenderFactoryRegistry, SchedulingService aSchedulingService, AnnotationSchemaService aAnnoService, - DocumentService aDocumentService, ProjectService aProjectService, - EntityManager aEntityManager, ApplicationEventPublisher aApplicationEventPublisher, + ProjectService aProjectService, EntityManager aEntityManager, + ApplicationEventPublisher aApplicationEventPublisher, SuggestionSupportRegistry aLayerRecommendtionSupportRegistry) { preferencesService = aPreferencesService; @@ -220,7 +192,6 @@ public RecommendationServiceImpl(PreferencesService aPreferencesService, recommenderFactoryRegistry = aRecommenderFactoryRegistry; schedulingService = aSchedulingService; schemaService = aAnnoService; - documentService = aDocumentService; projectService = aProjectService; entityManager = aEntityManager; applicationEventPublisher = aApplicationEventPublisher; @@ -234,13 +205,12 @@ public RecommendationServiceImpl(PreferencesService aPreferencesService, SessionRegistry aSessionRegistry, UserDao aUserRepository, RecommenderFactoryRegistry aRecommenderFactoryRegistry, SchedulingService aSchedulingService, AnnotationSchemaService aAnnoService, - DocumentService aDocumentService, SuggestionSupportRegistry aLayerRecommendtionSupportRegistry, EntityManager aEntityManager) { this(aPreferencesService, aSessionRegistry, aUserRepository, aRecommenderFactoryRegistry, - aSchedulingService, aAnnoService, aDocumentService, (ProjectService) null, - aEntityManager, null, aLayerRecommendtionSupportRegistry); + aSchedulingService, aAnnoService, (ProjectService) null, aEntityManager, null, + aLayerRecommendtionSupportRegistry); } @Override @@ -1042,7 +1012,6 @@ public void putContext(User aUser, Recommender aRecommender, RecommenderContext } } - @Deprecated @Override @Transactional public AnnotationFS correctSuggestion(String aSessionOwner, SourceDocument aDocument, @@ -1365,10 +1334,9 @@ public void removePredictions(Recommender aRecommender) // Remove from evaluatedRecommenders map. // We have to do this, otherwise training and prediction continues for the // recommender when a new task is triggered. - MultiValuedMap newEvaluatedRecommenders = // - new HashSetValuedHashMap<>(); - MapIterator it = evaluatedRecommenders - .mapIterator(); + var newEvaluatedRecommenders = // + new HashSetValuedHashMap(); + var it = evaluatedRecommenders.mapIterator(); while (it.hasNext()) { AnnotationLayer layer = it.next(); @@ -1423,459 +1391,6 @@ public void removeLearningRecords(LearningRecord aRecord) } } - private void computePredictions(LazyCas aOriginalCas, - EvaluatedRecommender aEvaluatedRecommender, Predictions activePredictions, - Predictions aPredictions, CAS predictionCas, SourceDocument aDocument, - User aSessionOwner, int aPredictionBegin, int aPredictionEnd) - throws IOException - { - var project = aDocument.getProject(); - var predictionBegin = aPredictionBegin; - var predictionEnd = aPredictionEnd; - - // Make sure we have the latest recommender config from the DB - the one - // from the active recommenders list may be outdated - var recommender = aEvaluatedRecommender.getRecommender(); - try { - recommender = getRecommender(recommender.getId()); - } - catch (NoResultException e) { - aPredictions.log(LogMessage.info(recommender.getName(), - "Recommender no longer available... skipping")); - LOG.info("{}[{}]: Recommender no longer available... skipping", aSessionOwner, - recommender.getName()); - return; - } - - if (!recommender.isEnabled()) { - aPredictions.log( - LogMessage.info(recommender.getName(), "Recommender disabled... skipping")); - LOG.debug("{}[{}]: Disabled - skipping", aSessionOwner, recommender.getName()); - return; - } - - var context = getContext(aSessionOwner.getUsername(), recommender); - - if (!context.isPresent()) { - aPredictions.log(LogMessage.info(recommender.getName(), - "Recommender has no context... skipping")); - LOG.info("No context available for recommender {} for user {} on document {} in " // - + "project {} - skipping recommender", recommender, aSessionOwner, aDocument, - aDocument.getProject()); - return; - } - - var maybeFactory = getRecommenderFactory(recommender); - - if (maybeFactory.isEmpty()) { - LOG.warn("{}[{}]: No factory found - skipping recommender", aSessionOwner, - recommender.getName()); - return; - } - - var factory = maybeFactory.get(); - - // Check that configured layer and feature are accepted - // by this type of recommender - if (!factory.accepts(recommender.getLayer(), recommender.getFeature())) { - aPredictions.log(LogMessage.info(recommender.getName(), - "Recommender configured with invalid layer or feature... skipping")); - LOG.info("{}[{}]: Recommender configured with invalid layer or feature " - + "- skipping recommender", aSessionOwner, recommender.getName()); - return; - } - - // We lazily load the CAS only at this point because that allows us to skip - // loading the CAS entirely if there is no enabled layer or recommender. - // If the CAS cannot be loaded, then we skip to the next document. - var originalCas = aOriginalCas.get(); - predictionBegin = aPredictionBegin < 0 ? 0 : aPredictionBegin; - predictionEnd = aPredictionEnd < 0 ? originalCas.getDocumentText().length() - : aPredictionEnd; - - try { - var engine = factory.build(recommender); - - if (!engine.isReadyForPrediction(context.get())) { - aPredictions.log(LogMessage.info(recommender.getName(), - "Recommender context is not ready... skipping")); - LOG.info("Recommender context {} for user {} in project {} is not ready for " // - + "prediction - skipping recommender", recommender, aSessionOwner, - aDocument.getProject()); - - // If possible, we inherit recommendations from a previous run while - // the recommender is still busy - if (activePredictions != null) { - inheritSuggestionsAtRecommenderLevel(aPredictions, originalCas, recommender, - activePredictions, aDocument, aSessionOwner); - } - - return; - } - - cloneAndMonkeyPatchCAS(project, originalCas, predictionCas); - - // If the recommender is not trainable and not sensitive to annotations, - // we can actually re-use the predictions. - if (TRAINING_NOT_SUPPORTED == engine.getTrainingCapability() - && PREDICTION_USES_TEXT_ONLY == engine.getPredictionCapability() - && activePredictions != null - && activePredictions.hasRunPredictionOnDocument(aDocument)) { - inheritSuggestionsAtRecommenderLevel(aPredictions, originalCas, - engine.getRecommender(), activePredictions, aDocument, aSessionOwner); - } - else { - var ctx = new PredictionContext(context.get()); - ctx.setUser(aSessionOwner); - generateSuggestions(aPredictions, ctx, engine, activePredictions, aDocument, - originalCas, predictionCas, predictionBegin, predictionEnd); - ctx.getMessages().forEach(aPredictions::log); - } - } - // Catching Throwable is intentional here as we want to continue the - // execution even if a particular recommender fails. - catch (Throwable e) { - aPredictions.log(LogMessage.error(recommender.getName(), "Failed: %s", e.getMessage())); - LOG.error("Error applying recommender {} for user {} to document {} in project {} - " // - + "skipping recommender", recommender, aSessionOwner, aDocument, - aDocument.getProject(), e); - - applicationEventPublisher.publishEvent(RecommenderTaskNotificationEvent - .builder(this, project, aSessionOwner.getUsername()) // - .withMessage(LogMessage.error(this, "Recommender [%s] failed: %s", - recommender.getName(), e.getMessage())) // - .build()); - - // If there was a previous successful run of the recommender, inherit - // its suggestions to avoid that all the suggestions of the recommender - // simply disappear. - if (activePredictions != null) { - inheritSuggestionsAtRecommenderLevel(aPredictions, originalCas, recommender, - activePredictions, aDocument, aSessionOwner); - } - - return; - } - } - - /** - * @param aPredictions - * the predictions to populate - * @param aPredictionCas - * the re-usable buffer CAS to use when calling recommenders - * @param aDocument - * the current document - * @param aPredictionBegin - * begin of the prediction window (< 0 for 0) - * @param aPredictionEnd - * end of the prediction window (< 0 for document-end) - * @param aDataOwner - * the annotation data owner - */ - private void computePredictions(Predictions aActivePredictions, Predictions aPredictions, - CAS aPredictionCas, SourceDocument aDocument, String aDataOwner, int aPredictionBegin, - int aPredictionEnd) - { - var aSessionOwner = aPredictions.getSessionOwner(); - - try { - var recommenders = getActiveRecommenders(aSessionOwner, aDocument.getProject()); - if (recommenders.isEmpty()) { - aPredictions.log(LogMessage.info(this, "No active recommenders")); - LOG.trace("[{}]: No active recommenders", aSessionOwner); - return; - } - - LazyCas originalCas = new LazyCas(aDocument, aDataOwner); - for (var recommender : recommenders) { - var layer = schemaService.getLayer(recommender.getRecommender().getLayer().getId()); - if (!layer.isEnabled()) { - continue; - } - - computePredictions(originalCas, recommender, aActivePredictions, aPredictions, - aPredictionCas, aDocument, aSessionOwner, aPredictionBegin, aPredictionEnd); - } - } - catch (IOException e) { - aPredictions.log(LogMessage.error(this, "Cannot read annotation CAS... skipping")); - LOG.error( - "Cannot read annotation CAS for user {} of document " - + "[{}]({}) in project [{}]({}) - skipping document", - aSessionOwner, aDocument.getName(), aDocument.getId(), - aDocument.getProject().getName(), aDocument.getProject().getId(), e); - return; - } - - // When all recommenders have completed on the document, we mark it as "complete" - aPredictions.markDocumentAsPredictionCompleted(aDocument); - } - - @Override - public Predictions computePredictions(User aSessionOwner, Project aProject, - List aDocuments, String aDataOwner, TaskMonitor aMonitor) - { - var activePredictions = getPredictions(aSessionOwner, aProject); - var predictions = activePredictions != null ? new Predictions(activePredictions) - : new Predictions(aSessionOwner, aDataOwner, aProject); - - try (var casHolder = new PredictionCasHolder()) { - // Generate new predictions or inherit at the recommender level - aMonitor.setMaxProgress(aDocuments.size()); - for (SourceDocument document : aDocuments) { - aMonitor.addMessage(LogMessage.info(this, "%s", document.getName())); - aMonitor.incrementProgress(); - computePredictions(activePredictions, predictions, casHolder.cas, document, - aDataOwner, -1, -1); - } - - return predictions; - } - catch (ResourceInitializationException e) { - predictions.log( - LogMessage.error(this, "Cannot create prediction CAS, stopping predictions!")); - LOG.error("Cannot create prediction CAS, stopping predictions!"); - return predictions; - } - } - - @Override - public Predictions computePredictions(User aSessionOwner, Project aProject, - SourceDocument aCurrentDocument, String aDataOwner, List aInherit, - int aPredictionBegin, int aPredictionEnd, TaskMonitor aMonitor) - { - aMonitor.setMaxProgress(1); - - var activePredictions = getPredictions(aSessionOwner, aProject); - var predictions = activePredictions != null ? new Predictions(activePredictions) - : new Predictions(aSessionOwner, aDataOwner, aProject); - - // Inherit at the document level. If inheritance at a recommender level is possible, - // this is done below. - if (activePredictions != null) { - for (var document : aInherit) { - inheritSuggestionsAtDocumentLevel(aProject, document, aSessionOwner, - activePredictions, predictions); - } - } - - try (var casHolder = new PredictionCasHolder()) { - final CAS predictionCas = casHolder.cas; - - // Generate new predictions or inherit at the recommender level - computePredictions(activePredictions, predictions, predictionCas, aCurrentDocument, - aDataOwner, aPredictionBegin, aPredictionEnd); - } - catch (ResourceInitializationException e) { - predictions.log( - LogMessage.error(this, "Cannot create prediction CAS, stopping predictions!")); - LOG.error("Cannot create prediction CAS, stopping predictions!"); - } - - aMonitor.setProgress(1); - - return predictions; - } - - /** - * Extracts existing predictions from the last prediction run so we do not have to recalculate - * them. This is useful when the engine is not trainable. - */ - private void inheritSuggestionsAtRecommenderLevel(Predictions predictions, CAS aOriginalCas, - Recommender aRecommender, Predictions activePredictions, SourceDocument document, - User aUser) - { - var suggestions = activePredictions.getPredictionsByRecommenderAndDocument(aRecommender, - document.getName()); - - if (suggestions.isEmpty()) { - LOG.debug("{} for user {} on document {} in project {} there " // - + "are no inheritable predictions", aRecommender, aUser, document, - aRecommender.getProject()); - predictions.log(LogMessage.info(aRecommender.getName(), - "No inheritable suggestions from previous run")); - return; - } - - LOG.debug("{} for user {} on document {} in project {} inherited {} " // - + "predictions", aRecommender, aUser, document, aRecommender.getProject(), - suggestions.size()); - - predictions.log(LogMessage.info(aRecommender.getName(), - "Inherited [%d] predictions from previous run", suggestions.size())); - - predictions.inheritSuggestions(suggestions); - } - - /** - * Extracts existing predictions from the last prediction run so we do not have to recalculate - * them. This is useful when the engine is not trainable. - */ - private void inheritSuggestionsAtDocumentLevel(Project aProject, SourceDocument aDocument, - User aUser, Predictions aOldPredictions, Predictions aNewPredictions) - { - if (!aOldPredictions.hasRunPredictionOnDocument(aDocument)) { - return; - } - - var suggestions = aOldPredictions.getPredictionsByDocument(aDocument.getName()); - - LOG.debug("[{}]({}) for user [{}] on document {} in project {} inherited {} predictions", - "ALL", "--", aUser.getUsername(), aDocument, aProject, suggestions.size()); - - aNewPredictions.inheritSuggestions(suggestions); - aNewPredictions.markDocumentAsPredictionCompleted(aDocument); - } - - /** - * Invokes the engine to produce new suggestions. - */ - void generateSuggestions(Predictions aIncomingPredictions, PredictionContext aCtx, - RecommendationEngine aEngine, Predictions aActivePredictions, SourceDocument aDocument, - CAS aOriginalCas, CAS aPredictionCas, int aPredictionBegin, int aPredictionEnd) - throws RecommendationException - { - var sessionOwner = aIncomingPredictions.getSessionOwner(); - var recommender = aEngine.getRecommender(); - - // Extract the suggestions from the data which the recommender has written into the CAS - var maybeSupportRegistry = suggestionSupportRegistry.findGenericExtension(recommender); - if (maybeSupportRegistry.isEmpty()) { - LOG.debug("There is no comparible suggestion support for {} - skipping prediction"); - aIncomingPredictions.log(LogMessage.warn(recommender.getName(), // - "Prediction skipped since there is no compatible suggestion support.")); - return; - } - - // Perform the actual prediction - aIncomingPredictions.log(LogMessage.info(recommender.getName(), - "Generating predictions for layer [%s]...", recommender.getLayer().getUiName())); - LOG.trace("{}[{}]: Generating predictions for layer [{}]", sessionOwner, - recommender.getName(), recommender.getLayer().getUiName()); - - var predictedRange = aEngine.predict(aCtx, aPredictionCas, aPredictionBegin, - aPredictionEnd); - - var extractionContext = new ExtractionContext(aIncomingPredictions.getGeneration(), - recommender, aDocument, aOriginalCas, aPredictionCas); - var generatedSuggestions = maybeSupportRegistry.get().extractSuggestions(extractionContext); - - // Reconcile new suggestions with suggestions from previous run - var reconciliationResult = reconcile(aActivePredictions, aDocument, recommender, - predictedRange, generatedSuggestions); - LOG.debug( - "{} for user {} on document {} in project {} generated {} predictions within range {} (+{}/-{}/={})", - recommender, sessionOwner, aDocument, recommender.getProject(), - generatedSuggestions.size(), predictedRange, reconciliationResult.added, - reconciliationResult.removed, reconciliationResult.aged); - aIncomingPredictions.log(LogMessage.info(recommender.getName(), // - "Generated [%d] predictions within range %s (+%d/-%d/=%d)", - generatedSuggestions.size(), predictedRange, reconciliationResult.added, - reconciliationResult.removed, reconciliationResult.aged)); - var suggestions = reconciliationResult.suggestions; - var added = reconciliationResult.added; - var removed = reconciliationResult.removed; - var aged = reconciliationResult.aged; - - // Inherit suggestions that are outside the range which was predicted. Note that the engine - // might actually predict a different range from what was requested. If the prediction - // covers the entire document, we can skip this. - if (aActivePredictions != null - && !predictedRange.equals(rangeCoveringDocument(aOriginalCas))) { - var inheritableSuggestions = aActivePredictions - .getPredictionsByRecommenderAndDocument(recommender, aDocument.getName()) - .stream() // - .filter(s -> !s.coveredBy(predictedRange)) // - .collect(toList()); - - LOG.debug("{} for user {} on document {} in project {} inherited {} " // - + "predictions", recommender, sessionOwner, aDocument, recommender.getProject(), - inheritableSuggestions.size()); - aIncomingPredictions.log(LogMessage.info(recommender.getName(), - "Inherited [%d] predictions from previous run", inheritableSuggestions.size())); - - for (var suggestion : inheritableSuggestions) { - aged++; - suggestion.incrementAge(); - } - suggestions.addAll(inheritableSuggestions); - } - - // Calculate the visibility of the suggestions. This happens via the original CAS which - // contains only the manually created annotations and *not* the suggestions. - var groupedSuggestions = SuggestionDocumentGroup.groupByType(suggestions); - for (var groupEntry : groupedSuggestions.entrySet()) { - calculateSuggestionVisibility(sessionOwner.getUsername(), aDocument, aOriginalCas, - aIncomingPredictions.getDataOwner(), aEngine.getRecommender().getLayer(), - groupEntry.getValue(), 0, aOriginalCas.getDocumentText().length()); - } - - aIncomingPredictions.putSuggestions(added, removed, aged, suggestions); - } - - static ReconciliationResult reconcile(Predictions aActivePredictions, SourceDocument aDocument, - Recommender recommender, Range predictedRange, - List aNewProtoSuggestions) - { - if (aActivePredictions == null) { - return new ReconciliationResult(aNewProtoSuggestions.size(), 0, 0, - aNewProtoSuggestions); - } - - var reconciledSuggestions = new LinkedHashSet(); - var addedSuggestions = new ArrayList(); - int agedSuggestionsCount = 0; - - var predictionsByRecommenderAndDocument = aActivePredictions - .getPredictionsByRecommenderAndDocument(recommender, aDocument.getName()); - - var existingSuggestionsByPosition = predictionsByRecommenderAndDocument.stream() // - .filter(s -> s.coveredBy(predictedRange)) // - .collect(groupingBy(AnnotationSuggestion::getPosition)); - - for (var newSuggestion : aNewProtoSuggestions) { - var existingSuggestions = existingSuggestionsByPosition - .getOrDefault(newSuggestion.getPosition(), emptyList()).stream() // - .filter(s -> matchesForReconciliation(newSuggestion, s)) // - .limit(2) // One to use, the second to warn that there was more than one - .toList(); - - if (existingSuggestions.isEmpty()) { - addedSuggestions.add(newSuggestion); - continue; - } - - if (existingSuggestions.size() > 1) { - LOG.debug("Recommender produced more than one suggestion with the same " - + "label, score and score explanation - reconciling with first one"); - } - - var existingSuggestion = existingSuggestions.get(0); - if (!reconciledSuggestions.contains(existingSuggestion)) { - existingSuggestion.incrementAge(); - agedSuggestionsCount++; - reconciledSuggestions.add(existingSuggestion); - } - } - - var removedSuggestions = predictionsByRecommenderAndDocument.stream() // - .filter(s -> s.coveredBy(predictedRange)) // - .filter(s -> !reconciledSuggestions.contains(s)) // - .toList(); - - var finalSuggestions = new ArrayList<>(reconciledSuggestions); - finalSuggestions.addAll(addedSuggestions); - return new ReconciliationResult(addedSuggestions.size(), removedSuggestions.size(), - agedSuggestionsCount, finalSuggestions); - } - - private static boolean matchesForReconciliation(AnnotationSuggestion aNew, - AnnotationSuggestion aExisting) - { - return aNew.getRecommenderId() == aExisting.getRecommenderId() && // - Objects.equals(aExisting.getLabel(), aNew.getLabel()); - } - @Override public void calculateSuggestionVisibility(String aSessionOwner, SourceDocument aDocument, CAS aCas, String aDataOwner, AnnotationLayer aLayer, @@ -1902,37 +1417,6 @@ public void calculateSuggestionVisibility(Strin } } - /** - * Clones the source CAS to the target CAS while adding the features required for encoding - * predictions to the respective types. - * - * @param aProject - * the project to which the CASes belong. - * @param aSourceCas - * the source CAS. - * @param aTargetCas - * the target CAS which is meant to be sent off to a recommender. - * @return the target CAS which is meant to be sent off to a recommender. - * @throws UIMAException - * if there was a CAS-related error. - * @throws IOException - * if there was a serialization-related errror. - */ - CAS cloneAndMonkeyPatchCAS(Project aProject, CAS aSourceCas, CAS aTargetCas) - throws UIMAException, IOException - { - try (var watch = new StopWatch(LOG, "adding score features")) { - var tsd = schemaService.getFullProjectTypeSystem(aProject); - var features = schemaService.listAnnotationFeature(aProject); - - RecommenderTypeSystemUtils.addPredictionFeaturesToTypeSystem(tsd, features); - - schemaService.upgradeCas(aSourceCas, aTargetCas, tsd); - } - - return aTargetCas; - } - private class TriggerTrainingTaskListener implements IRequestCycleListener { @@ -2275,50 +1759,4 @@ public void deleteSkippedSuggestions(String aSessionOwner, User aDataOwner, .setParameter("action", SKIPPED) // .executeUpdate(); } - - private class LazyCas - { - private final SourceDocument document; - private final String dataOwner; - - private CAS originalCas; - - public LazyCas(SourceDocument aDocument, String aDataOwner) - { - document = aDocument; - dataOwner = aDataOwner; - } - - public CAS get() throws IOException - { - if (originalCas == null) { - originalCas = documentService.readAnnotationCas(document, dataOwner, - AUTO_CAS_UPGRADE, SHARED_READ_ONLY_ACCESS); - } - - return originalCas; - } - } - - private static class PredictionCasHolder - implements AutoCloseable - { - private final CAS cas; - - public PredictionCasHolder() throws ResourceInitializationException - { - cas = WebAnnoCasUtil.createCas(); - CasStorageSession.get().add(PREDICTION_CAS, EXCLUSIVE_WRITE_ACCESS, cas); - } - - @Override - public void close() - { - CasStorageSession.get().remove(cas); - } - } - - final record ReconciliationResult(int added, int removed, int aged, - List suggestions) - {} } diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java index 9132feb44ac..3dd50f1d667 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java @@ -17,12 +17,29 @@ */ package de.tudarmstadt.ukp.inception.recommendation.tasks; +import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode.EXCLUSIVE_WRITE_ACCESS; +import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode.SHARED_READ_ONLY_ACCESS; +import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasUpgradeMode.AUTO_CAS_UPGRADE; +import static de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionCapability.PREDICTION_USES_TEXT_ONLY; +import static de.tudarmstadt.ukp.inception.recommendation.api.recommender.TrainingCapability.TRAINING_NOT_SUPPORTED; +import static de.tudarmstadt.ukp.inception.rendering.model.Range.rangeCoveringDocument; import static java.lang.System.currentTimeMillis; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; +import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; +import javax.persistence.NoResultException; + +import org.apache.uima.UIMAException; +import org.apache.uima.cas.CAS; +import org.apache.uima.resource.ResourceInitializationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -34,19 +51,38 @@ import de.tudarmstadt.ukp.inception.annotation.storage.CasStorageSession; import de.tudarmstadt.ukp.inception.documents.api.DocumentService; import de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService; +import de.tudarmstadt.ukp.inception.recommendation.api.RecommenderTypeSystemUtils; +import de.tudarmstadt.ukp.inception.recommendation.api.SuggestionSupport; +import de.tudarmstadt.ukp.inception.recommendation.api.SuggestionSupportRegistry; +import de.tudarmstadt.ukp.inception.recommendation.api.model.AnnotationSuggestion; import de.tudarmstadt.ukp.inception.recommendation.api.model.Predictions; +import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; +import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionDocumentGroup; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.ExtractionContext; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderTaskNotificationEvent; +import de.tudarmstadt.ukp.inception.rendering.model.Range; +import de.tudarmstadt.ukp.inception.scheduling.TaskMonitor; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; +import de.tudarmstadt.ukp.inception.support.StopWatch; import de.tudarmstadt.ukp.inception.support.WebAnnoConst; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; +import de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil; public class PredictionTask extends RecommendationTask_ImplBase { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String PREDICTION_CAS = "predictionCas"; + + private @Autowired AnnotationSchemaService schemaService; private @Autowired RecommendationService recommendationService; private @Autowired DocumentService documentService; private @Autowired ApplicationEventPublisher appEventPublisher; + private @Autowired SuggestionSupportRegistry suggestionSupportRegistry; private final SourceDocument currentDocument; private final int predictionBegin; @@ -83,6 +119,17 @@ public PredictionTask(User aSessionOwner, String aTrigger, SourceDocument aCurre dataOwner = aDataOwner; } + /** + * For testing. + * + * @param aSchemaService + * schema service + */ + void setSchemaService(AnnotationSchemaService aSchemaService) + { + schemaService = aSchemaService; + } + @Override public String getTitle() { @@ -94,13 +141,12 @@ public void execute() { try (var session = CasStorageSession.openNested()) { var project = getProject(); - var sessionOwner = getUser().orElseThrow(); - var sessionOwnerName = sessionOwner.getUsername(); + var sessionOwner = getSessionOwner(); var startTime = System.currentTimeMillis(); - var predictions = generatePredictions(sessionOwner); + var predictions = generatePredictions(); predictions.inheritLog(getLogMessages()); - logPredictionComplete(predictions, startTime, sessionOwnerName); + logPredictionComplete(predictions, startTime); recommendationService.putIncomingPredictions(sessionOwner, project, predictions); @@ -119,61 +165,735 @@ public void execute() LogMessage.info(this, "Prediction run produced no new suggestions")) // .build()); } - - // We reset this in case the state was not properly cleared, e.g. the AL session - // was started but then the browser closed. Places where it is set include - // - ActiveLearningSideBar::moveToNextRecommendation - recommendationService.setPredictForAllDocuments(sessionOwnerName, project, false); } } - private Predictions generatePredictions(User sessionOwner) + private User getSessionOwner() + { + return getUser().orElseThrow(); + } + + private Predictions generatePredictions() { var project = getProject(); - var sessionOwnerName = sessionOwner.getUsername(); var docs = documentService.listSourceDocuments(project); // Do we need to predict ALL documents (e.g. in active learning mode) - if (recommendationService.isPredictForAllDocuments(sessionOwnerName, project)) { - logPredictionStartedForAllDocuments(sessionOwnerName, project, docs); - return recommendationService.computePredictions(sessionOwner, project, docs, dataOwner, - getMonitor()); + if (recommendationService.isPredictForAllDocuments(getSessionOwner().getUsername(), + project)) { + try { + return generatePredictionsOnAllDocuments(docs, getMonitor()); + } + finally { + // We reset this in case the state was not properly cleared, e.g. the AL session + // was started but then the browser closed. Places where it is set include + // - ActiveLearningSideBar::moveToNextRecommendation + recommendationService.setPredictForAllDocuments(getSessionOwner().getUsername(), + project, false); + } + } + + return generatePredictionsOnSingleDocument(currentDocument, docs, getMonitor()); + } + + /** + * Generate predictions for all documents. No predictions are inherited. + * + * @param aDocuments + * the documents to compute the predictions for. + * @return the new predictions. + */ + private Predictions generatePredictionsOnAllDocuments(List aDocuments, + TaskMonitor aMonitor) + { + logPredictionStartedForAllDocuments(aDocuments); + + var sessionOwner = getSessionOwner(); + var project = getProject(); + var activePredictions = recommendationService.getPredictions(sessionOwner, project); + var predictions = activePredictions != null ? new Predictions(activePredictions) + : new Predictions(sessionOwner, dataOwner, project); + + try (var casHolder = new PredictionCasHolder()) { + aMonitor.setMaxProgress(aDocuments.size()); + + for (var document : aDocuments) { + aMonitor.addMessage(LogMessage.info(this, "%s", document.getName())); + aMonitor.incrementProgress(); + applyAllRecommendersToDocument(activePredictions, predictions, casHolder.cas, + document, -1, -1); + } + + return predictions; + } + catch (ResourceInitializationException e) { + logErrorCreationPredictionCas(predictions); + return predictions; + } + } + + /** + * Generate predictions for a single document. Any predictions available for other documents are + * inherited. + * + * @param aCurrentDocument + * the document to compute the predictions for. + * @param aDocuments + * all documents from the current project. + * @return the new predictions. + */ + private Predictions generatePredictionsOnSingleDocument(SourceDocument aCurrentDocument, + List aDocuments, TaskMonitor aMonitor) + { + var sessionOwner = getSessionOwner(); + var project = getProject(); + var activePredictions = recommendationService.getPredictions(sessionOwner, project); + var predictions = activePredictions != null ? new Predictions(activePredictions) + : new Predictions(sessionOwner, dataOwner, project); + + aMonitor.setMaxProgress(1); + + if (activePredictions != null) { + // Limit prediction to a single document and inherit the rest + var documentsToInheritSuggestionsFor = aDocuments.stream() // + .filter(d -> !d.equals(currentDocument)) // + .toList(); + + logPredictionStartedForOneDocumentWithInheritance(documentsToInheritSuggestionsFor); + + for (var document : documentsToInheritSuggestionsFor) { + inheritSuggestionsAtDocumentLevel(project, document, activePredictions, + predictions); + } + } + else { + logPredictionStartedForOneDocumentWithoutInheritance(); + } + + try (var casHolder = new PredictionCasHolder()) { + + final CAS predictionCas = casHolder.cas; + applyAllRecommendersToDocument(activePredictions, predictions, predictionCas, + aCurrentDocument, predictionBegin, predictionEnd); + } + catch (ResourceInitializationException e) { + logErrorCreationPredictionCas(predictions); + } + + aMonitor.setProgress(1); + + return predictions; + } + + /** + * @param aPredictions + * the predictions to populate + * @param aPredictionCas + * the re-usable buffer CAS to use when calling recommenders + * @param aDocument + * the current document + * @param aPredictionBegin + * begin of the prediction range (negative to predict from 0) + * @param aPredictionEnd + * end of the prediction range (negative to predict until the end of the document) + */ + private void applyAllRecommendersToDocument(Predictions aActivePredictions, + Predictions aPredictions, CAS aPredictionCas, SourceDocument aDocument, + int aPredictionBegin, int aPredictionEnd) + { + var activeRecommenders = recommendationService + .getActiveRecommenders(aPredictions.getSessionOwner(), aDocument.getProject()); + if (activeRecommenders.isEmpty()) { + logNoActiveRecommenders(aPredictions); + return; + } + + try { + var originalCas = new LazyCas(aDocument); + for (var activeRecommender : activeRecommenders) { + var layer = schemaService + .getLayer(activeRecommender.getRecommender().getLayer().getId()); + if (!layer.isEnabled()) { + continue; + } + + // Make sure we have the latest recommender config from the DB - the one + // from the active recommenders list may be outdated + var recommender = activeRecommender.getRecommender(); + try { + recommender = recommendationService.getRecommender(recommender.getId()); + } + catch (NoResultException e) { + logRecommenderNotAvailable(aPredictions, recommender); + continue; + } + + if (!recommender.isEnabled()) { + logRecommenderDisabled(aPredictions, recommender); + continue; + } + + applyRecomenderToDocument(originalCas, recommender, aActivePredictions, + aPredictions, aPredictionCas, aDocument, aPredictionBegin, aPredictionEnd); + } + } + catch (IOException e) { + logUnableToReadAnnotations(aPredictions, aDocument, e); + return; + } + + // When all recommenders have completed on the document, we mark it as "complete" + aPredictions.markDocumentAsPredictionCompleted(aDocument); + } + + private void applyRecomenderToDocument(LazyCas aOriginalCas, Recommender aRecommender, + Predictions activePredictions, Predictions aPredictions, CAS predictionCas, + SourceDocument aDocument, int aPredictionBegin, int aPredictionEnd) + throws IOException + { + var sessionOwner = getSessionOwner(); + var context = recommendationService.getContext(sessionOwner.getUsername(), aRecommender); + if (!context.isPresent()) { + logRecommenderHasNoContext(aPredictions, aDocument, aRecommender); + return; + } + + var maybeFactory = recommendationService.getRecommenderFactory(aRecommender); + if (maybeFactory.isEmpty()) { + logNoRecommenderFactory(aRecommender); + return; + } + var factory = maybeFactory.get(); + + // Check that configured layer and feature are accepted by this type of recommender + if (!factory.accepts(aRecommender.getLayer(), aRecommender.getFeature())) { + logInvalidRecommenderConfiguration(aPredictions, aRecommender); + return; + } + + // We lazily load the CAS only at this point because that allows us to skip + // loading the CAS entirely if there is no enabled layer or recommender. + // If the CAS cannot be loaded, then we skip to the next document. + var originalCas = aOriginalCas.get(); + + try { + var engine = factory.build(aRecommender); + + if (!engine.isReadyForPrediction(context.get())) { + logRecommenderContextNoReady(aPredictions, aDocument, aRecommender); + + // If possible, we inherit recommendations from a previous run while + // the recommender is still busy + if (activePredictions != null) { + inheritSuggestionsAtRecommenderLevel(aPredictions, originalCas, aRecommender, + activePredictions, aDocument); + } + + return; + } + + // If the recommender is not trainable and not sensitive to annotations, + // we can actually re-use the predictions. + if (TRAINING_NOT_SUPPORTED == engine.getTrainingCapability() + && PREDICTION_USES_TEXT_ONLY == engine.getPredictionCapability() + && activePredictions != null + && activePredictions.hasRunPredictionOnDocument(aDocument)) { + inheritSuggestionsAtRecommenderLevel(aPredictions, originalCas, + engine.getRecommender(), activePredictions, aDocument); + return; + } + + var ctx = new PredictionContext(context.get()); + cloneAndMonkeyPatchCAS(getProject(), originalCas, predictionCas); + var predictionRange = new Range(aPredictionBegin < 0 ? 0 : aPredictionBegin, + aPredictionEnd < 0 ? originalCas.getDocumentText().length() : aPredictionEnd); + invokeRecommender(aPredictions, ctx, engine, activePredictions, aDocument, originalCas, + predictionCas, predictionRange); + ctx.getMessages().forEach(aPredictions::log); } + // Catching Throwable is intentional here as we want to continue the + // execution even if a particular recommender fails. + catch (Throwable e) { + logErrorExecutingRecommender(aPredictions, aDocument, aRecommender, e); - // Limit prediction to a single document and inherit the rest - var inherit = docs.stream() // - .filter(d -> !d.equals(currentDocument)) // + appEventPublisher.publishEvent(RecommenderTaskNotificationEvent + .builder(this, getProject(), sessionOwner.getUsername()) // + .withMessage(LogMessage.error(this, "Recommender [%s] failed: %s", + aRecommender.getName(), e.getMessage())) // + .build()); + + // If there was a previous successful run of the recommender, inherit + // its suggestions to avoid that all the suggestions of the recommender + // simply disappear. + if (activePredictions != null) { + inheritSuggestionsAtRecommenderLevel(aPredictions, originalCas, aRecommender, + activePredictions, aDocument); + } + + return; + } + } + + /** + * Invokes the engine to produce new suggestions. + */ + private void invokeRecommender(Predictions aIncomingPredictions, PredictionContext aCtx, + RecommendationEngine aEngine, Predictions aActivePredictions, SourceDocument aDocument, + CAS aOriginalCas, CAS aPredictionCas, Range aPredictionRange) + throws RecommendationException + { + var recommender = aEngine.getRecommender(); + + // Extract the suggestions from the data which the recommender has written into the CAS + // We need this only for the extraction, but there is no point in investing the time for + // the prediction if we cannot extract the data afterwards - hence we obtain it now and + // skip the prediciton if it is not available + var maybeSuggestionSupport = suggestionSupportRegistry.findGenericExtension(recommender); + if (maybeSuggestionSupport.isEmpty()) { + logNoSuggestionSupportAvailable(aIncomingPredictions, recommender); + return; + } + var supportRegistry = maybeSuggestionSupport.get(); + + // Perform the actual prediction + var predictedRange = predict(aIncomingPredictions, aCtx, aEngine, aPredictionCas, + aPredictionRange); + + var generatedSuggestions = extractSuggestions(aIncomingPredictions, aDocument, aOriginalCas, + aPredictionCas, recommender, supportRegistry); + + // Reconcile new suggestions with suggestions from previous run + var reconciliationResult = reconcile(aActivePredictions, aDocument, recommender, + predictedRange, generatedSuggestions); + + logGeneratedPredictions(aIncomingPredictions, aDocument, recommender, predictedRange, + generatedSuggestions, reconciliationResult); + + // Inherit suggestions that are outside the range which was predicted. Note that the engine + // might actually predict a different range from what was requested. If the prediction + // covers the entire document, we can skip this. + var suggestions = reconciliationResult.suggestions; + var aged = reconciliationResult.aged; + if (aActivePredictions != null + && !predictedRange.equals(rangeCoveringDocument(aOriginalCas))) { + aged = inheritOutOfRangeSuggestions(aIncomingPredictions, aActivePredictions, aDocument, + recommender, predictedRange, suggestions, aged); + } + + // Calculate the visibility of the suggestions. This happens via the original CAS which + // contains only the manually created annotations and *not* the suggestions. + calculateVisibility(aIncomingPredictions, aEngine, aDocument, aOriginalCas, suggestions); + + aIncomingPredictions.putSuggestions(reconciliationResult.added, + reconciliationResult.removed, aged, suggestions); + } + + /** + * Extracts existing predictions from the last prediction run so we do not have to recalculate + * them. This is useful when the engine is not trainable. + */ + private void inheritSuggestionsAtRecommenderLevel(Predictions predictions, CAS aOriginalCas, + Recommender aRecommender, Predictions activePredictions, SourceDocument document) + { + var suggestions = activePredictions.getPredictionsByRecommenderAndDocument(aRecommender, + document.getName()); + + if (suggestions.isEmpty()) { + logNoInheritablePredictions(predictions, aRecommender, document); + return; + } + + logInheritedPredictions(predictions, aRecommender, document, suggestions); + + predictions.inheritSuggestions(suggestions); + } + + /** + * Extracts existing predictions from the last prediction run so we do not have to recalculate + * them. This is useful when the engine is not trainable. + */ + private void inheritSuggestionsAtDocumentLevel(Project aProject, SourceDocument aDocument, + Predictions aOldPredictions, Predictions aNewPredictions) + { + if (!aOldPredictions.hasRunPredictionOnDocument(aDocument)) { + return; + } + + var suggestions = aOldPredictions.getPredictionsByDocument(aDocument.getName()); + + logPredictionsInherited(aProject, aDocument, suggestions); + + aNewPredictions.inheritSuggestions(suggestions); + aNewPredictions.markDocumentAsPredictionCompleted(aDocument); + } + + private int inheritOutOfRangeSuggestions(Predictions aIncomingPredictions, + Predictions aActivePredictions, SourceDocument aDocument, Recommender recommender, + Range predictedRange, List suggestions, int aged) + { + var inheritableSuggestions = aActivePredictions + .getPredictionsByRecommenderAndDocument(recommender, aDocument.getName()).stream() // + .filter(s -> !s.coveredBy(predictedRange)) // .collect(toList()); - logPredictionStartedForOneDocument(sessionOwnerName, project, inherit); + logInheritedPredictions(aIncomingPredictions, recommender, aDocument, + inheritableSuggestions); + + for (var suggestion : inheritableSuggestions) { + aged++; + suggestion.incrementAge(); + } + suggestions.addAll(inheritableSuggestions); + return aged; + } - return recommendationService.computePredictions(sessionOwner, project, currentDocument, - dataOwner, inherit, predictionBegin, predictionEnd, getMonitor()); + private List extractSuggestions(Predictions aIncomingPredictions, + SourceDocument aDocument, CAS aOriginalCas, CAS aPredictionCas, Recommender recommender, + SuggestionSupport supportRegistry) + { + var extractionContext = new ExtractionContext(aIncomingPredictions.getGeneration(), + recommender, aDocument, aOriginalCas, aPredictionCas); + return supportRegistry.extractSuggestions(extractionContext); } - private void logPredictionComplete(Predictions aPredictions, long startTime, String username) + private Range predict(Predictions aIncomingPredictions, PredictionContext aCtx, + RecommendationEngine aEngine, CAS aPredictionCas, Range aPredictionRange) + throws RecommendationException + { + logStartGeneratingPredictions(aIncomingPredictions, aEngine.getRecommender()); + + return aEngine.predict(aCtx, aPredictionCas, aPredictionRange.getBegin(), + aPredictionRange.getEnd()); + } + + private void calculateVisibility(Predictions aIncomingPredictions, RecommendationEngine aEngine, + SourceDocument aDocument, CAS aOriginalCas, List suggestions) + { + var groupedSuggestions = SuggestionDocumentGroup.groupByType(suggestions); + for (var groupEntry : groupedSuggestions.entrySet()) { + recommendationService.calculateSuggestionVisibility( + aIncomingPredictions.getSessionOwner().getUsername(), aDocument, aOriginalCas, + aIncomingPredictions.getDataOwner(), aEngine.getRecommender().getLayer(), + groupEntry.getValue(), 0, aOriginalCas.getDocumentText().length()); + } + } + + static ReconciliationResult reconcile(Predictions aActivePredictions, SourceDocument aDocument, + Recommender recommender, Range predictedRange, + List aNewProtoSuggestions) + { + if (aActivePredictions == null) { + return new ReconciliationResult(aNewProtoSuggestions.size(), 0, 0, + aNewProtoSuggestions); + } + + var reconciledSuggestions = new LinkedHashSet(); + var addedSuggestions = new ArrayList(); + int agedSuggestionsCount = 0; + + var predictionsByRecommenderAndDocument = aActivePredictions + .getPredictionsByRecommenderAndDocument(recommender, aDocument.getName()); + + var existingSuggestionsByPosition = predictionsByRecommenderAndDocument.stream() // + .filter(s -> s.coveredBy(predictedRange)) // + .collect(groupingBy(AnnotationSuggestion::getPosition)); + + for (var newSuggestion : aNewProtoSuggestions) { + var existingSuggestions = existingSuggestionsByPosition + .getOrDefault(newSuggestion.getPosition(), emptyList()).stream() // + .filter(s -> matchesForReconciliation(newSuggestion, s)) // + .limit(2) // One to use, the second to warn that there was more than one + .toList(); + + if (existingSuggestions.isEmpty()) { + addedSuggestions.add(newSuggestion); + continue; + } + + if (existingSuggestions.size() > 1) { + LOG.debug("Recommender produced more than one suggestion with the same " + + "label, score and score explanation - reconciling with first one"); + } + + var existingSuggestion = existingSuggestions.get(0); + if (!reconciledSuggestions.contains(existingSuggestion)) { + existingSuggestion.incrementAge(); + agedSuggestionsCount++; + reconciledSuggestions.add(existingSuggestion); + } + } + + var removedSuggestions = predictionsByRecommenderAndDocument.stream() // + .filter(s -> s.coveredBy(predictedRange)) // + .filter(s -> !reconciledSuggestions.contains(s)) // + .toList(); + + var finalSuggestions = new ArrayList<>(reconciledSuggestions); + finalSuggestions.addAll(addedSuggestions); + return new ReconciliationResult(addedSuggestions.size(), removedSuggestions.size(), + agedSuggestionsCount, finalSuggestions); + } + + private static boolean matchesForReconciliation(AnnotationSuggestion aNew, + AnnotationSuggestion aExisting) + { + return aNew.getRecommenderId() == aExisting.getRecommenderId() && // + Objects.equals(aExisting.getLabel(), aNew.getLabel()); + } + + private void logNoSuggestionSupportAvailable(Predictions aIncomingPredictions, + Recommender recommender) + { + LOG.debug("There is no comparible suggestion support for {} - skipping prediction"); + aIncomingPredictions.log(LogMessage.warn(recommender.getName(), // + "Prediction skipped since there is no compatible suggestion support.")); + } + + private void logErrorCreationPredictionCas(Predictions predictions) + { + predictions + .log(LogMessage.error(this, "Cannot create prediction CAS, stopping predictions!")); + LOG.error("Cannot create prediction CAS, stopping predictions!"); + } + + private void logPredictionsInherited(Project aProject, SourceDocument aDocument, + List suggestions) + { + LOG.debug("[{}]({}) for user [{}] on document {} in project {} inherited {} predictions", + "ALL", "--", getSessionOwner().getUsername(), aDocument, aProject, + suggestions.size()); + } + + private void logErrorExecutingRecommender(Predictions aPredictions, SourceDocument aDocument, + Recommender recommender, Throwable e) + { + aPredictions.log(LogMessage.error(recommender.getName(), "Failed: %s", e.getMessage())); + LOG.error("Error applying recommender {} for user {} to document {} in project {} - " // + + "skipping recommender", recommender, getSessionOwner(), aDocument, + aDocument.getProject(), e); + } + + private void logStartGeneratingPredictions(Predictions aIncomingPredictions, + Recommender recommender) + { + aIncomingPredictions.log(LogMessage.info(recommender.getName(), + "Generating predictions for layer [%s]...", recommender.getLayer().getUiName())); + LOG.trace("{}[{}]: Generating predictions for layer [{}]", getSessionOwner(), + recommender.getName(), recommender.getLayer().getUiName()); + } + + private void logUnableToReadAnnotations(Predictions aPredictions, SourceDocument aDocument, + IOException e) + { + aPredictions.log(LogMessage.error(this, "Cannot read annotation CAS... skipping")); + LOG.error( + "Cannot read annotation CAS for user {} of document " + + "[{}]({}) in project [{}]({}) - skipping document", + getSessionOwner(), aDocument.getName(), aDocument.getId(), + aDocument.getProject().getName(), aDocument.getProject().getId(), e); + } + + private void logNoActiveRecommenders(Predictions aPredictions) + { + aPredictions.log(LogMessage.info(this, "No active recommenders")); + LOG.trace("[{}]: No active recommenders", getSessionOwner()); + } + + private void logInheritedPredictions(Predictions predictions, Recommender aRecommender, + SourceDocument document, List suggestions) + { + LOG.debug("[{}][{}]: {} on document {} in project {} inherited {} predictions", getId(), + getSessionOwner().getUsername(), aRecommender, document, aRecommender.getProject(), + suggestions.size()); + predictions.log(LogMessage.info(aRecommender.getName(), + "Inherited [%d] predictions from previous run", suggestions.size())); + } + + private void logNoInheritablePredictions(Predictions predictions, Recommender aRecommender, + SourceDocument document) + { + LOG.debug("[{}][{}]: {} on document {} in project {} there " // + + "are no inheritable predictions", getId(), getSessionOwner().getUsername(), + aRecommender, document, aRecommender.getProject()); + predictions.log(LogMessage.info(aRecommender.getName(), + "No inheritable suggestions from previous run")); + } + + private void logPredictionComplete(Predictions aPredictions, long startTime) { var duration = currentTimeMillis() - startTime; - LOG.debug("[{}][{}]: Prediction complete ({} ms)", getId(), username, duration); + LOG.debug("[{}][{}]: Prediction complete ({} ms)", getId(), getSessionOwner().getUsername(), + duration); aPredictions.log(LogMessage.info(this, "Prediction complete (%d ms).", duration)); } - private void logPredictionStartedForOneDocument(String username, Project project, - List inherit) + private void logPredictionStartedForOneDocumentWithInheritance(List inherit) { LOG.debug( "[{}][{}]: Starting prediction for project [{}] on one document " + "(inheriting [{}]) triggered by [{}]", - getId(), username, project, inherit.size(), getTrigger()); + getId(), getSessionOwner().getUsername(), getProject(), inherit.size(), + getTrigger()); + info("Starting prediction triggered by [%s]...", getTrigger()); + } + + private void logPredictionStartedForOneDocumentWithoutInheritance() + { + LOG.debug( + "[{}][{}]: Starting prediction for project [{}] on one document " + + "triggered by [{}]", + getId(), getSessionOwner().getUsername(), getProject(), getTrigger()); info("Starting prediction triggered by [%s]...", getTrigger()); } - private void logPredictionStartedForAllDocuments(String username, Project project, - List docs) + private void logPredictionStartedForAllDocuments(List docs) { LOG.debug( "[{}][{}]: Starting prediction for project [{}] on [{}] documents triggered by [{}]", - getId(), username, project, docs.size(), getTrigger()); + getId(), getSessionOwner().getUsername(), getProject(), docs.size(), getTrigger()); info("Starting prediction triggered by [%s]...", getTrigger()); } + + private void logGeneratedPredictions(Predictions aIncomingPredictions, SourceDocument aDocument, + Recommender recommender, Range predictedRange, + List generatedSuggestions, + ReconciliationResult reconciliationResult) + { + LOG.debug( + "{} for user {} on document {} in project {} generated {} predictions within range {} (+{}/-{}/={})", + recommender, getSessionOwner(), aDocument, recommender.getProject(), + generatedSuggestions.size(), predictedRange, reconciliationResult.added, + reconciliationResult.removed, reconciliationResult.aged); + aIncomingPredictions.log(LogMessage.info(recommender.getName(), // + "Generated [%d] predictions within range %s (+%d/-%d/=%d)", + generatedSuggestions.size(), predictedRange, reconciliationResult.added, + reconciliationResult.removed, reconciliationResult.aged)); + } + + private void logRecommenderContextNoReady(Predictions aPredictions, SourceDocument aDocument, + Recommender recommender) + { + aPredictions.log(LogMessage.info(recommender.getName(), + "Recommender context is not ready... skipping")); + LOG.info("Recommender context {} for user {} in project {} is not ready for " // + + "prediction - skipping recommender", recommender, getSessionOwner(), + aDocument.getProject()); + } + + private void logInvalidRecommenderConfiguration(Predictions aPredictions, + Recommender recommender) + { + aPredictions.log(LogMessage.info(recommender.getName(), + "Recommender configured with invalid layer or feature... skipping")); + LOG.info( + "[{}][{}]: Recommender configured with invalid layer or feature " + + "- skipping recommender", + getSessionOwner().getUsername(), recommender.getName()); + } + + private void logNoRecommenderFactory(Recommender recommender) + { + LOG.warn("[{}][{}]: No factory found - skipping recommender", + getSessionOwner().getUsername(), recommender.getName()); + } + + private void logRecommenderHasNoContext(Predictions aPredictions, SourceDocument aDocument, + Recommender recommender) + { + aPredictions.log( + LogMessage.info(recommender.getName(), "Recommender has no context... skipping")); + LOG.info("No context available for recommender {} for user {} on document {} in " // + + "project {} - skipping recommender", recommender, getSessionOwner(), aDocument, + aDocument.getProject()); + } + + private void logRecommenderDisabled(Predictions aPredictions, Recommender recommender) + { + aPredictions + .log(LogMessage.info(recommender.getName(), "Recommender disabled... skipping")); + LOG.debug("{}[{}]: Disabled - skipping", getSessionOwner(), recommender.getName()); + } + + private void logRecommenderNotAvailable(Predictions aPredictions, Recommender recommender) + { + aPredictions.log(LogMessage.info(recommender.getName(), + "Recommender no longer available... skipping")); + LOG.info("{}[{}]: Recommender no longer available... skipping", getSessionOwner(), + recommender.getName()); + } + + /** + * Clones the source CAS to the target CAS while adding the features required for encoding + * predictions to the respective types. + * + * @param aProject + * the project to which the CASes belong. + * @param aSourceCas + * the source CAS. + * @param aTargetCas + * the target CAS which is meant to be sent off to a recommender. + * @return the target CAS which is meant to be sent off to a recommender. + * @throws UIMAException + * if there was a CAS-related error. + * @throws IOException + * if there was a serialization-related errror. + */ + CAS cloneAndMonkeyPatchCAS(Project aProject, CAS aSourceCas, CAS aTargetCas) + throws UIMAException, IOException + { + try (var watch = new StopWatch(LOG, "adding score features")) { + var tsd = schemaService.getFullProjectTypeSystem(aProject); + var features = schemaService.listAnnotationFeature(aProject); + + RecommenderTypeSystemUtils.addPredictionFeaturesToTypeSystem(tsd, features); + + schemaService.upgradeCas(aSourceCas, aTargetCas, tsd); + } + + return aTargetCas; + } + + private class LazyCas + { + private final SourceDocument document; + + private CAS originalCas; + + public LazyCas(SourceDocument aDocument) + { + document = aDocument; + } + + public CAS get() throws IOException + { + if (originalCas == null) { + originalCas = documentService.readAnnotationCas(document, dataOwner, + AUTO_CAS_UPGRADE, SHARED_READ_ONLY_ACCESS); + } + + return originalCas; + } + } + + private static class PredictionCasHolder + implements AutoCloseable + { + private final CAS cas; + + public PredictionCasHolder() throws ResourceInitializationException + { + cas = WebAnnoCasUtil.createCas(); + CasStorageSession.get().add(PREDICTION_CAS, EXCLUSIVE_WRITE_ACCESS, cas); + } + + @Override + public void close() + { + CasStorageSession.get().remove(cas); + } + } + + final record ReconciliationResult(int added, int removed, int aged, + List suggestions) + {} } diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java index 81d2577d920..4ebbd09cf18 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/TrainingTask.java @@ -22,6 +22,7 @@ import static java.lang.System.currentTimeMillis; import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; +import java.lang.invoke.MethodHandles; import java.util.List; import javax.persistence.NoResultException; @@ -33,7 +34,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; import de.tudarmstadt.ukp.clarin.webanno.model.Project; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.clarin.webanno.security.model.User; @@ -42,19 +42,18 @@ import de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService; import de.tudarmstadt.ukp.inception.recommendation.api.model.EvaluatedRecommender; import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommenderContext; import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderTaskNotificationEvent; import de.tudarmstadt.ukp.inception.scheduling.SchedulingService; -import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.WebAnnoConst; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public class TrainingTask extends RecommendationTask_ImplBase { - private final Logger log = LoggerFactory.getLogger(getClass()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private @Autowired AnnotationSchemaService annoService; private @Autowired DocumentService documentService; private @Autowired RecommendationService recommenderService; private @Autowired SchedulingService schedulingService; @@ -63,6 +62,9 @@ public class TrainingTask private final SourceDocument currentDocument; private final String dataOwner; + private boolean seenSuccessfulTraining = false; + private boolean seenNonTrainingRecommender = false; + /** * Create a new training task. * @@ -101,236 +103,266 @@ public String getTitle() @Override public void execute() { - try (CasStorageSession session = CasStorageSession.open()) { + try (var session = CasStorageSession.open()) { executeTraining(); } + + schedulePredictionTask(); + } + + private User getSessionOwner() + { + return getUser().orElseThrow(); } private void executeTraining() { + var activeRecommenders = recommenderService.getActiveRecommenders(getSessionOwner(), + getProject()); + + if (activeRecommenders.isEmpty()) { + logNoActiveRecommenders(); + return; + } + long overallStartTime = currentTimeMillis(); - User sessionOwner = getUser().orElseThrow(); - logTrainingOverallStart(sessionOwner); + logTrainingOverallStart(); // Read the CASes only when they are accessed the first time. This allows us to skip // reading the CASes in case that no layer / recommender is available or if no // recommender requires evaluation. var casLoader = new LazyCasLoader(documentService, getProject(), dataOwner); - boolean seenSuccessfulTraining = false; - boolean seenNonTrainingRecommender = false; - - var listAnnotationLayers = annoService.listAnnotationLayer(getProject()); - getMonitor().setMaxProgress(listAnnotationLayers.size()); - for (var layer : listAnnotationLayers) { + getMonitor().setMaxProgress(activeRecommenders.size()); + for (var activeRecommender : activeRecommenders) { getMonitor().incrementProgress(); + getMonitor().addMessage( + LogMessage.info(this, "%s", activeRecommender.getRecommender().getName())); + + // Make sure we have the latest recommender config from the DB - the one from + // the active recommenders list may be outdated + Recommender recommender; + try { + recommender = recommenderService + .getRecommender(activeRecommender.getRecommender().getId()); + } + catch (NoResultException e) { + logRecommenderGone(activeRecommender); + continue; + } - if (!layer.isEnabled()) { + if (!recommender.isEnabled()) { + logRecommenderDisabled(recommender); continue; } - var evaluatedRecommenders = recommenderService.getActiveRecommenders(sessionOwner, - layer); + if (!recommender.getLayer().isEnabled()) { + logLayerDisabled(recommender); + continue; + } - if (evaluatedRecommenders.isEmpty()) { - logNoActiveRecommenders(sessionOwner, layer); + if (!recommender.getFeature().isEnabled()) { + logFeatureDisabled(recommender); continue; } - for (var evaluatedRecommender : evaluatedRecommenders) { - // Make sure we have the latest recommender config from the DB - the one from - // the active recommenders list may be outdated - Recommender recommender; - try { - recommender = recommenderService - .getRecommender(evaluatedRecommender.getRecommender().getId()); - } - catch (NoResultException e) { - logRecommenderGone(sessionOwner, evaluatedRecommender); - continue; - } - - if (!recommender.isEnabled()) { - logRecommenderDisabled(sessionOwner, evaluatedRecommender); - continue; - } - - long startTime = currentTimeMillis(); - - try { - var maybeFactory = recommenderService.getRecommenderFactory(recommender); - if (maybeFactory.isEmpty()) { - logUnsupportedRecommenderType(sessionOwner, evaluatedRecommender); - continue; - } - - var factory = maybeFactory.get(); - if (!factory.accepts(recommender.getLayer(), recommender.getFeature())) { - logInvalidRecommenderConfiguration(sessionOwner, evaluatedRecommender, - recommender); - continue; - } - - var engine = factory.build(recommender); - var ctx = engine.newContext( - recommenderService.getContext(sessionOwner.getUsername(), recommender) - .orElse(RecommenderContext.emptyContext())); - ctx.setUser(sessionOwner); - - // If engine does not support training, mark engine ready and skip to - // prediction - if (engine.getTrainingCapability() == TRAINING_NOT_SUPPORTED) { - seenNonTrainingRecommender = true; - logTrainingNotSupported(sessionOwner, recommender); - commitContext(sessionOwner, recommender, ctx); - continue; - } - - var trainingCasses = casLoader.getRelevantCasses(recommender); - - // If no data for training is available, but the engine requires training, - // do not mark as ready - if (trainingCasses.isEmpty() - && engine.getTrainingCapability() == TRAINING_REQUIRED) { - logNoDataAvailableForTraining(sessionOwner, layer, recommender); - // This can happen if there were already predictions based on existing - // annotations, but all annotations have been removed/deleted. To ensure - // that the prediction run removes the stale predictions, we need to - // call it a success here. - seenSuccessfulTraining = true; - continue; - } - - logTrainingRecommenderStart(sessionOwner, casLoader, layer, recommender, - trainingCasses); - - engine.train(ctx, trainingCasses); - inheritLog(ctx.getMessages()); - - long duration = currentTimeMillis() - startTime; - - if (!engine.isReadyForPrediction(ctx)) { - logTrainingFailure(sessionOwner, recommender, duration, casLoader, - trainingCasses); - continue; - } - - logTrainingSuccessful(sessionOwner, casLoader, recommender, trainingCasses, - duration); - seenSuccessfulTraining = true; - - commitContext(sessionOwner, recommender, ctx); - } - // Catching Throwable is intentional here as we want to continue the execution - // even if a particular recommender fails. - catch (Throwable e) { - handleError(sessionOwner, recommender, startTime, e); - } + try { + trainRecommender(recommender, casLoader); + } + // Catching Throwable is intentional here as we want to continue the execution + // even if a particular recommender fails. + catch (Throwable e) { + handleError(recommender, e); } } if (!seenSuccessfulTraining && !seenNonTrainingRecommender) { - logNothingWasTrained(sessionOwner); + logNothingWasTrained(); return; } - logTrainingOverallEnd(overallStartTime, sessionOwner); + logTrainingOverallEnd(overallStartTime); + } + + private void trainRecommender(Recommender aRecommender, LazyCasLoader casLoader) + throws ConcurrentException, RecommendationException + { + var startTime = currentTimeMillis(); + var sessionOwner = getSessionOwner(); + + var maybeFactory = recommenderService.getRecommenderFactory(aRecommender); + if (maybeFactory.isEmpty()) { + logUnsupportedRecommenderType(aRecommender); + return; + } + + var factory = maybeFactory.get(); + if (!factory.accepts(aRecommender.getLayer(), aRecommender.getFeature())) { + logInvalidRecommenderConfiguration(aRecommender); + return; + } + + var engine = factory.build(aRecommender); + var ctx = engine + .newContext(recommenderService.getContext(sessionOwner.getUsername(), aRecommender) + .orElse(RecommenderContext.emptyContext())); + ctx.setUser(sessionOwner); + + // If engine does not support training, mark engine ready and skip to + // prediction + if (engine.getTrainingCapability() == TRAINING_NOT_SUPPORTED) { + seenNonTrainingRecommender = true; + logTrainingNotSupported(aRecommender); + commitContext(sessionOwner, aRecommender, ctx); + return; + } + + var trainingCasses = casLoader.getRelevantCasses(aRecommender); + + // If no data for training is available, but the engine requires training, + // do not mark as ready + if (trainingCasses.isEmpty() && engine.getTrainingCapability() == TRAINING_REQUIRED) { + logNoDataAvailableForTraining(aRecommender); + // This can happen if there were already predictions based on existing + // annotations, but all annotations have been removed/deleted. To ensure + // that the prediction run removes the stale predictions, we need to + // call it a success here. + seenSuccessfulTraining = true; + return; + } + + logTrainingRecommenderStart(casLoader, aRecommender, trainingCasses); + + engine.train(ctx, trainingCasses); + inheritLog(ctx.getMessages()); - schedulePredictionTask(sessionOwner); + var duration = currentTimeMillis() - startTime; + + if (!engine.isReadyForPrediction(ctx)) { + logTrainingFailure(aRecommender, duration, casLoader, trainingCasses); + return; + } + + logTrainingSuccessful(casLoader, aRecommender, trainingCasses, duration); + seenSuccessfulTraining = true; + + commitContext(sessionOwner, aRecommender, ctx); } - private void schedulePredictionTask(User user) + private void schedulePredictionTask() { - var predictionTask = new PredictionTask(user, + var predictionTask = new PredictionTask(getSessionOwner(), String.format("TrainingTask %s complete", getId()), currentDocument, dataOwner); + predictionTask.inheritLog(this); + schedulingService.enqueue(predictionTask); } - private void logTrainingOverallEnd(long overallStartTime, User user) + private void commitContext(User user, Recommender recommender, RecommenderContext ctx) + { + ctx.close(); + recommenderService.putContext(user, recommender, ctx); + } + + private void logTrainingOverallEnd(long overallStartTime) { info("Training complete (%d ms).", currentTimeMillis() - overallStartTime); } - private void commitContext(User user, Recommender recommender, RecommenderContext ctx) + private void logUnsupportedRecommenderType(Recommender aRecommender) { - ctx.close(); - recommenderService.putContext(user, recommender, ctx); + LOG.warn("[{}][{}]: No factory found - skipping recommender", + getSessionOwner().getUsername(), aRecommender.getName()); } - private void logUnsupportedRecommenderType(User user, EvaluatedRecommender evaluatedRecommender) + private void logTrainingNotSupported(Recommender aRecommender) { - log.warn("[{}][{}]: No factory found - skipping recommender", user.getUsername(), - evaluatedRecommender.getRecommender().getName()); + LOG.debug("[{}][{}][{}]: Engine does not support training", getId(), + getSessionOwner().getUsername(), aRecommender.getName()); } - private void logTrainingNotSupported(User user, Recommender recommender) + private void logRecommenderDisabled(Recommender aRecommender) { - log.debug("[{}][{}][{}]: Engine does not support training", getId(), user.getUsername(), - recommender.getName()); + LOG.debug("[{}][{}][{}]: Recommenderdisabled - skipping", getSessionOwner().getUsername(), + getId(), aRecommender.getName()); + } + + private void logLayerDisabled(Recommender aRecommender) + { + LOG.debug("[{}][{}][{}]: Layer disabled - skipping", getSessionOwner().getUsername(), + getId(), aRecommender.getLayer().getUiName()); } - private void logRecommenderDisabled(User user, EvaluatedRecommender evaluatedRecommender) + private void logFeatureDisabled(Recommender aRecommender) { - log.debug("[{}][{}][{}]: Disabled - skipping", user.getUsername(), getId(), - evaluatedRecommender.getRecommender().getName()); + LOG.debug("[{}][{}][{}]: Feature disabled - skipping", getSessionOwner().getUsername(), + getId(), aRecommender.getFeature().getUiName()); } - private void logRecommenderGone(User user, EvaluatedRecommender evaluatedRecommender) + private void logRecommenderGone(EvaluatedRecommender evaluatedRecommender) { - log.debug("[{}][{}][{}]: Recommender no longer available... skipping", getId(), - user.getUsername(), evaluatedRecommender.getRecommender().getName()); + LOG.debug("[{}][{}][{}]: Recommender no longer available... skipping", getId(), + getSessionOwner().getUsername(), evaluatedRecommender.getRecommender().getName()); } - private void logNothingWasTrained(User user) + private void logNothingWasTrained() { - log.debug("[{}][{}]: No recommenders trained successfully and no non-training " - + "recommenders, skipping prediction.", getId(), user.getUsername()); + LOG.debug( + "[{}][{}]: No recommenders trained successfully and no non-training " + + "recommenders, skipping prediction.", + getId(), getSessionOwner().getUsername()); } - private void logNoActiveRecommenders(User user, AnnotationLayer layer) + private void logNoActiveRecommenders() { - log.trace("[{}][{}][{}]: No active recommenders, skipping training.", getId(), - user.getUsername(), layer.getUiName()); - info("No active recommenders for layer [%s], skipping training.", layer.getUiName()); + LOG.trace("[{}][{}]: No active recommenders, skipping training.", getId(), + getSessionOwner().getUsername()); + + info("No active recommenders, skipping training."); } - private void logInvalidRecommenderConfiguration(User user, EvaluatedRecommender r, - Recommender recommender) + private void logInvalidRecommenderConfiguration(Recommender recommender) { - log.debug( + LOG.debug( "[{}][{}][{}]: Recommender configured with invalid layer or " + "feature - skipping recommender", - getId(), user.getUsername(), r.getRecommender().getName()); + getId(), getSessionOwner().getUsername(), recommender.getName()); + error("Recommender [%s] configured with invalid layer or feature - skipping recommender.", - r.getRecommender().getName()); - appEventPublisher.publishEvent( - RecommenderTaskNotificationEvent.builder(this, getProject(), user.getUsername()) // - .withMessage(LogMessage.error(this, - "Recommender [%s] configured with invalid layer or " - + "feature - skipping training recommender.", - recommender.getName())) - .build()); + recommender.getName()); + + appEventPublisher.publishEvent(RecommenderTaskNotificationEvent + .builder(this, getProject(), getSessionOwner().getUsername()) // + .withMessage(LogMessage.error(this, + "Recommender [%s] configured with invalid layer or " + + "feature - skipping training recommender.", + recommender.getName())) + .build()); } - private void logNoDataAvailableForTraining(User user, AnnotationLayer layer, - Recommender recommender) + private void logNoDataAvailableForTraining(Recommender recommender) { - log.debug("[{}][{}][{}]: There are no annotations available to train on", getId(), - user.getUsername(), recommender.getName()); - warn("There are no [%s] annotations available to train on.", layer.getUiName()); + LOG.debug("[{}][{}][{}]: There are no annotations available to train on", getId(), + getSessionOwner().getUsername(), recommender.getName()); + + warn("There are no [%s] annotations available to train on.", + recommender.getLayer().getUiName()); } - private void logTrainingFailure(User user, Recommender recommender, long duration, - LazyCasLoader aLoader, List aTrainCasses) + private void logTrainingFailure(Recommender recommender, long duration, LazyCasLoader aLoader, + List aTrainCasses) { int docNum = aLoader.size(); int trainDocNum = aTrainCasses.size(); - log.debug("[{}][{}][{}]: Training on [{}] out of [{}] documents not successful ({} ms)", - getId(), user.getUsername(), recommender.getName(), trainDocNum, docNum, duration); + LOG.debug("[{}][{}][{}]: Training on [{}] out of [{}] documents not successful ({} ms)", + getId(), getSessionOwner().getUsername(), recommender.getName(), trainDocNum, + docNum, duration); + info("Training not successful (%d ms).", duration); + // The recommender may decide for legitimate reasons not to train and // then this event is annoying // appEventPublisher.publishEvent(new RecommenderTaskEvent(this, @@ -340,51 +372,51 @@ private void logTrainingFailure(User user, Recommender recommender, long duratio // recommender)); } - private void logTrainingSuccessful(User user, LazyCasLoader casses, Recommender recommender, + private void logTrainingSuccessful(LazyCasLoader casses, Recommender recommender, List cassesForTraining, long duration) throws ConcurrentException { - log.debug("[{}][{}][{}]: Training successful on [{}] out of [{}] documents ({} ms)", - getId(), user.getUsername(), recommender.getName(), cassesForTraining.size(), - casses.size(), duration); + LOG.debug("[{}][{}][{}]: Training successful on [{}] out of [{}] documents ({} ms)", + getId(), getSessionOwner().getUsername(), recommender.getName(), + cassesForTraining.size(), casses.size(), duration); + log(LogMessage.info(recommender.getName(), "Training successful on [%d] out of [%d] documents (%d ms)", cassesForTraining.size(), casses.size(), duration)); } - private void logTrainingOverallStart(User user) + private void logTrainingOverallStart() { - log.debug("[{}][{}]: Starting training for project {} triggered by [{}]...", getId(), - user.getUsername(), getProject(), getTrigger()); + LOG.debug("[{}][{}]: Starting training for project {} triggered by [{}]...", getId(), + getSessionOwner().getUsername(), getProject(), getTrigger()); info("Starting training triggered by [%s]...", getTrigger()); } - private void logTrainingRecommenderStart(User user, LazyCasLoader aLoader, - AnnotationLayer layer, Recommender recommender, List cassesForTraining) + private void logTrainingRecommenderStart(LazyCasLoader aLoader, Recommender recommender, + List cassesForTraining) throws ConcurrentException { getMonitor().addMessage(LogMessage.info(this, "%s", recommender.getName())); - log.debug("[{}][{}][{}]: Training model on [{}] out of [{}] documents ...", getId(), - user.getUsername(), recommender.getName(), cassesForTraining.size(), + + LOG.debug("[{}][{}][{}]: Training model on [{}] out of [{}] documents ...", getId(), + getSessionOwner().getUsername(), recommender.getName(), cassesForTraining.size(), aLoader.size()); + log(LogMessage.info(recommender.getName(), - "Training model for [%s] on [%d] out of [%d] documents ...", layer.getUiName(), - cassesForTraining.size(), aLoader.size())); + "Training model for [%s] on [%d] out of [%d] documents ...", + recommender.getLayer().getUiName(), cassesForTraining.size(), aLoader.size())); } - private void handleError(User user, Recommender recommender, long startTime, Throwable e) + private void handleError(Recommender recommender, Throwable e) { - long duration = currentTimeMillis() - startTime; - log.error("[{}][{}][{}]: Training failed ({} ms)", getId(), user.getUsername(), - recommender.getName(), (currentTimeMillis() - startTime), e); - - log(LogMessage.error(recommender.getName(), "Training failed (%d ms): %s", duration, - getRootCauseMessage(e))); - - appEventPublisher.publishEvent( - RecommenderTaskNotificationEvent.builder(this, getProject(), user.getUsername()) // - .withMessage(LogMessage.error(this, "Training failed (%d ms) with %s", - duration, e.getMessage())) - .build()); + LOG.error("[{}][{}][{}]: Training failed", getId(), getSessionOwner().getUsername(), + recommender.getName(), e); + + log(LogMessage.error(recommender.getName(), "Training failed: %s", getRootCauseMessage(e))); + + appEventPublisher.publishEvent(RecommenderTaskNotificationEvent + .builder(this, getProject(), getSessionOwner().getUsername()) // + .withMessage(LogMessage.error(this, "Training failed with %s", e.getMessage())) + .build()); } } diff --git a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java index ccb41fbad95..56d656c7cfc 100644 --- a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java +++ b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java @@ -19,31 +19,19 @@ import static de.tudarmstadt.ukp.clarin.webanno.model.OverlapMode.ANY_OVERLAP; import static de.tudarmstadt.ukp.clarin.webanno.model.OverlapMode.NO_OVERLAP; -import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_AUTO_ACCEPT_MODE_SUFFIX; -import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_IS_PREDICTION; -import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_SCORE_EXPLANATION_SUFFIX; -import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_SCORE_SUFFIX; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordChangeLocation.DETAIL_EDITOR; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordChangeLocation.MAIN_EDITOR; import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordUserAction.ACCEPTED; import static java.util.Arrays.asList; import static org.apache.uima.fit.factory.JCasFactory.createJCas; -import static org.apache.uima.fit.factory.JCasFactory.createText; -import static org.apache.uima.util.TypeSystemUtil.typeSystem2TypeSystemDescription; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import org.apache.uima.cas.CAS; -import org.apache.uima.cas.Feature; -import org.apache.uima.cas.Type; -import org.apache.uima.fit.util.CasUtil; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.metadata.TypeSystemDescription; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -57,7 +45,6 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.transaction.annotation.Transactional; -import de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode; import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature; import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; import de.tudarmstadt.ukp.clarin.webanno.model.Project; @@ -67,7 +54,6 @@ import de.tudarmstadt.ukp.inception.annotation.layer.relation.RelationLayerSupport; import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanAdapter; import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport; -import de.tudarmstadt.ukp.inception.annotation.storage.CasStorageSession; import de.tudarmstadt.ukp.inception.recommendation.api.RecommenderFactoryRegistry; import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecord; import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordUserAction; @@ -121,7 +107,7 @@ public void setUp() throws Exception layerRecommendtionSupportRegistry.init(); sut = new RecommendationServiceImpl(null, null, null, recommenderFactoryRegistry, null, - schemaService, null, layerRecommendtionSupportRegistry, + schemaService, layerRecommendtionSupportRegistry, testEntityManager.getEntityManager()); featureSupportRegistry = new FeatureSupportRegistryImpl(asList(new StringFeatureSupport())); @@ -210,38 +196,6 @@ public void getRecommenders_WithOtherRecommenderId_ShouldReturnEmptyList() assertThat(enabledRecommenders).as("Check that no recommender is found").isEmpty(); } - @Test - public void monkeyPatchTypeSystem_WithNer_CreatesScoreFeatures() throws Exception - { - try (CasStorageSession session = CasStorageSession.open()) { - JCas jCas = createText("I am text CAS", "de"); - session.add("jCas", CasAccessMode.EXCLUSIVE_WRITE_ACCESS, jCas.getCas()); - - when(schemaService.getFullProjectTypeSystem(project)) - .thenReturn(typeSystem2TypeSystemDescription(jCas.getTypeSystem())); - when(schemaService.listAnnotationFeature(project)).thenReturn(asList(spanLayerFeature)); - doCallRealMethod().when(schemaService).upgradeCas(any(CAS.class), any(CAS.class), - any(TypeSystemDescription.class)); - - sut.cloneAndMonkeyPatchCAS(project, jCas.getCas(), jCas.getCas()); - - Type type = CasUtil.getType(jCas.getCas(), spanLayer.getName()); - - assertThat(type.getFeatures()) // - .extracting(Feature::getShortName) // - .containsExactlyInAnyOrder( // - "sofa", // - "begin", // - "end", // - "value", // - spanLayerFeature.getName() + FEATURE_NAME_SCORE_SUFFIX, // - spanLayerFeature.getName() + FEATURE_NAME_SCORE_EXPLANATION_SUFFIX, // - spanLayerFeature.getName() + FEATURE_NAME_AUTO_ACCEPT_MODE_SUFFIX, // - "identifier", // - FEATURE_NAME_IS_PREDICTION); - } - } - @Test void testUpsertSpanFeature() throws Exception { diff --git a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java deleted file mode 100644 index 7b6b9bedbc7..00000000000 --- a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Technische Universität Darmstadt under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The Technische Universität Darmstadt - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package de.tudarmstadt.ukp.inception.recommendation.service; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; - -import java.util.Arrays; - -import org.apache.uima.fit.testing.factory.TokenBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; -import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; -import de.tudarmstadt.ukp.clarin.webanno.security.model.User; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; -import de.tudarmstadt.ukp.inception.recommendation.api.model.AnnotationSuggestion; -import de.tudarmstadt.ukp.inception.recommendation.api.model.Offset; -import de.tudarmstadt.ukp.inception.recommendation.api.model.Predictions; -import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; -import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion; -import de.tudarmstadt.ukp.inception.rendering.model.Range; - -class RecommendationServiceImplTest -{ - private TokenBuilder tokenBuilder; - private SourceDocument doc1; - private SourceDocument doc2; - private AnnotationLayer layer1; - - @BeforeEach - void setup() - { - tokenBuilder = new TokenBuilder<>(Token.class, Sentence.class); - doc1 = new SourceDocument("doc1", null, null); - doc1.setId(1l); - doc2 = new SourceDocument("doc2", null, null); - doc2.setId(2l); - layer1 = AnnotationLayer.builder().withId(1l).withName("layer1").build(); - } - - @Test - void testReconciliation() throws Exception - { - var sessionOwner = User.builder().withUsername("user").build(); - var doc = SourceDocument.builder().withName("doc1").build(); - var layer = AnnotationLayer.builder().withId(1l).build(); - var feature = AnnotationFeature.builder().withName("feature").withLayer(layer).build(); - var rec = Recommender.builder().withId(1l).withName("rec").withLayer(layer) - .withFeature(feature).build(); - var project = Project.builder().withId(1l).build(); - - var existingSuggestions = Arrays. asList( // - SpanSuggestion.builder() // - .withId(0) // - .withPosition(0, 10) // - .withDocument(doc) // - .withLabel("aged") // - .withRecommender(rec) // - .build(), - SpanSuggestion.builder() // - .withId(1) // - .withPosition(0, 10) // - .withDocument(doc) // - .withLabel("removed") // - .withRecommender(rec) // - .build()); - var activePredictions = new Predictions(sessionOwner, sessionOwner.getUsername(), project); - activePredictions.inheritSuggestions(existingSuggestions); - - var newSuggestions = Arrays. asList( // - SpanSuggestion.builder() // - .withId(2) // - .withPosition(0, 10) // - .withDocument(doc) // - .withLabel("aged") // - .withRecommender(rec) // - .build(), - SpanSuggestion.builder() // - .withId(3) // - .withPosition(new Offset(0, 10)) // - .withDocument(doc) // - .withLabel("added") // - .withRecommender(rec) // - .build()); - - var result = RecommendationServiceImpl.reconcile(activePredictions, doc, rec, - new Range(0, 10), newSuggestions); - - assertThat(result.suggestions()) // - .extracting(AnnotationSuggestion::getId, AnnotationSuggestion::getLabel, - AnnotationSuggestion::getAge) // - .containsExactlyInAnyOrder(tuple(0, "aged", 1), tuple(3, "added", 0)); - } -} diff --git a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTaskTest.java b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTaskTest.java new file mode 100644 index 00000000000..701e0bacc0c --- /dev/null +++ b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTaskTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.tasks; + +import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_AUTO_ACCEPT_MODE_SUFFIX; +import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_IS_PREDICTION; +import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_SCORE_EXPLANATION_SUFFIX; +import static de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService.FEATURE_NAME_SCORE_SUFFIX; +import static java.util.Arrays.asList; +import static org.apache.uima.fit.factory.JCasFactory.createText; +import static org.apache.uima.util.TypeSystemUtil.typeSystem2TypeSystemDescription; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.when; + +import java.util.Arrays; + +import org.apache.uima.cas.CAS; +import org.apache.uima.cas.Feature; +import org.apache.uima.resource.metadata.TypeSystemDescription; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode; +import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature; +import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.clarin.webanno.security.model.User; +import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; +import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport; +import de.tudarmstadt.ukp.inception.annotation.storage.CasStorageSession; +import de.tudarmstadt.ukp.inception.recommendation.api.model.AnnotationSuggestion; +import de.tudarmstadt.ukp.inception.recommendation.api.model.Offset; +import de.tudarmstadt.ukp.inception.recommendation.api.model.Predictions; +import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; +import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion; +import de.tudarmstadt.ukp.inception.rendering.model.Range; +import de.tudarmstadt.ukp.inception.schema.service.AnnotationSchemaServiceImpl; + +@ExtendWith(MockitoExtension.class) +class PredictionTaskTest +{ + private static final String TRIGGER = "test"; + private static final String DATA_OWNER = "user"; + + private User sessionOwner; + private Project project; + private SourceDocument document; + private AnnotationLayer layer; + private AnnotationFeature feature; + + @BeforeEach + void setup() + { + sessionOwner = User.builder().withUsername("user").build(); + project = Project.builder().build(); + document = SourceDocument.builder().withId(1l).withName("doc1").withProject(project) + .build(); + layer = AnnotationLayer.builder().withId(1l).forJCasClass(NamedEntity.class) + .withType(SpanLayerSupport.TYPE).build(); + feature = AnnotationFeature.builder().withId(1l).withName(NamedEntity._FeatName_value) + .withType(CAS.TYPE_NAME_STRING).withLayer(layer).build(); + } + + @Test + public void monkeyPatchTypeSystem_WithNer_CreatesScoreFeatures() throws Exception + { + var schemaService = Mockito.mock(AnnotationSchemaServiceImpl.class); + + var sut = new PredictionTask(sessionOwner, TRIGGER, document, DATA_OWNER); + sut.setSchemaService(schemaService); + + var jCas = createText("I am text CAS", "de"); + + when(schemaService.getFullProjectTypeSystem(project)) + .thenReturn(typeSystem2TypeSystemDescription(jCas.getTypeSystem())); + when(schemaService.listAnnotationFeature(project)).thenReturn(asList(feature)); + doCallRealMethod().when(schemaService).upgradeCas(any(CAS.class), any(CAS.class), + any(TypeSystemDescription.class)); + + try (var session = CasStorageSession.open()) { + session.add("jCas", CasAccessMode.EXCLUSIVE_WRITE_ACCESS, jCas.getCas()); + sut.cloneAndMonkeyPatchCAS(project, jCas.getCas(), jCas.getCas()); + } + + assertThat(jCas.getTypeSystem().getType(layer.getName())) // + .extracting(Feature::getShortName) // + .containsExactlyInAnyOrder( // + "sofa", // + "begin", // + "end", // + "value", // + feature.getName() + FEATURE_NAME_SCORE_SUFFIX, // + feature.getName() + FEATURE_NAME_SCORE_EXPLANATION_SUFFIX, // + feature.getName() + FEATURE_NAME_AUTO_ACCEPT_MODE_SUFFIX, // + "identifier", // + FEATURE_NAME_IS_PREDICTION); + } + + @Test + void testReconciliation() throws Exception + { + var rec = Recommender.builder().withId(1l).withName("rec").withLayer(layer) + .withFeature(feature).build(); + + var existingSuggestions = Arrays. asList( // + SpanSuggestion.builder() // + .withId(0) // + .withPosition(0, 10) // + .withDocument(document) // + .withLabel("aged") // + .withRecommender(rec) // + .build(), + SpanSuggestion.builder() // + .withId(1) // + .withPosition(0, 10) // + .withDocument(document) // + .withLabel("removed") // + .withRecommender(rec) // + .build()); + var activePredictions = new Predictions(sessionOwner, sessionOwner.getUsername(), project); + activePredictions.inheritSuggestions(existingSuggestions); + + var newSuggestions = Arrays. asList( // + SpanSuggestion.builder() // + .withId(2) // + .withPosition(0, 10) // + .withDocument(document) // + .withLabel("aged") // + .withRecommender(rec) // + .build(), + SpanSuggestion.builder() // + .withId(3) // + .withPosition(new Offset(0, 10)) // + .withDocument(document) // + .withLabel("added") // + .withRecommender(rec) // + .build()); + + var result = PredictionTask.reconcile(activePredictions, document, rec, new Range(0, 10), + newSuggestions); + + assertThat(result.suggestions()) // + .extracting(AnnotationSuggestion::getId, AnnotationSuggestion::getLabel, + AnnotationSuggestion::getAge) // + .containsExactlyInAnyOrder(tuple(0, "aged", 1), tuple(3, "added", 0)); + } +}