From 32c7b34d4c650faf874b5df4cab4bd5e91d78242 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 24 Nov 2020 21:33:49 +0100 Subject: [PATCH 1/5] Enable automated cross library search using a cross library query language. Signed-off-by: Dominik Voigt --- CHANGELOG.md | 1 + build.gradle | 2 + src/main/java/module-info.java | 1 + src/main/java/org/jabref/gui/JabRefFrame.java | 4 +- .../gui/StartLiteratureReviewAction.java | 72 ++++ .../jabref/gui/actions/StandardActions.java | 1 + .../org/jabref/logic/crawler/Crawler.java | 52 +++ .../LibraryEntryToFetcherConverter.java | 65 ++++ .../jabref/logic/crawler/StudyFetcher.java | 80 ++++ .../jabref/logic/crawler/StudyRepository.java | 344 ++++++++++++++++++ .../jabref/logic/crawler/git/GitHandler.java | 83 +++++ .../importer/fetcher/SpringerFetcher.java | 2 +- .../model/entry/types/EntryTypeFactory.java | 1 + ...tematicLiteratureReviewStudyEntryType.java | 33 ++ ...ratureReviewStudyEntryTypeDefinitions.java | 60 +++ .../org/jabref/model/study/FetchResult.java | 24 ++ .../org/jabref/model/study/QueryResult.java | 24 ++ .../java/org/jabref/model/study/Study.java | 98 +++++ .../model/study/StudyMetaDataField.java | 24 ++ src/main/resources/l10n/JabRef_en.properties | 19 +- .../org/jabref/logic/crawler/CrawlerTest.java | 95 +++++ .../LibraryEntryToFetcherConverterTest.java | 61 ++++ .../logic/crawler/StudyRepositoryTest.java | 287 +++++++++++++++ .../SearchBasedFetcherCapabilityTest.java | 2 +- .../org/jabref/model/study/StudyTest.java | 93 +++++ .../jabref/logic/crawler/ArXivQuantumMock.bib | 15 + .../crawler/SpringerCloud ComputingMock.bib | 9 + .../logic/crawler/SpringerQuantumMock.bib | 9 + .../org/jabref/logic/crawler/study.bib | 37 ++ 29 files changed, 1579 insertions(+), 19 deletions(-) create mode 100644 src/main/java/org/jabref/gui/StartLiteratureReviewAction.java create mode 100644 src/main/java/org/jabref/logic/crawler/Crawler.java create mode 100644 src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java create mode 100644 src/main/java/org/jabref/logic/crawler/StudyFetcher.java create mode 100644 src/main/java/org/jabref/logic/crawler/StudyRepository.java create mode 100644 src/main/java/org/jabref/logic/crawler/git/GitHandler.java create mode 100644 src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java create mode 100644 src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java create mode 100644 src/main/java/org/jabref/model/study/FetchResult.java create mode 100644 src/main/java/org/jabref/model/study/QueryResult.java create mode 100644 src/main/java/org/jabref/model/study/Study.java create mode 100644 src/main/java/org/jabref/model/study/StudyMetaDataField.java create mode 100644 src/test/java/org/jabref/logic/crawler/CrawlerTest.java create mode 100644 src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java create mode 100644 src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java create mode 100644 src/test/java/org/jabref/model/study/StudyTest.java create mode 100644 src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/study.bib diff --git a/CHANGELOG.md b/CHANGELOG.md index 38c9e5bd2dc..bfcd7cb3ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ to the page field for cases where the page numbers are missing. [#7019](https:// - We added a new formatter to output shorthand month format. [#6579](https://github.com/JabRef/jabref/issues/6579) - We added support for the new Microsoft Edge browser in all platforms. [#7056](https://github.com/JabRef/jabref/pull/7056) - We reintroduced emacs/bash-like keybindings. [#6017](https://github.com/JabRef/jabref/issues/6017) +- We added a feature to provide automated cross library search using a cross library query language. This provides support for the search step of systematic literature reviews (SLRs). [koppor#369](https://github.com/koppor/jabref/issues/369) ### Changed diff --git a/build.gradle b/build.gradle index 6e04f5b0b22..6e2326393b8 100644 --- a/build.gradle +++ b/build.gradle @@ -139,6 +139,8 @@ dependencies { exclude group: 'org.apache.lucene', module: 'lucene-sandbox' } + implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.9.0.202009080501-r' + implementation group: 'org.mariadb.jdbc', name: 'mariadb-java-client', version: '2.7.0' implementation 'org.postgresql:postgresql:42.2.18' diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index eb7e0102e92..c080fccc99e 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -91,4 +91,5 @@ requires com.h2database.mvstore; requires lucene.queryparser; requires lucene.core; + requires org.eclipse.jgit; } diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 963df97bb57..1b48ad339fc 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -815,7 +815,9 @@ private MenuBar createMenu() { new SeparatorMenuItem(), factory.createMenuItem(StandardActions.SEND_AS_EMAIL, new SendAsEMailAction(dialogService, stateManager)), - pushToApplicationMenuItem + pushToApplicationMenuItem, + new SeparatorMenuItem(), + factory.createMenuItem(StandardActions.START_SYSTEMATIC_LITERATURE_REVIEW, new StartLiteratureReviewAction(this)) ); SidePaneComponent webSearch = sidePaneManager.getComponent(SidePaneType.WEB_SEARCH); diff --git a/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java new file mode 100644 index 00000000000..a641bd1e514 --- /dev/null +++ b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java @@ -0,0 +1,72 @@ +package org.jabref.gui; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; + +import org.jabref.gui.actions.SimpleCommand; +import org.jabref.gui.importer.actions.OpenDatabaseAction; +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.FileDialogConfiguration; +import org.jabref.logic.crawler.Crawler; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.l10n.Localization; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StartLiteratureReviewAction extends SimpleCommand { + private static final Logger LOGGER = LoggerFactory.getLogger(StartLiteratureReviewAction.class); + private final JabRefFrame frame; + private final DialogService dialogService; + + public StartLiteratureReviewAction(JabRefFrame frame) { + this.frame = frame; + this.dialogService = frame.getDialogService(); + } + + @Override + public void execute() { + FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() + .withInitialDirectory(getInitialDirectory()) + .build(); + + Optional studyDefinitionFile = dialogService.showFileOpenDialog(fileDialogConfiguration); + if (studyDefinitionFile.isEmpty()) { + // Do nothing if selection was canceled + return; + } + final Crawler crawler; + try { + crawler = new Crawler(studyDefinitionFile.get(), Globals.getFileUpdateMonitor(), JabRefPreferences.getInstance().getSavePreferences(), Globals.entryTypesManager); + } catch (IOException | ParseException | GitAPIException e) { + LOGGER.info("Error during reading of study definition file.", e); + dialogService.showErrorDialogAndWait(Localization.lang("Error during reading of study definition file.")); + return; + } + BackgroundTask.wrap(() -> { + crawler.performCrawl(); + return 0; // Return any value to make this a callable instead of a runnable. This allows throwing exceptions. + }) + .onFailure(e -> { + LOGGER.info("Error during persistence of crawling results."); + dialogService.showErrorDialogAndWait(Localization.lang("Error during persistence of crawling results."), e); + }) + .onSuccess(unused -> new OpenDatabaseAction(frame).openFile(Path.of(studyDefinitionFile.get().getParent().toString(), "studyResult.bib"), true)) + .executeWith(Globals.TASK_EXECUTOR); + } + + /** + * @return Path of current panel database directory or the working directory + */ + private Path getInitialDirectory() { + if (frame.getBasePanelCount() == 0) { + return Globals.prefs.getWorkingDir(); + } else { + Optional databasePath = frame.getCurrentLibraryTab().getBibDatabaseContext().getDatabasePath(); + return databasePath.map(Path::getParent).orElse(Globals.prefs.getWorkingDir()); + } + } +} diff --git a/src/main/java/org/jabref/gui/actions/StandardActions.java b/src/main/java/org/jabref/gui/actions/StandardActions.java index 2b316dca80a..0767d4021bb 100644 --- a/src/main/java/org/jabref/gui/actions/StandardActions.java +++ b/src/main/java/org/jabref/gui/actions/StandardActions.java @@ -88,6 +88,7 @@ public enum StandardActions implements Action { PARSE_LATEX(Localization.lang("Search for citations in LaTeX files..."), IconTheme.JabRefIcons.LATEX_CITATIONS), NEW_SUB_LIBRARY_FROM_AUX(Localization.lang("New sublibrary based on AUX file") + "...", Localization.lang("New BibTeX sublibrary") + Localization.lang("This feature generates a new library based on which entries are needed in an existing LaTeX document."), IconTheme.JabRefIcons.NEW), WRITE_XMP(Localization.lang("Write XMP metadata to PDFs"), Localization.lang("Will write XMP metadata to the PDFs linked from selected entries."), KeyBinding.WRITE_XMP), + START_SYSTEMATIC_LITERATURE_REVIEW(Localization.lang("Start systematic literature review")), OPEN_DATABASE_FOLDER(Localization.lang("Reveal in file explorer")), OPEN_FOLDER(Localization.lang("Open folder"), Localization.lang("Open folder"), KeyBinding.OPEN_FOLDER), OPEN_FILE(Localization.lang("Open file"), Localization.lang("Open file"), IconTheme.JabRefIcons.FILE, KeyBinding.OPEN_FILE), diff --git a/src/main/java/org/jabref/logic/crawler/Crawler.java b/src/main/java/org/jabref/logic/crawler/Crawler.java new file mode 100644 index 00000000000..f3114100e87 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/Crawler.java @@ -0,0 +1,52 @@ +package org.jabref.logic.crawler; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ParseException; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; + +/** + * This class provides a service for SLR support by conducting an automated search and persistance + * of studies using the queries and E-Libraries specified in the provided study definition file. + * + * It composes a StudyRepository for repository management, + * and a StudyFetcher that manages the crawling over the selected E-Libraries. + */ +public class Crawler { + private final StudyRepository studyRepository; + private final StudyFetcher studyFetcher; + + /** + * Creates a crawler for retrieving studies from E-Libraries + * + * @param studyDefinitionFile The path to the study definition file that contains the list of targeted E-Libraries and used cross-library queries + */ + public Crawler(Path studyDefinitionFile, FileUpdateMonitor fileUpdateMonitor, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IllegalArgumentException, IOException, ParseException, GitAPIException { + Path studyRepositoryRoot = studyDefinitionFile.getParent(); + studyRepository = new StudyRepository(studyRepositoryRoot, new GitHandler(studyRepositoryRoot), JabRefPreferences.getInstance().getImportFormatPreferences(), fileUpdateMonitor, savePreferences, bibEntryTypesManager); + Study study = studyRepository.getStudy(); + LibraryEntryToFetcherConverter libraryEntryToFetcherConverter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), JabRefPreferences.getInstance().getImportFormatPreferences()); + this.studyFetcher = new StudyFetcher(libraryEntryToFetcherConverter.getActiveFetchers(), study.getSearchQueryStrings()); + } + + /** + * This methods performs the crawling of the active libraries defined in the study definition file. + * This method also persists the results in the same folder the study definition file is stored in. + * + * @throws IOException Thrown if a problem occurred during the persistence of the result. + */ + public void performCrawl() throws IOException, GitAPIException { + List results = studyFetcher.crawl(); + studyRepository.persist(results); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java new file mode 100644 index 00000000000..ee2f0f00c71 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java @@ -0,0 +1,65 @@ +package org.jabref.logic.crawler; + +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.WebFetchers; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.UnknownField; + +/** + * Converts library entries from the given study into their corresponding fetchers. + */ +class LibraryEntryToFetcherConverter { + private final List libraryEntries; + private final ImportFormatPreferences importFormatPreferences; + + public LibraryEntryToFetcherConverter(List libraryEntries, ImportFormatPreferences importFormatPreferences) { + this.libraryEntries = libraryEntries; + this.importFormatPreferences = importFormatPreferences; + } + + /** + * Returns a list of instances of all active library fetchers. + * + * A fetcher is considered active if there exists an library entry of the library the fetcher is associated with that is enabled. + * + * @return Instances of all active fetchers defined in the study definition. + */ + public List getActiveFetchers() { + return getFetchersFromLibraryEntries(this.libraryEntries); + } + + /** + * Transforms a list of libraryEntries into a list of SearchBasedFetcher instances. + * + * @param libraryEntries List of entries + * @return List of fetcher instances + */ + private List getFetchersFromLibraryEntries(List libraryEntries) { + return libraryEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().getName().equals("library")) + .map(this::createFetcherFromLibraryEntry) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + /** + * Transforms a library entry into a SearchBasedFetcher instance. This only works if the library entry specifies a supported fetcher. + * + * @param libraryEntry the entry that will be converted + * @return An instance of the fetcher defined by the library entry. + */ + private SearchBasedFetcher createFetcherFromLibraryEntry(BibEntry libraryEntry) { + Set searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences); + String libraryNameFromFetcher = libraryEntry.getField(new UnknownField("name")).orElse(""); + return searchBasedFetchers.stream() + .filter(searchBasedFetcher -> searchBasedFetcher.getName().toLowerCase().equals(libraryNameFromFetcher.toLowerCase())) + .findAny() + .orElse(null); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java new file mode 100644 index 00000000000..c39ba7efe52 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java @@ -0,0 +1,80 @@ +package org.jabref.logic.crawler; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Delegates the search of the provided set of targeted E-Libraries with the provided queries to the E-Library specific fetchers, + * and aggregates the results returned by the fetchers by query and E-Library. + */ +class StudyFetcher { + private static final Logger LOGGER = LoggerFactory.getLogger(StudyFetcher.class); + private static final int MAX_AMOUNT_OF_RESULTS_PER_FETCHER = 100; + + private final List activeFetchers; + private final List searchQueries; + + StudyFetcher(List activeFetchers, List searchQueries) throws IllegalArgumentException { + this.searchQueries = searchQueries; + this.activeFetchers = activeFetchers; + } + + /** + * Each Map Entry contains the results for one search term for all libraries. + * Each entry of the internal map contains the results for a given library. + * If any library API is not available, its corresponding entry is missing from the internal map. + */ + public List crawl() { + return searchQueries.parallelStream() + .map(this::getQueryResult) + .collect(Collectors.toList()); + } + + private QueryResult getQueryResult(String searchQuery) { + return new QueryResult(searchQuery, performSearchOnQuery(searchQuery)); + } + + /** + * Queries all Databases on the given searchQuery. + * + * @param searchQuery The query the search is performed for. + * @return Mapping of each fetcher by name and all their retrieved publications as a BibDatabase + */ + private List performSearchOnQuery(String searchQuery) { + return activeFetchers.parallelStream() + .map(fetcher -> performSearchOnQueryForFetcher(searchQuery, fetcher)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private FetchResult performSearchOnQueryForFetcher(String searchQuery, SearchBasedFetcher fetcher) { + try { + List fetchResult = new ArrayList<>(); + if (fetcher instanceof PagedSearchBasedFetcher) { + int pages = ((int) Math.ceil(((double) MAX_AMOUNT_OF_RESULTS_PER_FETCHER) / ((PagedSearchBasedFetcher) fetcher).getPageSize())); + for (int page = 0; page < pages; page++) { + fetchResult.addAll(((PagedSearchBasedFetcher) fetcher).performSearchPaged(searchQuery, page).getContent()); + } + } else { + fetchResult = fetcher.performSearch(searchQuery); + } + return new FetchResult(fetcher.getName(), new BibDatabase(fetchResult)); + } catch (FetcherException e) { + LOGGER.warn(String.format("%s API request failed", fetcher.getName()), e); + return null; + } + } +} diff --git a/src/main/java/org/jabref/logic/crawler/StudyRepository.java b/src/main/java/org/jabref/logic/crawler/StudyRepository.java new file mode 100644 index 00000000000..6b16d1e07dc --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/StudyRepository.java @@ -0,0 +1,344 @@ +package org.jabref.logic.crawler; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.logic.citationkeypattern.CitationKeyGenerator; +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.database.DatabaseMerger; +import org.jabref.logic.exporter.BibtexDatabaseWriter; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.OpenDatabase; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class manages all aspects of the study process related to the repository. + * + * It includes the parsing of the study definition file (study.bib) into a Study instance, + * the structured persistence of the crawling results for the study within the file based repository, + * as well as the sharing, and versioning of results using git. + */ +class StudyRepository { + // Tests work with study.bib + private static final String STUDY_DEFINITION_FILE_NAME = "study.bib"; + private static final Logger LOGGER = LoggerFactory.getLogger(StudyRepository.class); + + private final Path repositoryPath; + private final Path studyDefinitionBib; + private final GitHandler gitHandler; + private final Study study; + private final ImportFormatPreferences importFormatPreferences; + private final FileUpdateMonitor fileUpdateMonitor; + private final SavePreferences savePreferences; + private final BibEntryTypesManager bibEntryTypesManager; + + /** + * Creates a study repository. + * + * @param pathToRepository Where the repository root is located. + * @param gitHandler The git handler that managages any interaction with the remote repository + * @throws IllegalArgumentException If the repository root directory does not exist, or the root directory does not contain the study definition file. + * @throws IOException Thrown if the given repository does not exists, or the study definition file does not exist + * @throws ParseException Problem parsing the study definition file. + */ + public StudyRepository(Path pathToRepository, GitHandler gitHandler, ImportFormatPreferences importFormatPreferences, FileUpdateMonitor fileUpdateMonitor, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IOException, ParseException, GitAPIException { + this.repositoryPath = pathToRepository; + this.gitHandler = gitHandler; + try { + gitHandler.updateLocalRepository(); + } catch (GitAPIException e) { + LOGGER.info("Updating repository from remote failed"); + } + this.importFormatPreferences = importFormatPreferences; + this.fileUpdateMonitor = fileUpdateMonitor; + this.studyDefinitionBib = Path.of(repositoryPath.toString(), STUDY_DEFINITION_FILE_NAME); + this.savePreferences = savePreferences; + this.bibEntryTypesManager = bibEntryTypesManager; + + if (Files.notExists(repositoryPath)) { + throw new IOException("The given repository does not exists."); + } else if (Files.notExists(studyDefinitionBib)) { + throw new IOException("The study definition file does not exist in the given repository."); + } + study = parseStudyFile(); + this.setUpRepositoryStructure(); + } + + /** + * Returns entries stored in the repository for a certain query and fetcher + */ + public BibDatabaseContext getFetcherResultEntries(String query, String fetcherName) throws IOException { + return OpenDatabase.loadDatabase(getPathToFetcherResultFile(query, fetcherName), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * Returns the merged entries stored in the repository for a certain query + */ + public BibDatabaseContext getQueryResultEntries(String query) throws IOException { + return OpenDatabase.loadDatabase(getPathToQueryResultFile(query), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * Returns the merged entries stored in the repository for all queries + */ + public BibDatabaseContext getStudyResultEntries() throws IOException { + return OpenDatabase.loadDatabase(getPathToStudyResultFile(), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * The study definition file contains all the definitions of a study. This method extracts the BibEntries from the study BiB file. + * + * @return Returns the BibEntries parsed from the study definition file. + * @throws IOException Problem opening the input stream. + * @throws ParseException Problem parsing the study definition file. + */ + private Study parseStudyFile() throws IOException, ParseException { + BibtexParser parser = new BibtexParser(importFormatPreferences, fileUpdateMonitor); + List parsedEntries = new ArrayList<>(); + try (InputStream inputStream = Files.newInputStream(studyDefinitionBib)) { + parsedEntries.addAll(parser.parseEntries(inputStream)); + } + + BibEntry studyEntry = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY)).findAny() + .orElseThrow(() -> new ParseException("Study definition file does not contain a study entry")); + List queryEntries = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY)) + .collect(Collectors.toList()); + List libraryEntries = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY)) + .collect(Collectors.toList()); + + return new Study(studyEntry, queryEntries, libraryEntries); + } + + public Study getStudy() { + return study; + } + + public void persist(List crawlResults) throws IOException, GitAPIException { + try { + gitHandler.updateLocalRepository(); + } catch (GitAPIException e) { + LOGGER.info("Updating repository from remote failed"); + } + persistResults(crawlResults); + study.setLastSearchDate(LocalDate.now()); + persistStudy(); + try { + gitHandler.updateRemoteRepository("Conducted search " + LocalDate.now()); + } catch (GitAPIException e) { + LOGGER.info("Updating remote repository failed"); + } + } + + private void persistStudy() throws IOException { + writeResultToFile(studyDefinitionBib, new BibDatabase(study.getAllEntries())); + } + + /** + * Create for each query a folder, and for each fetcher a bib file in the query folder to store its results. + */ + private void setUpRepositoryStructure() throws IOException { + // Cannot use stream here since IOException has to be thrown + LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); + for (String query : study.getSearchQueryStrings()) { + createQueryResultFolder(query); + converter.getActiveFetchers() + .forEach(searchBasedFetcher -> createFetcherResultFile(query, searchBasedFetcher)); + createQueryResultFile(query); + } + createStudyResultFile(); + } + + /** + * Creates a folder using the query and its corresponding query id. + * This folder name is unique for each query, as long as the query id in the study definition is unique for each query. + * + * @param query The query the folder is created for + */ + private void createQueryResultFolder(String query) throws IOException { + Path queryResultFolder = getPathToQueryDirectory(query); + createFolder(queryResultFolder); + } + + private void createFolder(Path folder) throws IOException { + if (Files.notExists(folder)) { + try { + Files.createDirectory(folder); + } catch (IOException e) { + throw new IOException("Error during creation of repository structure.", e); + } + } + } + + private void createFetcherResultFile(String query, SearchBasedFetcher searchBasedFetcher) { + String fetcherName = searchBasedFetcher.getName(); + Path fetcherResultFile = getPathToFetcherResultFile(query, fetcherName); + createBibFile(fetcherResultFile); + } + + private void createQueryResultFile(String query) { + Path queryResultFile = getPathToFetcherResultFile(query, "result"); + createBibFile(queryResultFile); + } + + private void createStudyResultFile() { + createBibFile(getPathToStudyResultFile()); + } + + private void createBibFile(Path file) { + if (Files.notExists(file)) { + try { + Files.createFile(file); + } catch (IOException e) { + throw new IllegalStateException("Error during creation of repository structure.", e); + } + } + } + + /** + * Returns a string that can be used as a folder name. + * This removes all characters from the query that are illegal for directory names. + * Structure: ID-trimmed query + * + * Examples: + * Input: '(title: test-title AND abstract: Test)' as a query entry with id 1 + * Output: '1 - title= test-title AND abstract= Test' + * + * Input: 'abstract: Test*' as a query entry with id 1 + * Output: '1 - abstract= Test' + * + * Input: '"test driven"' as a query entry with id 1 + * Output: '1 - test driven' + * + * @param query that is trimmed and combined with its query id + * @return a unique folder name for any query. + */ + private String trimNameAndAddID(String query) { + // Replace all field: with field= for folder name + String trimmedNamed = query.replaceAll(":", "="); + trimmedNamed = trimmedNamed.replaceAll("[^A-Za-z0-9_.\\s=-]", ""); + if (query.length() > 240) { + trimmedNamed = query.substring(0, 240); + } + String id = findQueryIDByQueryString(query); + return id + " - " + trimmedNamed; + } + + /** + * Helper to find the query id for folder name creation. + * Returns the id of the first SearchQuery BibEntry with a query field that matches the given query. + * + * @param query The query whose ID is searched + * @return ID of the query defined in the study definition. + */ + private String findQueryIDByQueryString(String query) { + return study.getSearchQueryEntries() + .parallelStream() + .filter(bibEntry -> bibEntry.getField(new UnknownField("query")).orElse("").equals(query)) + .map(BibEntry::getCitationKey) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst() + .orElseThrow() + .replaceFirst("query", ""); + } + + /** + * Persists the crawling results in the local file based repository. + * + * @param crawlResults The results that shall be persisted. + */ + private void persistResults(List crawlResults) throws IOException { + DatabaseMerger merger = new DatabaseMerger(); + BibDatabase newStudyResultEntries = new BibDatabase(); + + for (QueryResult result : crawlResults) { + BibDatabase queryResultEntries = new BibDatabase(); + for (FetchResult fetcherResult : result.getResultsPerFetcher()) { + BibDatabase fetcherEntries = fetcherResult.getFetchResult(); + BibDatabaseContext existingFetcherResult = getFetcherResultEntries(result.getQuery(), fetcherResult.getFetcherName()); + + // Create citation keys for all entries that do not have one + generateCiteKeys(existingFetcherResult, fetcherEntries); + + // Merge new entries into fetcher result file + merger.merge(existingFetcherResult.getDatabase(), fetcherEntries); + // Aggregate each fetcher result into the query result + merger.merge(queryResultEntries, fetcherEntries); + + writeResultToFile(getPathToFetcherResultFile(result.getQuery(), fetcherResult.getFetcherName()), existingFetcherResult.getDatabase()); + } + BibDatabase existingQueryEntries = getQueryResultEntries(result.getQuery()).getDatabase(); + + // Merge new entries into query result file + merger.merge(existingQueryEntries, queryResultEntries); + // Aggregate all new entries for every query into the study result + merger.merge(newStudyResultEntries, queryResultEntries); + + writeResultToFile(getPathToQueryResultFile(result.getQuery()), existingQueryEntries); + } + BibDatabase existingStudyResultEntries = getStudyResultEntries().getDatabase(); + + // Merge new entries into study result file + merger.merge(existingStudyResultEntries, newStudyResultEntries); + + writeResultToFile(getPathToStudyResultFile(), existingStudyResultEntries); + } + + private void generateCiteKeys(BibDatabaseContext existingEntries, BibDatabase targetEntries) { + CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(existingEntries, JabRefPreferences.getInstance().getCitationKeyPatternPreferences()); + targetEntries.getEntries().stream().filter(bibEntry -> !bibEntry.hasCitationKey()).forEach(citationKeyGenerator::generateAndSetKey); + } + + private void writeResultToFile(Path pathToFile, BibDatabase entries) throws IOException { + try (Writer fileWriter = new FileWriter(pathToFile.toFile())) { + BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(fileWriter, savePreferences, bibEntryTypesManager); + databaseWriter.saveDatabase(new BibDatabaseContext(entries)); + } + } + + private Path getPathToFetcherResultFile(String query, String fetcherName) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query), fetcherName + ".bib"); + } + + private Path getPathToQueryResultFile(String query) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query), "result.bib"); + } + + private Path getPathToStudyResultFile() { + return Path.of(repositoryPath.toString(), "studyResult.bib"); + } + + private Path getPathToQueryDirectory(String query) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query)); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/git/GitHandler.java b/src/main/java/org/jabref/logic/crawler/git/GitHandler.java new file mode 100644 index 00000000000..439f08dfccd --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/git/GitHandler.java @@ -0,0 +1,83 @@ +package org.jabref.logic.crawler.git; + +import java.io.IOException; +import java.nio.file.Path; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.Status; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class handles the updating of the local and remote git repository that is located at the repository path + */ +public class GitHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(GitHandler.class); + private final Path repositoryPath; + private final CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(System.getenv("GIT_EMAIL"), System.getenv("GIT_PW")); + + /** + * Initialize the handler for the given repository + * + * @param repositoryPath The root of the intialized git repository + */ + public GitHandler(Path repositoryPath) { + this.repositoryPath = repositoryPath; + } + + /** + * Updates the local repository based on the main branch of the original remote repository + */ + public void updateLocalRepository() throws IOException, GitAPIException { + try (Git git = Git.open(this.repositoryPath.toFile())) { + git.pull() + .setRemote("origin") + .setRemoteBranchName("main") + .setCredentialsProvider(credentialsProvider) + .call(); + } + } + + /** + * Adds all the added, changed, and removed files to the index and updates the remote origin repository + * If pushiong to remote fails it fails silently + * + * @param commitMessage The commit message used for the commit to the remote repository + */ + public void updateRemoteRepository(String commitMessage) throws IOException, GitAPIException { + // First get up to date + this.updateLocalRepository(); + try (Git git = Git.open(this.repositoryPath.toFile())) { + Status status = git.status().call(); + if (!status.isClean()) { + // Add new and changed files to index + git.add() + .addFilepattern(".") + .call(); + // Add all removed files to index + if (!status.getMissing().isEmpty()) { + RmCommand removeCommand = git.rm() + .setCached(true); + status.getMissing().forEach(removeCommand::addFilepattern); + removeCommand.call(); + } + git.commit() + .setAllowEmpty(false) + .setMessage(commitMessage) + .call(); + try { + + git.push() + .setCredentialsProvider(credentialsProvider) + .call(); + } catch (GitAPIException e) { + LOGGER.info("Failed to push"); + } + } + } + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index a547dbe2175..1064a7f272e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -178,7 +178,7 @@ private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery complexSearchQuery.getTitlePhrases().forEach(title -> searchTerms.add("title:" + title)); complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("journal:" + journal)); // Since Springer API does not support year range search, we ignore formYear and toYear and use "singleYear" only - complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add("year:" + year.toString())); + complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add("date:" + year.toString() + "*")); searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); return String.join(" AND ", searchTerms); } diff --git a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java index 1ecf238382f..29422891f4e 100644 --- a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java +++ b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java @@ -50,6 +50,7 @@ public static EntryType parse(String typeName) { List types = new ArrayList<>(Arrays.asList(StandardEntryType.values())); types.addAll(Arrays.asList(IEEETranEntryType.values())); + types.addAll(Arrays.asList(SystematicLiteratureReviewStudyEntryType.values())); return types.stream().filter(type -> type.getName().equals(typeName.toLowerCase(Locale.ENGLISH))).findFirst().orElse(new UnknownEntryType(typeName)); } diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java new file mode 100644 index 00000000000..1d9bd4be112 --- /dev/null +++ b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java @@ -0,0 +1,33 @@ +package org.jabref.model.entry.types; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; + +public enum SystematicLiteratureReviewStudyEntryType implements EntryType { + STUDY_ENTRY("Study"), + SEARCH_QUERY_ENTRY("SearchQuery"), + LIBRARY_ENTRY("Library"); + + private final String displayName; + + SystematicLiteratureReviewStudyEntryType(String displayName) { + this.displayName = displayName; + } + + public static Optional fromName(String name) { + return Arrays.stream(SystematicLiteratureReviewStudyEntryType.values()) + .filter(field -> field.getName().equalsIgnoreCase(name)) + .findAny(); + } + + @Override + public String getName() { + return displayName.toLowerCase(Locale.ENGLISH); + } + + @Override + public String getDisplayName() { + return displayName; + } +} diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java new file mode 100644 index 00000000000..e9f31b2e630 --- /dev/null +++ b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java @@ -0,0 +1,60 @@ +package org.jabref.model.entry.types; + +import java.util.Arrays; +import java.util.List; + +import org.jabref.model.entry.BibEntryType; +import org.jabref.model.entry.BibEntryTypeBuilder; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; + +/** + * This class represents all supported entry types used in a study definition file + */ +public class SystematicLiteratureReviewStudyEntryTypeDefinitions { + + /** + * Entry type used for study meta data within a study definition file + * + *
    + *
  • Required fields: author, lastsearchdate, name, enabled
    • + *
  • Optional fields:
    • + *
+ */ + private static final BibEntryType STUDY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) + .withRequiredFields(StandardField.AUTHOR, new UnknownField("lastsearchdate"), new UnknownField("name"), new UnknownField("researchquestions")) + .build(); + + /** + * Entry type for the queries within the study definition file + * + *
    + *
  • Required fields: query
    • + *
  • Optional fields:
    • + *
+ */ + private static final BibEntryType SEARCH_QUERY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY) + .withRequiredFields(new UnknownField("query")) + .build(); + + /** + * Entry type for the targeted libraries within a study definition file + * + *
    + *
  • Required fields: name, enabled
    • + *
  • Optional fields: comment
    • + *
+ */ + private static final BibEntryType LIBRARY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) + .withRequiredFields(new UnknownField("name"), new UnknownField("enabled")) + .withImportantFields(new UnknownField("comment")) + .build(); + + public static final List ALL = Arrays.asList(STUDY_ENTRY, SEARCH_QUERY_ENTRY, LIBRARY_ENTRY); + + private SystematicLiteratureReviewStudyEntryTypeDefinitions() { + } +} diff --git a/src/main/java/org/jabref/model/study/FetchResult.java b/src/main/java/org/jabref/model/study/FetchResult.java new file mode 100644 index 00000000000..80637feb4ab --- /dev/null +++ b/src/main/java/org/jabref/model/study/FetchResult.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import org.jabref.model.database.BibDatabase; + +/** + * Represents the result of fetching the results for a query for a specific library + */ +public class FetchResult { + private final String fetcherName; + private final BibDatabase fetchResult; + + public FetchResult(String fetcherName, BibDatabase fetcherResult) { + this.fetcherName = fetcherName; + this.fetchResult = fetcherResult; + } + + public String getFetcherName() { + return fetcherName; + } + + public BibDatabase getFetchResult() { + return fetchResult; + } +} diff --git a/src/main/java/org/jabref/model/study/QueryResult.java b/src/main/java/org/jabref/model/study/QueryResult.java new file mode 100644 index 00000000000..2976b5224fe --- /dev/null +++ b/src/main/java/org/jabref/model/study/QueryResult.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import java.util.List; + +/** + * Represents the result of fetching the results from all active fetchers for a specific query. + */ +public class QueryResult { + private final String query; + private final List resultsPerLibrary; + + public QueryResult(String query, List resultsPerLibrary) { + this.query = query; + this.resultsPerLibrary = resultsPerLibrary; + } + + public String getQuery() { + return query; + } + + public List getResultsPerFetcher() { + return resultsPerLibrary; + } +} diff --git a/src/main/java/org/jabref/model/study/Study.java b/src/main/java/org/jabref/model/study/Study.java new file mode 100644 index 00000000000..37ed6e2328a --- /dev/null +++ b/src/main/java/org/jabref/model/study/Study.java @@ -0,0 +1,98 @@ +package org.jabref.model.study; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.UnknownField; + +/** + * This class represents a scientific study. + * + * This class defines all aspects of a scientific study relevant to the application. It is a proxy for the file based study definition. + */ +public class Study { + private static final String SEARCH_QUERY_FIELD_NAME = "query"; + + private final BibEntry studyEntry; + private final List queryEntries; + private final List libraryEntries; + + public Study(BibEntry studyEntry, List queryEntries, List libraryEntries) { + this.studyEntry = studyEntry; + this.queryEntries = queryEntries; + this.libraryEntries = libraryEntries; + } + + public List getAllEntries() { + List allEntries = new ArrayList<>(); + allEntries.add(studyEntry); + allEntries.addAll(queryEntries); + allEntries.addAll(libraryEntries); + return allEntries; + } + + /** + * Returns all query strings + * + * @return List of all queries as Strings. + */ + public List getSearchQueryStrings() { + return queryEntries.parallelStream() + .map(bibEntry -> bibEntry.getField(new UnknownField(SEARCH_QUERY_FIELD_NAME))) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * This method returns the SearchQuery entries. + * This is required when the BibKey of the search term entry is required in combination with the search query (e.g. + * for the creation of the study repository structure). + */ + public List getSearchQueryEntries() { + return queryEntries; + } + + /** + * Returns a meta data entry of the first study entry found in the study definition file of the provided type. + * + * @param metaDataField The type of requested meta-data + * @return returns the requested meta data type of the first found study entry + * @throws IllegalArgumentException If the study file does not contain a study entry. + */ + public Optional getStudyMetaDataField(StudyMetaDataField metaDataField) throws IllegalArgumentException { + return studyEntry.getField(metaDataField.toField()); + } + + /** + * Sets the lastSearchDate field of the study entry + * + * @param date date the last time a search was conducted + */ + public void setLastSearchDate(LocalDate date) { + studyEntry.setField(StudyMetaDataField.STUDY_LAST_SEARCH.toField(), date.toString()); + } + + /** + * Extracts all active LibraryEntries from the BibEntries. + * + * @return List of BibEntries of type Library + * @throws IllegalArgumentException If a transformation from Library entry to LibraryDefinition fails + */ + public List getActiveLibraryEntries() throws IllegalArgumentException { + return libraryEntries + .parallelStream() + .filter(bibEntry -> { + // If enabled is not defined, the fetcher is active. + return bibEntry.getField(new UnknownField("enabled")) + .map(enabled -> enabled.equals("true")) + .orElse(true); + }) + .collect(Collectors.toList()); + } +} + diff --git a/src/main/java/org/jabref/model/study/StudyMetaDataField.java b/src/main/java/org/jabref/model/study/StudyMetaDataField.java new file mode 100644 index 00000000000..6dbea2a2dc8 --- /dev/null +++ b/src/main/java/org/jabref/model/study/StudyMetaDataField.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; + +/** + * This enum represents the different fields in the study entry + */ +public enum StudyMetaDataField { + STUDY_NAME(new UnknownField("name")), STUDY_RESEARCH_QUESTIONS(new UnknownField("researchQuestions")), + STUDY_AUTHORS(StandardField.AUTHOR), STUDY_GIT_REPOSITORY(new UnknownField("gitRepositoryURL")), + STUDY_LAST_SEARCH(new UnknownField("lastSearchDate")); + + private final Field field; + + StudyMetaDataField(Field field) { + this.field = field; + } + + public Field toField() { + return this.field; + } +} diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index b58510e6d5c..d66d978e315 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -295,32 +295,21 @@ Entry\ owner=Entry owner Entry\ preview=Entry preview Entry\ table=Entry table - Entry\ table\ columns=Entry table columns Entry\ Title\ (Required\ to\ deliver\ recommendations.)=Entry Title (Required to deliver recommendations.) - Entry\ type=Entry type - Error=Error - Error\ occurred\ when\ parsing\ entry=Error occurred when parsing entry - Error\ opening\ file=Error opening file - Error\ while\ writing=Error while writing - +Error\ during\ persistence\ of\ crawling\ results.=Error during persistence of crawling results. +Error\ during\ reading\ of\ study\ definition\ file.=Error during reading of study definition file. '%0'\ exists.\ Overwrite\ file?='%0' exists. Overwrite file? - Export=Export - Export\ preferences=Export preferences - Export\ preferences\ to\ file=Export preferences to file - Export\ to\ clipboard=Export to clipboard - Export\ to\ text\ file.=Export to text file. - Exporting=Exporting Extension=Extension @@ -644,11 +633,9 @@ Previous\ preview\ layout=Previous preview layout Available=Available Selected=Selected Selected\ Layouts\ can\ not\ be\ empty=Selected Layouts can not be empty - +Start\ systematic\ literature\ review=Start systematic literature review Reset\ default\ preview\ style=Reset default preview style - Previous\ entry=Previous entry - Primary\ sort\ criterion=Primary sort criterion Problem\ with\ parsing\ entry=Problem with parsing entry Processing\ %0=Processing %0 diff --git a/src/test/java/org/jabref/logic/crawler/CrawlerTest.java b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java new file mode 100644 index 00000000000..daeb90b4f07 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java @@ -0,0 +1,95 @@ +package org.jabref.logic.crawler; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.eclipse.jgit.api.Git; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Integration test of the components used for SLR support + */ +class CrawlerTest { + @TempDir + Path tempRepositoryDirectory; + SavePreferences preferences; + BibEntryTypesManager entryTypesManager; + + @Test + public void testWhetherAllFilesAreCreated() throws Exception { + setUp(); + Crawler testCrawler = new Crawler(getPathToStudyDefinitionFile(), + new DummyFileUpdateMonitor(), + preferences, + entryTypesManager + ); + + testCrawler.performCrawl(); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "studyResult.bib"))); + } + + private Path getPathToStudyDefinitionFile() { + return tempRepositoryDirectory.resolve("study.bib"); + } + + /** + * Set up mocks and copies the study definition file into the test repository + */ + private void setUp() throws Exception { + setUpRepository(); + preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(preferences.getEncoding()).thenReturn(null); + when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + entryTypesManager = new BibEntryTypesManager(); + } + + private void setUpRepository() throws Exception { + Git git = Git.init() + .setDirectory(tempRepositoryDirectory.toFile()) + .call(); + setUpTestStudyDefinitionFile(); + git.add() + .addFilepattern(".") + .call(); + git.commit() + .setMessage("Initialize") + .call(); + git.close(); + } + + private void setUpTestStudyDefinitionFile() throws Exception { + Path destination = tempRepositoryDirectory.resolve("study.bib"); + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } +} diff --git a/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java new file mode 100644 index 00000000000..cce997fe564 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java @@ -0,0 +1,61 @@ +package org.jabref.logic.crawler; + +import java.net.URL; +import java.nio.file.Path; +import java.util.List; + +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.study.Study; +import org.jabref.model.util.DummyFileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class LibraryEntryToFetcherConverterTest { + SavePreferences preferences; + BibEntryTypesManager entryTypesManager; + GitHandler gitHandler; + @TempDir + Path tempRepositoryDirectory; + + @BeforeEach + void setUpMocks() { + preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(preferences.getEncoding()).thenReturn(null); + when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + entryTypesManager = new BibEntryTypesManager(); + gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + } + + @Test + public void getActiveFetcherInstances() throws Exception { + Path studyDefinition = tempRepositoryDirectory.resolve("study.bib"); + copyTestStudyDefinitionFileIntoDirectory(studyDefinition); + + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager).getStudy(); + LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), JabRefPreferences.getInstance().getImportFormatPreferences()); + List result = converter.getActiveFetchers(); + + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(result.get(0).getName(), "Springer"); + Assertions.assertEquals(result.get(1).getName(), "ArXiv"); + } + + private void copyTestStudyDefinitionFileIntoDirectory(Path destination) throws Exception { + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } +} diff --git a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java new file mode 100644 index 00000000000..72717b2a0a2 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java @@ -0,0 +1,287 @@ +package org.jabref.logic.crawler; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.jabref.logic.citationkeypattern.CitationKeyGenerator; +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.database.DatabaseMerger; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.study.StudyMetaDataField; +import org.jabref.model.util.DummyFileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StudyRepositoryTest { + private static final String NON_EXISTING_DIRECTORY = "nonExistingTestRepositoryDirectory"; + SavePreferences preferences; + BibEntryTypesManager entryTypesManager; + @TempDir + Path tempRepositoryDirectory; + StudyRepository studyRepository; + GitHandler gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + + @Test + void providePathToNonExistentRepositoryThrowsException() { + Path nonExistingRepositoryDirectory = tempRepositoryDirectory.resolve(NON_EXISTING_DIRECTORY); + + assertThrows(IOException.class, () -> new StudyRepository(nonExistingRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager)); + } + + @Test + void providePathToExistentRepositoryWithOutStudyDefinitionFileThrowsException() { + assertThrows(IOException.class, () -> new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager)); + } + + /** + * Tests whether the StudyRepository correctly imports the study file. + */ + @Test + void studyFileCorrectlyImported() throws Exception { + setUpTestRepository(); + List expectedSearchterms = List.of("Quantum", "Cloud Computing", "TestSearchQuery3"); + List expectedActiveFetchersByName = List.of("Springer", "ArXiv"); + + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager).getStudy(); + + assertEquals(expectedSearchterms, study.getSearchQueryStrings()); + assertEquals("TestStudyName", study.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); + assertEquals("Jab Ref", study.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); + assertEquals("Question1; Question2", study.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); + assertEquals(expectedActiveFetchersByName, study.getActiveLibraryEntries() + .stream() + .filter(bibEntry -> bibEntry.getType().getName().equals("library")) + .map(bibEntry -> bibEntry.getField(new UnknownField("name")).orElse("")) + .collect(Collectors.toList()) + ); + } + + /** + * Tests whether the file structure of the repository is created correctly from the study definitions file. + */ + @Test + void repositoryStructureCorrectlyCreated() throws Exception { + // When repository is instantiated the directory structure is created + getTestStudyRepository(); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "IEEEXplore.bib"))); + } + + /** + * This tests whether the repository returns the stored bib entries correctly. + */ + @Test + void bibEntriesCorrectlyStored() throws Exception { + StudyRepository repository = getTestStudyRepository(); + setUpTestResultFile(); + List result = repository.getFetcherResultEntries("Quantum", "ArXiv").getEntries(); + assertEquals(getArXivQuantumMockResults(), result); + } + + @Test + void fetcherResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(getArXivQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "ArXiv").getEntries()); + assertEquals(getSpringerQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "Springer").getEntries()); + assertEquals(getSpringerCloudComputingMockResults(), getTestStudyRepository().getFetcherResultEntries("Cloud Computing", "Springer").getEntries()); + } + + @Test + void mergedResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + List expected = new ArrayList<>(); + expected.addAll(getArXivQuantumMockResults()); + expected.add(getSpringerQuantumMockResults().get(1)); + expected.add(getSpringerQuantumMockResults().get(2)); + + getTestStudyRepository().persist(mockResults); + + // All Springer results are duplicates for "Quantum" + assertEquals(expected, getTestStudyRepository().getQueryResultEntries("Quantum").getEntries()); + assertEquals(getSpringerCloudComputingMockResults(), getTestStudyRepository().getQueryResultEntries("Cloud Computing").getEntries()); + } + + @Test + void setsLastSearchDatePersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(LocalDate.now().toString(), getTestStudyRepository().getStudy().getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + } + + @Test + void studyResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(new HashSet<>(getNonDuplicateBibEntryResult().getEntries()), new HashSet<>(getTestStudyRepository().getStudyResultEntries().getEntries())); + } + + private StudyRepository getTestStudyRepository() throws Exception { + if (Objects.isNull(studyRepository)) { + setUpTestRepository(); + studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager); + } + return studyRepository; + } + + /** + * Set up mocks and copies the study definition file into the test repository + */ + private void setUpTestRepository() throws URISyntaxException { + setUpTestStudyDefinitionFile(); + preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(preferences.getEncoding()).thenReturn(null); + when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + entryTypesManager = new BibEntryTypesManager(); + } + + private void setUpTestStudyDefinitionFile() throws URISyntaxException { + Path destination = tempRepositoryDirectory.resolve("study.bib"); + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } + + /** + * This overwrites the existing result file in the repository with a result file containing multiple BibEntries. + * The repository has to exist before this method is called. + */ + private void setUpTestResultFile() throws URISyntaxException { + Path queryDirectory = Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"); + Path resultFileLocation = Path.of(queryDirectory.toString(), "ArXiv" + ".bib"); + URL resultFile = this.getClass().getResource("ArXivQuantumMock.bib"); + FileUtil.copyFile(Path.of(resultFile.toURI()), resultFileLocation, true); + resultFileLocation = Path.of(queryDirectory.toString(), "Springer" + ".bib"); + resultFile = this.getClass().getResource("SpringerQuantumMock.bib"); + FileUtil.copyFile(Path.of(resultFile.toURI()), resultFileLocation, true); + } + + private BibDatabase getNonDuplicateBibEntryResult() { + BibDatabase mockResults = new BibDatabase(getSpringerCloudComputingMockResults()); + DatabaseMerger merger = new DatabaseMerger(); + merger.merge(mockResults, new BibDatabase(getSpringerQuantumMockResults())); + merger.merge(mockResults, new BibDatabase(getArXivQuantumMockResults())); + return mockResults; + } + + private List getMockResults() { + QueryResult resultQuantum = + new QueryResult("Quantum", List.of( + new FetchResult("ArXiv", new BibDatabase(stripCitationKeys(getArXivQuantumMockResults()))), + new FetchResult("Springer", new BibDatabase(stripCitationKeys(getSpringerQuantumMockResults()))))); + QueryResult resultCloudComputing = new QueryResult("Cloud Computing", List.of(new FetchResult("Springer", new BibDatabase(getSpringerCloudComputingMockResults())))); + return List.of(resultQuantum, resultCloudComputing); + } + + /** + * Strips the citation key from fetched entries as these normally do not have a citation key + */ + private List stripCitationKeys(List entries) { + entries.forEach(bibEntry -> bibEntry.setCitationKey("")); + return entries; + } + + private List getArXivQuantumMockResults() { + BibEntry entry1 = new BibEntry() + .withCitationKey("Blaha") + .withField(StandardField.AUTHOR, "Stephen Blaha") + .withField(StandardField.TITLE, "Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Kaye") + .withField(StandardField.AUTHOR, "Phillip Kaye and Michele Mosca") + .withField(StandardField.TITLE, "Quantum Networks for Generating Arbitrary Quantum States"); + entry2.setType(StandardEntryType.Article); + BibEntry entry3 = new BibEntry() + .withCitationKey("Watrous") + .withField(StandardField.AUTHOR, "John Watrous") + .withField(StandardField.TITLE, "Quantum Computational Complexity"); + entry3.setType(StandardEntryType.Article); + + return List.of(entry1, entry2, entry3); + } + + private List getSpringerQuantumMockResults() { + // This is a duplicate of entry 1 of ArXiv + BibEntry entry1 = new BibEntry() + .withCitationKey("Blaha") + .withField(StandardField.AUTHOR, "Stephen Blaha") + .withField(StandardField.TITLE, "Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Kroeger") + .withField(StandardField.AUTHOR, "H. Kröger") + .withField(StandardField.TITLE, "Nonlinear Dynamics In Quantum Physics -- Quantum Chaos and Quantum Instantons"); + entry2.setType(StandardEntryType.Article); + BibEntry entry3 = new BibEntry() + .withField(StandardField.AUTHOR, "Zieliński, Cezary") + .withField(StandardField.TITLE, "Automatic Control, Robotics, and Information Processing"); + entry3.setType(StandardEntryType.Article); + + CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(new BibDatabaseContext(), JabRefPreferences.getInstance().getCitationKeyPatternPreferences()); + citationKeyGenerator.generateAndSetKey(entry3); + + return List.of(entry1, entry2, entry3); + } + + private List getSpringerCloudComputingMockResults() { + BibEntry entry1 = new BibEntry() + .withCitationKey("Gritzalis") + .withField(StandardField.AUTHOR, "Gritzalis, Dimitris and Stergiopoulos, George and Vasilellis, Efstratios and Anagnostopoulou, Argiro") + .withField(StandardField.TITLE, "Readiness Exercises: Are Risk Assessment Methodologies Ready for the Cloud?"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Rangras") + .withField(StandardField.AUTHOR, "Rangras, Jimit and Bhavsar, Sejal") + .withField(StandardField.TITLE, "Design of Framework for Disaster Recovery in Cloud Computing"); + entry2.setType(StandardEntryType.Article); + return List.of(entry1, entry2); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index 73afa6d63c9..c31bd348b0c 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -21,7 +21,7 @@ /** * Defines the set of capability tests that each tests a given search capability, e.g. author based search. * The idea is to code the capabilities of a fetcher into Java code. - * This way, a) the capbilities of a fetcher are checked automatically (because they can change from time-to-time by the provider) + * This way, a) the capabilities of a fetcher are checked automatically (because they can change from time-to-time by the provider) * and b) the queries sent to the fetchers can be debugged directly without a route through to some fetcher code. */ interface SearchBasedFetcherCapabilityTest { diff --git a/src/test/java/org/jabref/model/study/StudyTest.java b/src/test/java/org/jabref/model/study/StudyTest.java new file mode 100644 index 00000000000..cd22449fcd1 --- /dev/null +++ b/src/test/java/org/jabref/model/study/StudyTest.java @@ -0,0 +1,93 @@ +package org.jabref.model.study; + +import java.time.LocalDate; +import java.util.List; + +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class StudyTest { + Study testStudy; + + @BeforeEach + public void setUpTestStudy() { + BibEntry studyEntry = new BibEntry() + .withField(new UnknownField("name"), "TestStudyName") + .withField(StandardField.AUTHOR, "Jab Ref") + .withField(new UnknownField("researchQuestions"), "Question1; Question2") + .withField(new UnknownField("gitRepositoryURL"), "https://github.com/eclipse/jgit.git"); + studyEntry.setType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY); + + // Create three SearchTerm entries. + BibEntry searchQuery1 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery1"); + searchQuery1.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery1.setCitationKey("query1"); + + BibEntry searchQuery2 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery2"); + searchQuery2.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery2.setCitationKey("query2"); + + BibEntry searchQuery3 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery3"); + searchQuery3.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery3.setCitationKey("query3"); + + // Create two Library entries + BibEntry library1 = new BibEntry() + .withField(new UnknownField("name"), "acm") + .withField(new UnknownField("enabled"), "false") + .withField(new UnknownField("comment"), "disabled, because no good results"); + library1.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); + library1.setCitationKey("library1"); + + BibEntry library2 = new BibEntry() + .withField(new UnknownField("name"), "arxiv") + .withField(new UnknownField("enabled"), "true") + .withField(new UnknownField("Comment"), ""); + library2.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); + library2.setCitationKey("library2"); + + testStudy = new Study(studyEntry, List.of(searchQuery1, searchQuery2, searchQuery3), List.of(library1, library2)); + } + + @Test + void getSearchTermsAsStrings() { + List expectedSearchTerms = List.of("TestSearchQuery1", "TestSearchQuery2", "TestSearchQuery3"); + Assertions.assertEquals(expectedSearchTerms, testStudy.getSearchQueryStrings()); + } + + @Test + void setLastSearchTime() { + LocalDate date = LocalDate.now(); + testStudy.setLastSearchDate(date); + Assertions.assertEquals(date.toString(), testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + } + + @Test + void getStudyName() { + Assertions.assertEquals("TestStudyName", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); + } + + @Test + void getStudyAuthor() { + Assertions.assertEquals("Jab Ref", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); + } + + @Test + void getResearchQuestions() { + Assertions.assertEquals("Question1; Question2", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); + } + + @Test + void getGitRepositoryURL() { + Assertions.assertEquals("https://github.com/eclipse/jgit.git", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_GIT_REPOSITORY).get()); + } +} diff --git a/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib b/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib new file mode 100644 index 00000000000..85df0f1060b --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib @@ -0,0 +1,15 @@ + +@Article{Blaha, + author = {Stephen Blaha}, + title = {Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language}, +} + +@Article{Kaye, + author = {Phillip Kaye and Michele Mosca}, + title = {Quantum Networks for Generating Arbitrary Quantum States}, +} + +@Article{Watrous, + author = {John Watrous}, + title = {Quantum Computational Complexity}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib b/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib new file mode 100644 index 00000000000..627166213fa --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib @@ -0,0 +1,9 @@ +@InCollection{Gritzalis, + author = {Gritzalis, Dimitris and Stergiopoulos, George and Vasilellis, Efstratios and Anagnostopoulou, Argiro}, + title = {Readiness Exercises: Are Risk Assessment Methodologies Ready for the Cloud?}, +} + +@InCollection{Rangras, + author = {Rangras, Jimit and Bhavsar, Sejal}, + title = {Design of Framework for Disaster Recovery in Cloud Computing}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib b/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib new file mode 100644 index 00000000000..3cfa2f88487 --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib @@ -0,0 +1,9 @@ +@Article{Zielinski, + author = {Zieliński, Cezary}, + title = {Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language}, +} + +@Article{Kaye, + author = {H. Kröger}, + title = {Quantum Networks for Generating Arbitrary Quantum States}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/study.bib b/src/test/resources/org/jabref/logic/crawler/study.bib new file mode 100644 index 00000000000..3f9809a82e5 --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/study.bib @@ -0,0 +1,37 @@ +% Encoding: UTF-8 + +@Study{v10, + name={TestStudyName}, + author={Jab Ref}, + researchQuestions={Question1; Question2}, +} + +@SearchQuery{query1, + query={Quantum}, +} + +@SearchQuery{query2, + query={Cloud Computing}, +} + +@SearchQuery{query3, + query={TestSearchQuery3}, +} + +@Library{library1, + name = {Springer}, + enabled = {true}, + comment = {}, +} + +@Library{library2, + name = {ArXiv}, + enabled = {true}, + comment = {}, +} + +@Library{library3, + name = {IEEEXplore}, + enabled = {false}, + comment = {}, +} From 7e45908fde29a2d43906ad39ed2a3c4e09adb0c0 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 25 Nov 2020 14:57:41 +0100 Subject: [PATCH 2/5] Pull Global upward through constructor. --- src/main/java/org/jabref/gui/JabRefFrame.java | 3 ++- .../java/org/jabref/gui/StartLiteratureReviewAction.java | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 1b48ad339fc..80b54fd911d 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -136,6 +136,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.field.SpecialField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.JabRefPreferences; import org.jabref.preferences.LastFocusedTabPreferences; @@ -817,7 +818,7 @@ private MenuBar createMenu() { factory.createMenuItem(StandardActions.SEND_AS_EMAIL, new SendAsEMailAction(dialogService, stateManager)), pushToApplicationMenuItem, new SeparatorMenuItem(), - factory.createMenuItem(StandardActions.START_SYSTEMATIC_LITERATURE_REVIEW, new StartLiteratureReviewAction(this)) + factory.createMenuItem(StandardActions.START_SYSTEMATIC_LITERATURE_REVIEW, new StartLiteratureReviewAction(this, Globals.getFileUpdateMonitor())) ); SidePaneComponent webSearch = sidePaneManager.getComponent(SidePaneType.WEB_SEARCH); diff --git a/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java index a641bd1e514..cf6d3cfc32f 100644 --- a/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java +++ b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java @@ -11,6 +11,8 @@ import org.jabref.logic.crawler.Crawler; import org.jabref.logic.importer.ParseException; import org.jabref.logic.l10n.Localization; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.JabRefPreferences; import org.eclipse.jgit.api.errors.GitAPIException; @@ -21,10 +23,12 @@ public class StartLiteratureReviewAction extends SimpleCommand { private static final Logger LOGGER = LoggerFactory.getLogger(StartLiteratureReviewAction.class); private final JabRefFrame frame; private final DialogService dialogService; + private final FileUpdateMonitor fileUpdateMonitor; - public StartLiteratureReviewAction(JabRefFrame frame) { + public StartLiteratureReviewAction(JabRefFrame frame, FileUpdateMonitor fileUpdateMonitor) { this.frame = frame; this.dialogService = frame.getDialogService(); + this.fileUpdateMonitor = fileUpdateMonitor; } @Override @@ -40,7 +44,7 @@ public void execute() { } final Crawler crawler; try { - crawler = new Crawler(studyDefinitionFile.get(), Globals.getFileUpdateMonitor(), JabRefPreferences.getInstance().getSavePreferences(), Globals.entryTypesManager); + crawler = new Crawler(studyDefinitionFile.get(), fileUpdateMonitor, JabRefPreferences.getInstance().getSavePreferences(), new BibEntryTypesManager()); } catch (IOException | ParseException | GitAPIException e) { LOGGER.info("Error during reading of study definition file.", e); dialogService.showErrorDialogAndWait(Localization.lang("Error during reading of study definition file.")); From 2b6f56c67627af4ef82bf05c0c5ff38fb152b294 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 25 Nov 2020 15:03:44 +0100 Subject: [PATCH 3/5] Pull Globals and ImportFormatPreferences up through constructor Signed-off-by: Dominik Voigt --- src/main/java/org/jabref/gui/JabRefFrame.java | 2 +- .../gui/StartLiteratureReviewAction.java | 27 +++++++++++-------- .../org/jabref/logic/crawler/Crawler.java | 7 ++--- .../org/jabref/logic/crawler/CrawlerTest.java | 22 ++++++++++----- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 80b54fd911d..58f2146d1a6 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -818,7 +818,7 @@ private MenuBar createMenu() { factory.createMenuItem(StandardActions.SEND_AS_EMAIL, new SendAsEMailAction(dialogService, stateManager)), pushToApplicationMenuItem, new SeparatorMenuItem(), - factory.createMenuItem(StandardActions.START_SYSTEMATIC_LITERATURE_REVIEW, new StartLiteratureReviewAction(this, Globals.getFileUpdateMonitor())) + factory.createMenuItem(StandardActions.START_SYSTEMATIC_LITERATURE_REVIEW, new StartLiteratureReviewAction(this, Globals.getFileUpdateMonitor(), Globals.prefs.getWorkingDir(), Globals.TASK_EXECUTOR)) ); SidePaneComponent webSearch = sidePaneManager.getComponent(SidePaneType.WEB_SEARCH); diff --git a/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java index cf6d3cfc32f..d05d0f817f5 100644 --- a/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java +++ b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java @@ -8,6 +8,7 @@ import org.jabref.gui.importer.actions.OpenDatabaseAction; import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.FileDialogConfiguration; +import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.crawler.Crawler; import org.jabref.logic.importer.ParseException; import org.jabref.logic.l10n.Localization; @@ -24,17 +25,21 @@ public class StartLiteratureReviewAction extends SimpleCommand { private final JabRefFrame frame; private final DialogService dialogService; private final FileUpdateMonitor fileUpdateMonitor; + private final Path workingDirectory; + private final TaskExecutor taskExecutor; - public StartLiteratureReviewAction(JabRefFrame frame, FileUpdateMonitor fileUpdateMonitor) { + public StartLiteratureReviewAction(JabRefFrame frame, FileUpdateMonitor fileUpdateMonitor, Path standardWorkingDirectory, TaskExecutor taskExecutor) { this.frame = frame; this.dialogService = frame.getDialogService(); this.fileUpdateMonitor = fileUpdateMonitor; + this.workingDirectory = getInitialDirectory(standardWorkingDirectory); + this.taskExecutor = taskExecutor; } @Override public void execute() { FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() - .withInitialDirectory(getInitialDirectory()) + .withInitialDirectory(workingDirectory) .build(); Optional studyDefinitionFile = dialogService.showFileOpenDialog(fileDialogConfiguration); @@ -44,10 +49,10 @@ public void execute() { } final Crawler crawler; try { - crawler = new Crawler(studyDefinitionFile.get(), fileUpdateMonitor, JabRefPreferences.getInstance().getSavePreferences(), new BibEntryTypesManager()); + crawler = new Crawler(studyDefinitionFile.get(), fileUpdateMonitor, JabRefPreferences.getInstance().getImportFormatPreferences(), JabRefPreferences.getInstance().getSavePreferences(), new BibEntryTypesManager()); } catch (IOException | ParseException | GitAPIException e) { - LOGGER.info("Error during reading of study definition file.", e); - dialogService.showErrorDialogAndWait(Localization.lang("Error during reading of study definition file.")); + LOGGER.error("Error during reading of study definition file.", e); + dialogService.showErrorDialogAndWait(Localization.lang("Error during reading of study definition file."), e); return; } BackgroundTask.wrap(() -> { @@ -55,22 +60,22 @@ public void execute() { return 0; // Return any value to make this a callable instead of a runnable. This allows throwing exceptions. }) .onFailure(e -> { - LOGGER.info("Error during persistence of crawling results."); + LOGGER.error("Error during persistence of crawling results."); dialogService.showErrorDialogAndWait(Localization.lang("Error during persistence of crawling results."), e); }) .onSuccess(unused -> new OpenDatabaseAction(frame).openFile(Path.of(studyDefinitionFile.get().getParent().toString(), "studyResult.bib"), true)) - .executeWith(Globals.TASK_EXECUTOR); + .executeWith(taskExecutor); } /** - * @return Path of current panel database directory or the working directory + * @return Path of current panel database directory or the standard working directory */ - private Path getInitialDirectory() { + private Path getInitialDirectory(Path standardWorkingDirectory) { if (frame.getBasePanelCount() == 0) { - return Globals.prefs.getWorkingDir(); + return standardWorkingDirectory; } else { Optional databasePath = frame.getCurrentLibraryTab().getBibDatabaseContext().getDatabasePath(); - return databasePath.map(Path::getParent).orElse(Globals.prefs.getWorkingDir()); + return databasePath.map(Path::getParent).orElse(standardWorkingDirectory); } } } diff --git a/src/main/java/org/jabref/logic/crawler/Crawler.java b/src/main/java/org/jabref/logic/crawler/Crawler.java index f3114100e87..8b5bd520789 100644 --- a/src/main/java/org/jabref/logic/crawler/Crawler.java +++ b/src/main/java/org/jabref/logic/crawler/Crawler.java @@ -6,6 +6,7 @@ import org.jabref.logic.crawler.git.GitHandler; import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.study.QueryResult; @@ -31,11 +32,11 @@ public class Crawler { * * @param studyDefinitionFile The path to the study definition file that contains the list of targeted E-Libraries and used cross-library queries */ - public Crawler(Path studyDefinitionFile, FileUpdateMonitor fileUpdateMonitor, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IllegalArgumentException, IOException, ParseException, GitAPIException { + public Crawler(Path studyDefinitionFile, FileUpdateMonitor fileUpdateMonitor, ImportFormatPreferences importFormatPreferences, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IllegalArgumentException, IOException, ParseException, GitAPIException { Path studyRepositoryRoot = studyDefinitionFile.getParent(); - studyRepository = new StudyRepository(studyRepositoryRoot, new GitHandler(studyRepositoryRoot), JabRefPreferences.getInstance().getImportFormatPreferences(), fileUpdateMonitor, savePreferences, bibEntryTypesManager); + studyRepository = new StudyRepository(studyRepositoryRoot, new GitHandler(studyRepositoryRoot), importFormatPreferences, fileUpdateMonitor, savePreferences, bibEntryTypesManager); Study study = studyRepository.getStudy(); - LibraryEntryToFetcherConverter libraryEntryToFetcherConverter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), JabRefPreferences.getInstance().getImportFormatPreferences()); + LibraryEntryToFetcherConverter libraryEntryToFetcherConverter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); this.studyFetcher = new StudyFetcher(libraryEntryToFetcherConverter.getActiveFetchers(), study.getSearchQueryStrings()); } diff --git a/src/test/java/org/jabref/logic/crawler/CrawlerTest.java b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java index daeb90b4f07..7c6b53e85a2 100644 --- a/src/test/java/org/jabref/logic/crawler/CrawlerTest.java +++ b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java @@ -1,10 +1,13 @@ package org.jabref.logic.crawler; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import org.jabref.logic.bibtex.FieldContentFormatterPreferences; import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.metadata.SaveOrderConfig; @@ -25,7 +28,8 @@ class CrawlerTest { @TempDir Path tempRepositoryDirectory; - SavePreferences preferences; + ImportFormatPreferences importFormatPreferences; + SavePreferences savePreferences; BibEntryTypesManager entryTypesManager; @Test @@ -33,7 +37,8 @@ public void testWhetherAllFilesAreCreated() throws Exception { setUp(); Crawler testCrawler = new Crawler(getPathToStudyDefinitionFile(), new DummyFileUpdateMonitor(), - preferences, + importFormatPreferences, + savePreferences, entryTypesManager ); @@ -66,10 +71,15 @@ private Path getPathToStudyDefinitionFile() { */ private void setUp() throws Exception { setUpRepository(); - preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); - when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); - when(preferences.getEncoding()).thenReturn(null); - when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); + savePreferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(savePreferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(savePreferences.getEncoding()).thenReturn(null); + when(savePreferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + when(importFormatPreferences.getKeywordSeparator()).thenReturn(','); + when(importFormatPreferences.getFieldContentFormatterPreferences()).thenReturn(new FieldContentFormatterPreferences()); + when(importFormatPreferences.isKeywordSyncEnabled()).thenReturn(false); + when(importFormatPreferences.getEncoding()).thenReturn(StandardCharsets.UTF_8); entryTypesManager = new BibEntryTypesManager(); } From 93bb5073cacdbf884f693ff1cc65f0160c98959e Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 25 Nov 2020 16:56:14 +0100 Subject: [PATCH 4/5] Integrate requested changes and fix architecture tests by correcting test classes Signed-off-by: Dominik Voigt --- .../LibraryEntryToFetcherConverter.java | 4 +- .../jabref/logic/crawler/StudyRepository.java | 24 +++---- ...ratureReviewStudyEntryTypeDefinitions.java | 2 +- .../LibraryEntryToFetcherConverterTest.java | 24 ++++--- .../logic/crawler/StudyRepositoryTest.java | 69 +++++++++++++------ .../org/jabref/model/study/StudyTest.java | 17 ++--- 6 files changed, 88 insertions(+), 52 deletions(-) diff --git a/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java index ee2f0f00c71..cadf5b2978e 100644 --- a/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java +++ b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java @@ -11,6 +11,8 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.UnknownField; +import static org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY; + /** * Converts library entries from the given study into their corresponding fetchers. */ @@ -42,7 +44,7 @@ public List getActiveFetchers() { */ private List getFetchersFromLibraryEntries(List libraryEntries) { return libraryEntries.parallelStream() - .filter(bibEntry -> bibEntry.getType().getName().equals("library")) + .filter(bibEntry -> bibEntry.getType().getName().equals(LIBRARY_ENTRY.getName())) .map(this::createFetcherFromLibraryEntry) .filter(Objects::nonNull) .collect(Collectors.toList()); diff --git a/src/main/java/org/jabref/logic/crawler/StudyRepository.java b/src/main/java/org/jabref/logic/crawler/StudyRepository.java index 6b16d1e07dc..b302e065946 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyRepository.java +++ b/src/main/java/org/jabref/logic/crawler/StudyRepository.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; @@ -49,6 +50,8 @@ class StudyRepository { // Tests work with study.bib private static final String STUDY_DEFINITION_FILE_NAME = "study.bib"; private static final Logger LOGGER = LoggerFactory.getLogger(StudyRepository.class); + private static final Pattern MATCHCOLON = Pattern.compile(":"); + private static final Pattern MATCHILLEGALCHARACTERS = Pattern.compile("[^A-Za-z0-9_.\\s=-]"); private final Path repositoryPath; private final Path studyDefinitionBib; @@ -74,7 +77,7 @@ public StudyRepository(Path pathToRepository, GitHandler gitHandler, ImportForma try { gitHandler.updateLocalRepository(); } catch (GitAPIException e) { - LOGGER.info("Updating repository from remote failed"); + LOGGER.error("Updating repository from remote failed"); } this.importFormatPreferences = importFormatPreferences; this.fileUpdateMonitor = fileUpdateMonitor; @@ -147,7 +150,7 @@ public void persist(List crawlResults) throws IOException, GitAPIEx try { gitHandler.updateLocalRepository(); } catch (GitAPIException e) { - LOGGER.info("Updating repository from remote failed"); + LOGGER.error("Updating repository from remote failed"); } persistResults(crawlResults); study.setLastSearchDate(LocalDate.now()); @@ -155,7 +158,7 @@ public void persist(List crawlResults) throws IOException, GitAPIEx try { gitHandler.updateRemoteRepository("Conducted search " + LocalDate.now()); } catch (GitAPIException e) { - LOGGER.info("Updating remote repository failed"); + LOGGER.error("Updating remote repository failed"); } } @@ -191,11 +194,7 @@ private void createQueryResultFolder(String query) throws IOException { private void createFolder(Path folder) throws IOException { if (Files.notExists(folder)) { - try { - Files.createDirectory(folder); - } catch (IOException e) { - throw new IOException("Error during creation of repository structure.", e); - } + Files.createDirectory(folder); } } @@ -244,8 +243,8 @@ private void createBibFile(Path file) { */ private String trimNameAndAddID(String query) { // Replace all field: with field= for folder name - String trimmedNamed = query.replaceAll(":", "="); - trimmedNamed = trimmedNamed.replaceAll("[^A-Za-z0-9_.\\s=-]", ""); + String trimmedNamed = MATCHCOLON.matcher(query).replaceAll("="); + trimmedNamed = MATCHILLEGALCHARACTERS.matcher(trimmedNamed).replaceAll(""); if (query.length() > 240) { trimmedNamed = query.substring(0, 240); } @@ -261,15 +260,16 @@ private String trimNameAndAddID(String query) { * @return ID of the query defined in the study definition. */ private String findQueryIDByQueryString(String query) { + String queryField = "query"; return study.getSearchQueryEntries() .parallelStream() - .filter(bibEntry -> bibEntry.getField(new UnknownField("query")).orElse("").equals(query)) + .filter(bibEntry -> bibEntry.getField(new UnknownField(queryField)).orElse("").equals(query)) .map(BibEntry::getCitationKey) .filter(Optional::isPresent) .map(Optional::get) .findFirst() .orElseThrow() - .replaceFirst("query", ""); + .replaceFirst(queryField, ""); } /** diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java index e9f31b2e630..5d1bf665bfe 100644 --- a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java +++ b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java @@ -50,7 +50,7 @@ public class SystematicLiteratureReviewStudyEntryTypeDefinitions { private static final BibEntryType LIBRARY_ENTRY = new BibEntryTypeBuilder() .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) .withRequiredFields(new UnknownField("name"), new UnknownField("enabled")) - .withImportantFields(new UnknownField("comment")) + .withImportantFields(StandardField.COMMENT) .build(); public static final List ALL = Arrays.asList(STUDY_ENTRY, SEARCH_QUERY_ENTRY, LIBRARY_ENTRY); diff --git a/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java index cce997fe564..629fad93ec4 100644 --- a/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java +++ b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java @@ -1,18 +1,20 @@ package org.jabref.logic.crawler; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.List; +import org.jabref.logic.bibtex.FieldContentFormatterPreferences; import org.jabref.logic.crawler.git.GitHandler; import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.SearchBasedFetcher; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.metadata.SaveOrderConfig; import org.jabref.model.study.Study; import org.jabref.model.util.DummyFileUpdateMonitor; -import org.jabref.preferences.JabRefPreferences; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -24,7 +26,8 @@ import static org.mockito.Mockito.when; class LibraryEntryToFetcherConverterTest { - SavePreferences preferences; + ImportFormatPreferences importFormatPreferences; + SavePreferences savePreferences; BibEntryTypesManager entryTypesManager; GitHandler gitHandler; @TempDir @@ -32,10 +35,15 @@ class LibraryEntryToFetcherConverterTest { @BeforeEach void setUpMocks() { - preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); - when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); - when(preferences.getEncoding()).thenReturn(null); - when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); + savePreferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(savePreferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(savePreferences.getEncoding()).thenReturn(null); + when(savePreferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + when(importFormatPreferences.getKeywordSeparator()).thenReturn(','); + when(importFormatPreferences.getFieldContentFormatterPreferences()).thenReturn(new FieldContentFormatterPreferences()); + when(importFormatPreferences.isKeywordSyncEnabled()).thenReturn(false); + when(importFormatPreferences.getEncoding()).thenReturn(StandardCharsets.UTF_8); entryTypesManager = new BibEntryTypesManager(); gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); } @@ -45,8 +53,8 @@ public void getActiveFetcherInstances() throws Exception { Path studyDefinition = tempRepositoryDirectory.resolve("study.bib"); copyTestStudyDefinitionFileIntoDirectory(studyDefinition); - Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager).getStudy(); - LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), JabRefPreferences.getInstance().getImportFormatPreferences()); + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager).getStudy(); + LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); List result = converter.getActiveFetchers(); Assertions.assertEquals(2, result.size()); diff --git a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java index 72717b2a0a2..8a69c6d7a01 100644 --- a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java +++ b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java @@ -1,8 +1,8 @@ package org.jabref.logic.crawler; import java.io.IOException; -import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDate; @@ -12,10 +12,14 @@ import java.util.Objects; import java.util.stream.Collectors; +import org.jabref.logic.bibtex.FieldContentFormatterPreferences; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; +import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; +import org.jabref.logic.citationkeypattern.GlobalCitationKeyPattern; import org.jabref.logic.crawler.git.GitHandler; import org.jabref.logic.database.DatabaseMerger; import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -30,12 +34,13 @@ import org.jabref.model.study.Study; import org.jabref.model.study.StudyMetaDataField; import org.jabref.model.util.DummyFileUpdateMonitor; -import org.jabref.preferences.JabRefPreferences; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.mockito.Answers; +import static org.jabref.logic.citationkeypattern.CitationKeyGenerator.DEFAULT_UNWANTED_CHARACTERS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -44,23 +49,52 @@ class StudyRepositoryTest { private static final String NON_EXISTING_DIRECTORY = "nonExistingTestRepositoryDirectory"; - SavePreferences preferences; + CitationKeyPatternPreferences citationKeyPatternPreferences; + ImportFormatPreferences importFormatPreferences; + SavePreferences savePreferences; BibEntryTypesManager entryTypesManager; @TempDir Path tempRepositoryDirectory; StudyRepository studyRepository; GitHandler gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + /** + * Set up mocks + */ + @BeforeEach + public void setUpMocks() { + savePreferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); + citationKeyPatternPreferences = new CitationKeyPatternPreferences( + false, + false, + false, + CitationKeyPatternPreferences.KeySuffix.SECOND_WITH_A, + "", + "", + DEFAULT_UNWANTED_CHARACTERS, + GlobalCitationKeyPattern.fromPattern("[auth][year]"), + ','); + when(savePreferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(savePreferences.getEncoding()).thenReturn(null); + when(savePreferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + when(importFormatPreferences.getKeywordSeparator()).thenReturn(','); + when(importFormatPreferences.getFieldContentFormatterPreferences()).thenReturn(new FieldContentFormatterPreferences()); + when(importFormatPreferences.isKeywordSyncEnabled()).thenReturn(false); + when(importFormatPreferences.getEncoding()).thenReturn(StandardCharsets.UTF_8); + entryTypesManager = new BibEntryTypesManager(); + } + @Test void providePathToNonExistentRepositoryThrowsException() { Path nonExistingRepositoryDirectory = tempRepositoryDirectory.resolve(NON_EXISTING_DIRECTORY); - assertThrows(IOException.class, () -> new StudyRepository(nonExistingRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager)); + assertThrows(IOException.class, () -> new StudyRepository(nonExistingRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager)); } @Test void providePathToExistentRepositoryWithOutStudyDefinitionFileThrowsException() { - assertThrows(IOException.class, () -> new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager)); + assertThrows(IOException.class, () -> new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager)); } /** @@ -68,11 +102,11 @@ void providePathToExistentRepositoryWithOutStudyDefinitionFileThrowsException() */ @Test void studyFileCorrectlyImported() throws Exception { - setUpTestRepository(); + setUpTestStudyDefinitionFile(); List expectedSearchterms = List.of("Quantum", "Cloud Computing", "TestSearchQuery3"); List expectedActiveFetchersByName = List.of("Springer", "ArXiv"); - Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager).getStudy(); + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager).getStudy(); assertEquals(expectedSearchterms, study.getSearchQueryStrings()); assertEquals("TestStudyName", study.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); @@ -165,25 +199,16 @@ void studyResultsPersistedCorrectly() throws Exception { private StudyRepository getTestStudyRepository() throws Exception { if (Objects.isNull(studyRepository)) { - setUpTestRepository(); - studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager); + setUpTestStudyDefinitionFile(); + studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager); } return studyRepository; } /** - * Set up mocks and copies the study definition file into the test repository + * Copies the study definition file into the test repository */ - private void setUpTestRepository() throws URISyntaxException { - setUpTestStudyDefinitionFile(); - preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); - when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); - when(preferences.getEncoding()).thenReturn(null); - when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); - entryTypesManager = new BibEntryTypesManager(); - } - - private void setUpTestStudyDefinitionFile() throws URISyntaxException { + private void setUpTestStudyDefinitionFile() throws Exception { Path destination = tempRepositoryDirectory.resolve("study.bib"); URL studyDefinition = this.getClass().getResource("study.bib"); FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); @@ -193,7 +218,7 @@ private void setUpTestStudyDefinitionFile() throws URISyntaxException { * This overwrites the existing result file in the repository with a result file containing multiple BibEntries. * The repository has to exist before this method is called. */ - private void setUpTestResultFile() throws URISyntaxException { + private void setUpTestResultFile() throws Exception { Path queryDirectory = Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"); Path resultFileLocation = Path.of(queryDirectory.toString(), "ArXiv" + ".bib"); URL resultFile = this.getClass().getResource("ArXivQuantumMock.bib"); @@ -265,7 +290,7 @@ private List getSpringerQuantumMockResults() { .withField(StandardField.TITLE, "Automatic Control, Robotics, and Information Processing"); entry3.setType(StandardEntryType.Article); - CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(new BibDatabaseContext(), JabRefPreferences.getInstance().getCitationKeyPatternPreferences()); + CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(new BibDatabaseContext(), citationKeyPatternPreferences); citationKeyGenerator.generateAndSetKey(entry3); return List.of(entry1, entry2, entry3); diff --git a/src/test/java/org/jabref/model/study/StudyTest.java b/src/test/java/org/jabref/model/study/StudyTest.java index cd22449fcd1..9ab34fcd55e 100644 --- a/src/test/java/org/jabref/model/study/StudyTest.java +++ b/src/test/java/org/jabref/model/study/StudyTest.java @@ -8,11 +8,12 @@ import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class StudyTest { +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class StudyTest { Study testStudy; @BeforeEach @@ -61,33 +62,33 @@ public void setUpTestStudy() { @Test void getSearchTermsAsStrings() { List expectedSearchTerms = List.of("TestSearchQuery1", "TestSearchQuery2", "TestSearchQuery3"); - Assertions.assertEquals(expectedSearchTerms, testStudy.getSearchQueryStrings()); + assertEquals(expectedSearchTerms, testStudy.getSearchQueryStrings()); } @Test void setLastSearchTime() { LocalDate date = LocalDate.now(); testStudy.setLastSearchDate(date); - Assertions.assertEquals(date.toString(), testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + assertEquals(date.toString(), testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); } @Test void getStudyName() { - Assertions.assertEquals("TestStudyName", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); + assertEquals("TestStudyName", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); } @Test void getStudyAuthor() { - Assertions.assertEquals("Jab Ref", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); + assertEquals("Jab Ref", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); } @Test void getResearchQuestions() { - Assertions.assertEquals("Question1; Question2", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); + assertEquals("Question1; Question2", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); } @Test void getGitRepositoryURL() { - Assertions.assertEquals("https://github.com/eclipse/jgit.git", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_GIT_REPOSITORY).get()); + assertEquals("https://github.com/eclipse/jgit.git", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_GIT_REPOSITORY).get()); } } From b4e2f10aba9c41c24b2f52d814c3b319e2796890 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 25 Nov 2020 17:08:30 +0100 Subject: [PATCH 5/5] Remove unused imports Signed-off-by: Dominik Voigt --- src/main/java/org/jabref/gui/JabRefFrame.java | 5 ++--- src/main/java/org/jabref/logic/crawler/Crawler.java | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 58f2146d1a6..d44f6307084 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -136,7 +136,6 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.field.SpecialField; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.JabRefPreferences; import org.jabref.preferences.LastFocusedTabPreferences; @@ -995,7 +994,7 @@ public void addParserResult(ParserResult parserResult, boolean focusPanel) { * This method causes all open LibraryTabs to set up their tables anew. When called from PreferencesDialogViewModel, * this updates to the new settings. * We need to notify all tabs about the changes to avoid problems when changing the column set. - * */ + */ public void setupAllTables() { tabbedPane.getTabs().forEach(tab -> { LibraryTab libraryTab = (LibraryTab) tab; @@ -1016,7 +1015,7 @@ private ContextMenu createTabContextMenu(KeyBindingRepository keyBindingReposito new SeparatorMenuItem(), factory.createMenuItem(StandardActions.OPEN_DATABASE_FOLDER, new OpenDatabaseFolder()), factory.createMenuItem(StandardActions.OPEN_CONSOLE, new OpenConsoleAction(stateManager)) - ); + ); return contextMenu; } diff --git a/src/main/java/org/jabref/logic/crawler/Crawler.java b/src/main/java/org/jabref/logic/crawler/Crawler.java index 8b5bd520789..eade3b55a59 100644 --- a/src/main/java/org/jabref/logic/crawler/Crawler.java +++ b/src/main/java/org/jabref/logic/crawler/Crawler.java @@ -12,7 +12,6 @@ import org.jabref.model.study.QueryResult; import org.jabref.model.study.Study; import org.jabref.model.util.FileUpdateMonitor; -import org.jabref.preferences.JabRefPreferences; import org.eclipse.jgit.api.errors.GitAPIException;