New fetcher (#1929)

* Refactor search based fetcher based on website and parser

* Refactor BibtexParser

* Refactor cleanup preferences

* Add search-based fetcher for the Astrophysics Data System

* Introduce EntryBasedFetcher

* Add ADS as EntyBasedFetcher

* Add comment about new ADS API

* Add MathSciNet entry based fetcher

* Add MathSciNet search fetcher

* Add zbMath fetcher

* Add Changelog

* Remove header

* Remove more headers

* Include feedback

* Rename GVKParser

* Remove big

* Remove unused imports

* Fix failing GVK tests

* Fix failing tests due to no subscription

CHANGELOG.md

@@ -12,6 +12,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 ## [Unreleased]
 
 ### Changed
+- Added fetcher for [MathSciNet](http://www.ams.org/mathscinet), [zbMATH](https://www.zbmath.org/) and [Astrophysics Data System](http://www.adsabs.harvard.edu/)
 - Implemented [#825](https://github.com/JabRef/jabref/issues/825): Search Bar across all bib files instead each having its own
 - Implemented [#573](https://github.com/JabRef/jabref/issues/573): Add key shortcut for global search (`ctrl+shift+f`, if the searchfield is empty it will be focused instead)
 - The search result Window will now show which entry belongs to which bib file

src/jmh/java/net/sf/jabref/benchmarks/Benchmarks.java

@@ -73,10 +73,8 @@ public void init() throws Exception {
 
     @Benchmark
     public ParserResult parse() throws IOException {
-        StringReader bibtexStringReader = new StringReader(bibtexString);
-        BibtexParser parser = new BibtexParser(bibtexStringReader,
-                Globals.prefs.getImportFormatPreferences());
-        return parser.parse();
+        BibtexParser parser = new BibtexParser(Globals.prefs.getImportFormatPreferences());
+        return parser.parse(new StringReader(bibtexString));
     }
 
     @Benchmark

src/main/java/net/sf/jabref/cli/CrossrefFetcherEvaluator.java

@@ -27,8 +27,8 @@ public class CrossrefFetcherEvaluator {
     public static void main(String[] args) throws IOException, InterruptedException {
         Globals.prefs = JabRefPreferences.getInstance();
         try (FileReader reader = new FileReader(args[0])) {
-            BibtexParser parser = new BibtexParser(reader, Globals.prefs.getImportFormatPreferences());
-            ParserResult result = parser.parse();
+            BibtexParser parser = new BibtexParser(Globals.prefs.getImportFormatPreferences());
+            ParserResult result = parser.parse(reader);
             BibDatabase db = result.getDatabase();
 
             List<BibEntry> entries = db.getEntries();

src/main/java/net/sf/jabref/gui/ClipBoardManager.java

@@ -95,9 +95,8 @@ public List<BibEntry> extractBibEntriesFromClipboard() {
                     entry.ifPresent(result::add);
                 } else {
                     // parse bibtex string
-                    BibtexParser bp = new BibtexParser(new StringReader(data),
-                            Globals.prefs.getImportFormatPreferences());
-                    BibDatabase db = bp.parse().getDatabase();
+                    BibtexParser bp = new BibtexParser(Globals.prefs.getImportFormatPreferences());
+                    BibDatabase db = bp.parse(new StringReader(data)).getDatabase();
                     LOGGER.info("Parsed " + db.getEntryCount() + " entries from clipboard text");
                     if (db.hasEntries()) {
                         result = db.getEntries();

src/main/java/net/sf/jabref/gui/actions/CleanupAction.java

@@ -5,7 +5,6 @@
 
 import javax.swing.JOptionPane;
 
-import net.sf.jabref.BibDatabaseContext;
 import net.sf.jabref.Globals;
 import net.sf.jabref.gui.BasePanel;
 import net.sf.jabref.gui.JabRefFrame;
@@ -145,11 +144,8 @@ private int showDialog(CleanupPresetPanel presetPanel) {
      */
     private void doCleanup(CleanupPreset preset, BibEntry entry, NamedCompound ce) {
         // Create and run cleaner
-        BibDatabaseContext bibDatabaseContext = panel.getBibDatabaseContext();
-        CleanupWorker cleaner = new CleanupWorker(bibDatabaseContext,
-                Globals.prefs.get(JabRefPreferences.IMPORT_FILENAMEPATTERN),
-                Globals.prefs.getLayoutFormatterPreferences(Globals.journalAbbreviationLoader),
-                Globals.prefs.getFileDirectoryPreferences());
+        CleanupWorker cleaner = new CleanupWorker(panel.getBibDatabaseContext(), preferences.getCleanupPreferences(
+                Globals.journalAbbreviationLoader));
         List<FieldChange> changes = cleaner.cleanup(preset, entry);
 
         unsuccessfulRenames = cleaner.getUnsuccessfulRenames();
@@ -163,4 +159,5 @@ private void doCleanup(CleanupPreset preset, BibEntry entry, NamedCompound ce) {
             ce.addEdit(new UndoableFieldChange(change));
         }
     }
+
 }

src/main/java/net/sf/jabref/gui/entryeditor/EntryEditor.java

@@ -759,11 +759,9 @@ public synchronized void switchTo(BibEntry switchEntry) {
     }
 
     private boolean storeSource() {
-        BibtexParser bibtexParser = new BibtexParser(new StringReader(source.getText()),
-                Globals.prefs.getImportFormatPreferences());
-
+        BibtexParser bibtexParser = new BibtexParser(Globals.prefs.getImportFormatPreferences());
         try {
-            ParserResult parserResult = bibtexParser.parse();
+            ParserResult parserResult = bibtexParser.parse(new StringReader(source.getText()));
             BibDatabase database = parserResult.getDatabase();
 
             if (database.getEntryCount() > 1) {

src/main/java/net/sf/jabref/gui/importer/fetcher/EntryFetchers.java

@@ -8,9 +8,12 @@
 import net.sf.jabref.Globals;
 import net.sf.jabref.logic.importer.IdBasedFetcher;
 import net.sf.jabref.logic.importer.fetcher.ArXiv;
+import net.sf.jabref.logic.importer.fetcher.AstrophysicsDataSystem;
 import net.sf.jabref.logic.importer.fetcher.DiVA;
 import net.sf.jabref.logic.importer.fetcher.GvkFetcher;
 import net.sf.jabref.logic.importer.fetcher.IsbnFetcher;
+import net.sf.jabref.logic.importer.fetcher.MathSciNet;
+import net.sf.jabref.logic.importer.fetcher.zbMATH;
 import net.sf.jabref.logic.journals.JournalAbbreviationLoader;
 
 public class EntryFetchers {
@@ -34,6 +37,10 @@ public EntryFetchers(JournalAbbreviationLoader abbreviationLoader) {
 
         entryFetchers.add(new SearchBasedEntryFetcher(new ArXiv()));
         entryFetchers.add(new SearchBasedEntryFetcher(new GvkFetcher()));
+        entryFetchers.add(
+                new SearchBasedEntryFetcher(new AstrophysicsDataSystem(Globals.prefs.getImportFormatPreferences())));
+        entryFetchers.add(new SearchBasedEntryFetcher(new MathSciNet(Globals.prefs.getImportFormatPreferences())));
+        entryFetchers.add(new SearchBasedEntryFetcher(new zbMATH(Globals.prefs.getImportFormatPreferences())));
     }
 
     public List<EntryFetcher> getEntryFetchers() {

src/main/java/net/sf/jabref/gui/importer/fetcher/GoogleScholarFetcher.java

@@ -242,9 +242,8 @@ private String getCitationsFromUrl(String urlQuery, Map<String, JLabel> ids) thr
     private BibEntry downloadEntry(String link) throws IOException {
         try {
             String s = new URLDownload(link).downloadToString(StandardCharsets.UTF_8);
-            BibtexParser bp = new BibtexParser(new StringReader(s),
-                    Globals.prefs.getImportFormatPreferences());
-            ParserResult pr = bp.parse();
+            BibtexParser bp = new BibtexParser(Globals.prefs.getImportFormatPreferences());
+            ParserResult pr = bp.parse(new StringReader(s));
             if ((pr != null) && (pr.getDatabase() != null)) {
                 Collection<BibEntry> entries = pr.getDatabase().getEntries();
                 if (entries.size() == 1) {

src/main/java/net/sf/jabref/logic/bibtex/FieldContentParser.java

@@ -1,6 +1,7 @@
 package net.sf.jabref.logic.bibtex;
 
 import java.util.HashSet;
+import java.util.Objects;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -21,6 +22,8 @@ public class FieldContentParser {
 
 
     public FieldContentParser(FieldContentParserPreferences prefs) {
+        Objects.requireNonNull(prefs);
+
         multiLineFields = new HashSet<>();
         // the following two are also coded in net.sf.jabref.logic.bibtex.LatexFieldFormatter.format(String, String)
         multiLineFields.add(FieldName.ABSTRACT);

src/main/java/net/sf/jabref/logic/cleanup/CleanupPreferences.java

@@ -0,0 +1,30 @@
+package net.sf.jabref.logic.cleanup;
+
+import net.sf.jabref.FileDirectoryPreferences;
+import net.sf.jabref.logic.layout.LayoutFormatterPreferences;
+
+public class CleanupPreferences {
+
+    private final String fileNamePattern;
+    private final LayoutFormatterPreferences layoutFormatterPreferences;
+    private final FileDirectoryPreferences fileDirectoryPreferences;
+
+    public CleanupPreferences(String fileNamePattern, LayoutFormatterPreferences layoutFormatterPreferences,
+            FileDirectoryPreferences fileDirectoryPreferences) {
+        this.fileNamePattern = fileNamePattern;
+        this.layoutFormatterPreferences = layoutFormatterPreferences;
+        this.fileDirectoryPreferences = fileDirectoryPreferences;
+    }
+
+    public String getFileNamePattern() {
+        return fileNamePattern;
+    }
+
+    public LayoutFormatterPreferences getLayoutFormatterPreferences() {
+        return layoutFormatterPreferences;
+    }
+
+    public FileDirectoryPreferences getFileDirectoryPreferences() {
+        return fileDirectoryPreferences;
+    }
+}

src/main/java/net/sf/jabref/logic/cleanup/CleanupWorker.java

@@ -19,12 +19,11 @@ public class CleanupWorker {
     private int unsuccessfulRenames;
 
 
-    public CleanupWorker(BibDatabaseContext databaseContext, String fileNamePattern, LayoutFormatterPreferences prefs,
-            FileDirectoryPreferences fileDirectoryPreferences) {
+    public CleanupWorker(BibDatabaseContext databaseContext, CleanupPreferences cleanupPreferences) {
         this.databaseContext = databaseContext;
-        this.fileNamePattern = fileNamePattern;
-        this.prefs = prefs;
-        this.fileDirectoryPreferences = fileDirectoryPreferences;
+        this.fileNamePattern = cleanupPreferences.getFileNamePattern();
+        this.prefs = cleanupPreferences.getLayoutFormatterPreferences();
+        this.fileDirectoryPreferences = cleanupPreferences.getFileDirectoryPreferences();
     }
 
     public int getUnsuccessfulRenames() {

src/main/java/net/sf/jabref/logic/cleanup/MoveFieldCleanup.java

@@ -0,0 +1,31 @@
+package net.sf.jabref.logic.cleanup;
+
+import java.util.List;
+import java.util.Optional;
+
+import net.sf.jabref.logic.util.OptionalUtil;
+import net.sf.jabref.model.FieldChange;
+import net.sf.jabref.model.entry.BibEntry;
+
+/**
+ * Moves the content of one field to another field.
+ */
+public class MoveFieldCleanup implements CleanupJob {
+
+    private String sourceField;
+    private String targetField;
+
+    public MoveFieldCleanup(String sourceField, String targetField) {
+        this.sourceField = sourceField;
+        this.targetField = targetField;
+    }
+
+    @Override
+    public List<FieldChange> cleanup(BibEntry entry) {
+
+        Optional<FieldChange> setFieldChange = entry.getField(sourceField).flatMap(
+                value -> entry.setField(targetField, value));
+        Optional<FieldChange> clearFieldChange = entry.clearField(sourceField);
+        return OptionalUtil.toList(setFieldChange, clearFieldChange);
+    }
+}

src/main/java/net/sf/jabref/logic/importer/EntryBasedFetcher.java

@@ -0,0 +1,21 @@
+package net.sf.jabref.logic.importer;
+
+import java.util.List;
+
+import net.sf.jabref.model.entry.BibEntry;
+
+/**
+ * Searches web resources for bibliographic information based on a {@link BibEntry}.
+ * Useful to complete an existing entry with fetched information.
+ * May return multiple search hits.
+ */
+public interface EntryBasedFetcher extends WebFetcher {
+
+    /**
+     * Looks for hits which are matched by the given {@link BibEntry}.
+     *
+     * @param entry entry to search bibliographic information for
+     * @return a list of {@link BibEntry}, which are matched by the query (may be empty)
+     */
+    List<BibEntry> performSearch(BibEntry entry) throws FetcherException;
+}

src/main/java/net/sf/jabref/logic/importer/EntryBasedParserFetcher.java

@@ -0,0 +1,71 @@
+package net.sf.jabref.logic.importer;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+import java.util.Objects;
+
+import net.sf.jabref.logic.formatter.Formatter;
+import net.sf.jabref.model.entry.BibEntry;
+
+/**
+ * Provides a convenient interface for entry-based fetcher, which follow the usual three-step procedure:
+ * 1. Open a URL based on the entry
+ * 2. Parse the response to get a list of {@link BibEntry}
+ * 3. Post-process fetched entries
+ */
+public interface EntryBasedParserFetcher extends EntryBasedFetcher {
+
+    /**
+     * Constructs a URL based on the {@link BibEntry}.
+     * @param entry the entry to look information for
+     */
+    URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException;
+
+    /**
+     * Returns the parser used to convert the response to a list of {@link BibEntry}.
+     */
+    Parser getParser();
+
+    /**
+     * Performs a cleanup of the fetched entry.
+     *
+     * Only systematic errors of the fetcher should be corrected here
+     * (i.e. if information is consistently contained in the wrong field or the wrong format)
+     * but not cosmetic issues which may depend on the user's taste (for example, LateX code vs HTML in the abstract).
+     *
+     * Try to reuse existing {@link Formatter} for the cleanup. For example,
+     * {@code new FieldFormatterCleanup(FieldName.TITLE, new RemoveBracesFormatter()).cleanup(entry);}
+     *
+     * By default, no cleanup is done.
+     * @param entry the entry to be cleaned-up
+     */
+    default void doPostCleanup(BibEntry entry) {
+        // Do nothing by default
+    }
+
+    @Override
+    default List<BibEntry> performSearch(BibEntry entry) throws FetcherException {
+        Objects.requireNonNull(entry);
+
+        try (InputStream stream = new BufferedInputStream(getURLForEntry(entry).openStream())) {
+            List<BibEntry> fetchedEntries = getParser().parseEntries(stream);
+
+            // Post-cleanup
+            fetchedEntries.forEach(this::doPostCleanup);
+
+            return fetchedEntries;
+        } catch (URISyntaxException e) {
+            throw new FetcherException("Search URI is malformed", e);
+        } catch (IOException e) {
+            // TODO: Catch HTTP Response 401 errors and report that user has no rights to access resource
+            throw new FetcherException("An I/O exception occurred", e);
+        } catch (ParserException e) {
+            throw new FetcherException("An internal parser error occurred", e);
+        }
+    }
+}

src/main/java/net/sf/jabref/logic/importer/Parser.java

@@ -0,0 +1,14 @@
+package net.sf.jabref.logic.importer;
+
+import java.io.InputStream;
+import java.util.List;
+
+import net.sf.jabref.model.entry.BibEntry;
+
+/**
+ * A parser converts an {@link InputStream} into a list of {@link BibEntry}.
+ */
+public interface Parser {
+
+    List<BibEntry> parseEntries(InputStream inputStream) throws ParserException;
+}

src/main/java/net/sf/jabref/logic/importer/ParserException.java

@@ -0,0 +1,17 @@
+package net.sf.jabref.logic.importer;
+
+public class ParserException extends Exception {
+
+
+    public ParserException(String errorMessage, Exception cause) {
+        super(errorMessage, cause);
+    }
+
+    public ParserException(String errorMessage) {
+        super(errorMessage);
+    }
+
+    public ParserException(Exception cause) {
+        super(cause);
+    }
+}