Add option to parse new references from plain text using GROBID service [solving #4826]

CHANGELOG.md

@@ -13,6 +13,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 
 ### Changed
 
+- We reintroduced the possibility to extract references from plain text (using GROBID) [#5614](https://github.com/JabRef/jabref/pull/5614) 
 - We changed the open office panel to show buttons in rows of three instead of going straight down to save space as the button expanded out to take up unnecessary horizontal space. [#5479](https://github.com/JabRef/jabref/issues/5479)
 - We cleaned up the group add/edit dialog. [#5826](https://github.com/JabRef/jabref/pull/5826)
 - We reintroduced the index column. [#5844](https://github.com/JabRef/jabref/pull/5844)

src/main/java/org/jabref/gui/JabRefFrame.java

@@ -481,6 +481,7 @@ private Node createToolbar() {
         HBox rightSide = new HBox(
                 factory.createIconButton(StandardActions.NEW_ARTICLE, new NewEntryAction(this, StandardEntryType.Article, dialogService, Globals.prefs, stateManager)),
                 factory.createIconButton(StandardActions.NEW_ENTRY, new NewEntryAction(this, dialogService, Globals.prefs, stateManager)),
+                factory.createIconButton(StandardActions.NEW_ENTRY_FROM_PLAIN_TEXT, new ExtractBibtexAction(stateManager)),
                 factory.createIconButton(StandardActions.DELETE_ENTRY, new OldDatabaseCommandWrapper(Actions.DELETE, this, stateManager)),
                 new Separator(Orientation.VERTICAL),
                 factory.createIconButton(StandardActions.UNDO, new OldDatabaseCommandWrapper(Actions.UNDO, this, stateManager)),
@@ -729,6 +730,7 @@ private MenuBar createMenu() {
         //@formatter:off
         library.getItems().addAll(
                 factory.createMenuItem(StandardActions.NEW_ENTRY, new NewEntryAction(this, dialogService, Globals.prefs, stateManager)),
+                factory.createMenuItem(StandardActions.NEW_ENTRY_FROM_PLAIN_TEXT, new ExtractBibtexAction(stateManager)),
                 factory.createMenuItem(StandardActions.DELETE_ENTRY, new OldDatabaseCommandWrapper(Actions.DELETE, this, stateManager)),
 
                 new SeparatorMenuItem(),
@@ -768,7 +770,6 @@ private MenuBar createMenu() {
                 factory.createMenuItem(StandardActions.FIND_UNLINKED_FILES, new FindUnlinkedFilesAction(this, stateManager)),
                 factory.createMenuItem(StandardActions.WRITE_XMP, new OldDatabaseCommandWrapper(Actions.WRITE_XMP, this, stateManager)),
                 factory.createMenuItem(StandardActions.COPY_LINKED_FILES, new CopyFilesAction(stateManager, this.getDialogService())),
-                factory.createMenuItem(StandardActions.EXTRACT_BIBTEX, new ExtractBibtexAction(stateManager)),
 
                 new SeparatorMenuItem(),
 

src/main/java/org/jabref/gui/actions/StandardActions.java

@@ -120,7 +120,7 @@ public enum StandardActions implements Action {
 
     NEW_ENTRY(Localization.lang("New entry"), IconTheme.JabRefIcons.ADD_ENTRY, KeyBinding.NEW_ENTRY),
     NEW_ARTICLE(Localization.lang("New article"), IconTheme.JabRefIcons.ADD_ARTICLE),
-    NEW_ENTRY_FROM_PLAINTEX(Localization.lang("New entry from plain text"), KeyBinding.NEW_FROM_PLAIN_TEXT),
+    NEW_ENTRY_FROM_PLAIN_TEXT(Localization.lang("New entry from plain text"), IconTheme.JabRefIcons.NEW_ENTRY_FROM_PLAIN_TEXT, KeyBinding.NEW_ENTRY_FROM_PLAIN_TEXT),
     LIBRARY_PROPERTIES(Localization.lang("Library properties")),
     EDIT_PREAMBLE(Localization.lang("Edit preamble")),
     EDIT_STRINGS(Localization.lang("Edit string constants"), IconTheme.JabRefIcons.EDIT_STRINGS, KeyBinding.EDIT_STRINGS),
@@ -138,7 +138,6 @@ public enum StandardActions implements Action {
     DOWNLOAD_FULL_TEXT(Localization.lang("Search full text documents online"), IconTheme.JabRefIcons.FILE_SEARCH, KeyBinding.DOWNLOAD_FULL_TEXT),
     CLEANUP_ENTRIES(Localization.lang("Cleanup entries"), IconTheme.JabRefIcons.CLEANUP_ENTRIES, KeyBinding.CLEANUP),
     SET_FILE_LINKS(Localization.lang("Automatically set file links"), KeyBinding.AUTOMATICALLY_LINK_FILES),
-    EXTRACT_BIBTEX(Localization.lang("Extract BibTeX from plain text")),
 
     HELP(Localization.lang("Online help"), IconTheme.JabRefIcons.HELP, KeyBinding.HELP),
     HELP_KEY_PATTERNS(Localization.lang("Help on key patterns"), IconTheme.JabRefIcons.HELP, KeyBinding.HELP),

src/main/java/org/jabref/gui/bibtexextractor/BibtexExtractorViewModel.java

@@ -3,40 +3,60 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import javax.swing.undo.UndoManager;
+
 import javafx.beans.property.SimpleStringProperty;
 import javafx.beans.property.StringProperty;
 
 import org.jabref.Globals;
+import org.jabref.gui.DialogService;
+import org.jabref.gui.StateManager;
+import org.jabref.gui.externalfiles.ImportHandler;
+import org.jabref.gui.externalfiletype.ExternalFileTypes;
+import org.jabref.gui.util.BackgroundTask;
+import org.jabref.gui.util.TaskExecutor;
+import org.jabref.logic.importer.fetcher.GrobidCitationFetcher;
+import org.jabref.logic.l10n.Localization;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
-import org.jabref.model.entry.types.EntryType;
-import org.jabref.model.entry.types.StandardEntryType;
+import org.jabref.model.util.FileUpdateMonitor;
+import org.jabref.preferences.JabRefPreferences;
 
 public class BibtexExtractorViewModel {
 
     private final StringProperty inputTextProperty = new SimpleStringProperty("");
-    private final BibDatabaseContext bibdatabaseContext;
-
-    public BibtexExtractorViewModel(BibDatabaseContext bibdatabaseContext) {
-        this.bibdatabaseContext = bibdatabaseContext;
+    private DialogService dialogService;
+    private GrobidCitationFetcher currentCitationfetcher;
+    private TaskExecutor taskExecutor;
+    private ImportHandler importHandler;
+
+    public BibtexExtractorViewModel(BibDatabaseContext bibdatabaseContext, DialogService dialogService,
+                                    JabRefPreferences jabRefPreferences, FileUpdateMonitor fileUpdateMonitor, TaskExecutor taskExecutor, UndoManager undoManager, StateManager stateManager) {
+        this.dialogService = dialogService;
+        currentCitationfetcher = new GrobidCitationFetcher(jabRefPreferences.getImportFormatPreferences());
+        this.taskExecutor = taskExecutor;
+        this.importHandler = new ImportHandler(dialogService, bibdatabaseContext, ExternalFileTypes.getInstance(), jabRefPreferences.getFilePreferences(), jabRefPreferences.getImportFormatPreferences(), jabRefPreferences.getUpdateFieldPreferences(), fileUpdateMonitor, undoManager, stateManager);
     }
 
     public StringProperty inputTextProperty() {
         return this.inputTextProperty;
     }
 
-    public void startExtraction() {
-
-        BibtexExtractor extractor = new BibtexExtractor();
-        BibEntry entity = extractor.extract(inputTextProperty.getValue());
-        this.bibdatabaseContext.getDatabase().insertEntry(entity);
-        trackNewEntry(StandardEntryType.Article);
+    public void startParsing() {
+        BackgroundTask.wrap(() -> currentCitationfetcher.performSearch(inputTextProperty.getValue()))
+                .onRunning(() -> dialogService.notify(Localization.lang("Your text is being parsed...")))
+                .onSuccess(parsedEntries -> {
+                    dialogService.notify(Localization.lang("%0 entries were parsed from your query.", String.valueOf(parsedEntries.size())));
+                    importHandler.importEntries(parsedEntries);
+                    for (BibEntry bibEntry : parsedEntries) {
+                        trackNewEntry(bibEntry);
+                    }
+                }).executeWith(taskExecutor);
     }
 
-    private void trackNewEntry(EntryType type) {
+    private void trackNewEntry(BibEntry bibEntry) {
         Map<String, String> properties = new HashMap<>();
-        properties.put("EntryType", type.getName());
-
-        Globals.getTelemetryClient().ifPresent(client -> client.trackEvent("NewEntry", properties, new HashMap<>()));
+        properties.put("EntryType", bibEntry.typeProperty().getValue().getName());
+        Globals.getTelemetryClient().ifPresent(client -> client.trackEvent("ParseWithGrobid", properties, new HashMap<>()));
     }
 }

src/main/java/org/jabref/gui/bibtexextractor/ExtractBibtexDialog.fxml

@@ -4,11 +4,17 @@
 <?import javafx.scene.control.DialogPane?>
 <?import javafx.scene.control.TextArea?>
 
+<?import javafx.scene.layout.VBox?>
+
 <DialogPane prefHeight="430.0" prefWidth="586.0" xmlns="http://javafx.com/javafx/8.0.171"
             xmlns:fx="http://javafx.com/fxml/1" fx:controller="org.jabref.gui.bibtexextractor.ExtractBibtexDialog">
     <content>
-        <TextArea fx:id="input" minHeight="-Infinity" prefHeight="350.0" prefWidth="586.0"/>
+        <VBox fx:id="contentVbox" minHeight="-Infinity" prefHeight="200.0" prefWidth="100.0">
+        <children>
+            <TextArea fx:id="input" minHeight="-Infinity" prefHeight="350.0" prefWidth="586.0" wrapText="true"/>
+        </children>
+        </VBox>
     </content>
-    <ButtonType fx:id="extractButtonType" buttonData="OK_DONE" text="%Extract"/>
+    <ButtonType fx:id="parseButtonType" buttonData="OK_DONE" text = "%Add to current library"/>
     <ButtonType fx:constant="CANCEL"/>
 </DialogPane>

src/main/java/org/jabref/gui/bibtexextractor/ExtractBibtexDialog.java

@@ -1,17 +1,22 @@
 package org.jabref.gui.bibtexextractor;
 
 import javax.inject.Inject;
+import javax.swing.undo.UndoManager;
 
 import javafx.fxml.FXML;
 import javafx.scene.control.Button;
 import javafx.scene.control.ButtonType;
 import javafx.scene.control.TextArea;
 import javafx.scene.control.Tooltip;
 
+import org.jabref.gui.DialogService;
 import org.jabref.gui.StateManager;
 import org.jabref.gui.util.BaseDialog;
+import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.util.FileUpdateMonitor;
+import org.jabref.preferences.JabRefPreferences;
 
 import com.airhacks.afterburner.views.ViewLoader;
 
@@ -20,31 +25,34 @@
  */
 public class ExtractBibtexDialog extends BaseDialog<Void> {
 
-    private final Button buttonExtract;
+    private final Button buttonParse;
     @FXML private TextArea input;
-    @FXML private ButtonType extractButtonType;
+    @FXML private ButtonType parseButtonType;
     private BibtexExtractorViewModel viewModel;
-
     @Inject private StateManager stateManager;
+    @Inject private DialogService dialogService;
+    @Inject private FileUpdateMonitor fileUpdateMonitor;
+    @Inject private TaskExecutor taskExecutor;
+    @Inject private UndoManager undoManager;
 
     public ExtractBibtexDialog() {
-
         ViewLoader.view(this)
                   .load()
                   .setAsDialogPane(this);
-
-        this.setTitle(Localization.lang("Input text to parse"));
-        buttonExtract = (Button) getDialogPane().lookupButton(extractButtonType);
-        buttonExtract.setTooltip(new Tooltip((Localization.lang("Starts the extraction of the BibTeX entry"))));
-        buttonExtract.setOnAction(e -> viewModel.startExtraction());
-        buttonExtract.disableProperty().bind(viewModel.inputTextProperty().isEmpty());
+        this.setTitle(Localization.lang("Plain References Parser"));
+        input.setPromptText(Localization.lang("Please enter the plain references to extract from separated by double empty lines."));
+        input.selectAll();
+
+        buttonParse = (Button) getDialogPane().lookupButton(parseButtonType);
+        buttonParse.setTooltip(new Tooltip((Localization.lang("Starts the extraction and adds the resulting entries to the currently opened database"))));
+        buttonParse.setOnAction(event -> viewModel.startParsing());
+        buttonParse.disableProperty().bind(viewModel.inputTextProperty().isEmpty());
     }
 
     @FXML
     private void initialize() {
         BibDatabaseContext database = stateManager.getActiveDatabase().orElseThrow(() -> new NullPointerException("Database null"));
-        this.viewModel = new BibtexExtractorViewModel(database);
-
+        this.viewModel = new BibtexExtractorViewModel(database, dialogService, JabRefPreferences.getInstance(), fileUpdateMonitor, taskExecutor,undoManager,stateManager);
         input.textProperty().bindBidirectional(viewModel.inputTextProperty());
     }
 }

src/main/java/org/jabref/gui/icon/IconTheme.java

@@ -298,7 +298,8 @@ public enum JabRefIcons implements JabRefIcon {
         OPEN_ABBREVIATION_LIST(MaterialDesignIcon.FOLDER_OUTLINE),
         REMOVE_ABBREVIATION_LIST(MaterialDesignIcon.FOLDER_REMOVE),
         ADD_ABBREVIATION(MaterialDesignIcon.PLAYLIST_PLUS),
-        REMOVE_ABBREVIATION(MaterialDesignIcon.PLAYLIST_MINUS);
+        REMOVE_ABBREVIATION(MaterialDesignIcon.PLAYLIST_MINUS),
+        NEW_ENTRY_FROM_PLAIN_TEXT(MaterialDesignIcon.PLUS_BOX);
 
         private final JabRefIcon icon;
 

src/main/java/org/jabref/gui/keyboard/KeyBinding.java

@@ -48,7 +48,7 @@ public enum KeyBinding {
     NEW_ARTICLE("New article", Localization.lang("New article"), "ctrl+shift+A", KeyBindingCategory.BIBTEX),
     NEW_BOOK("New book", Localization.lang("New book"), "ctrl+shift+B", KeyBindingCategory.BIBTEX),
     NEW_ENTRY("New entry", Localization.lang("New entry"), "ctrl+N", KeyBindingCategory.BIBTEX),
-    NEW_FROM_PLAIN_TEXT("New from plain text", Localization.lang("New from plain text"), "ctrl+shift+N", KeyBindingCategory.BIBTEX),
+    NEW_ENTRY_FROM_PLAIN_TEXT("New entry from plain text", Localization.lang("New entry from plain text"), "ctrl+shift+N", KeyBindingCategory.BIBTEX),
     NEW_INBOOK("New inbook", Localization.lang("New inbook"), "ctrl+shift+I", KeyBindingCategory.BIBTEX),
     NEW_MASTERSTHESIS("New mastersthesis", Localization.lang("New mastersthesis"), "ctrl+shift+M", KeyBindingCategory.BIBTEX),
     NEW_PHDTHESIS("New phdthesis", Localization.lang("New phdthesis"), "ctrl+shift+T", KeyBindingCategory.BIBTEX),

src/main/java/org/jabref/logic/importer/WebFetchers.java

@@ -18,6 +18,7 @@
 import org.jabref.logic.importer.fetcher.DoiFetcher;
 import org.jabref.logic.importer.fetcher.DoiResolution;
 import org.jabref.logic.importer.fetcher.GoogleScholar;
+import org.jabref.logic.importer.fetcher.GrobidCitationFetcher;
 import org.jabref.logic.importer.fetcher.GvkFetcher;
 import org.jabref.logic.importer.fetcher.IEEE;
 import org.jabref.logic.importer.fetcher.INSPIREFetcher;
@@ -98,6 +99,7 @@ public static SortedSet<SearchBasedFetcher> getSearchBasedFetchers(ImportFormatP
         set.add(new CiteSeer());
         set.add(new DOAJFetcher(importFormatPreferences));
         set.add(new IEEE(importFormatPreferences));
+        set.add(new GrobidCitationFetcher(importFormatPreferences));
         return set;
     }
 

src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java

@@ -0,0 +1,77 @@
+package org.jabref.logic.importer.fetcher;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.ParseException;
+import org.jabref.logic.importer.SearchBasedFetcher;
+import org.jabref.logic.importer.fileformat.BibtexParser;
+import org.jabref.logic.importer.util.GrobidService;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.util.DummyFileUpdateMonitor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GrobidCitationFetcher implements SearchBasedFetcher {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(GrobidCitationFetcher.class);
+    private static final String GROBID_URL = "http://grobid.cm.in.tum.de:8070";
+    private ImportFormatPreferences importFormatPreferences;
+    private GrobidService grobidService;
+
+    public GrobidCitationFetcher(ImportFormatPreferences importFormatPreferences) {
+        this.importFormatPreferences = importFormatPreferences;
+        this.grobidService = new GrobidService(GROBID_URL);
+    }
+
+    /**
+     * Passes request to grobid server, using consolidateCitations option to improve result.
+     * Takes a while, since the server has to look up the entry.
+     * @return A BibTeX-String if extraction is successful and an empty String otherwise.
+     */
+    private String parseUsingGrobid(String plainText) {
+        try {
+            return grobidService.processCitation(plainText, GrobidService.ConsolidateCitations.WITH_METADATA);
+        } catch (IOException e) {
+            LOGGER.debug("Could not process citation", e);
+            return "";
+        }
+    }
+
+    private Optional<BibEntry> parseBibToBibEntry(String bibtexString) {
+        try {
+            return BibtexParser.singleFromString(bibtexString,
+                    importFormatPreferences, new DummyFileUpdateMonitor());
+        } catch (ParseException e) {
+            return Optional.empty();
+        }
+    }
+
+    @Override
+    public List<BibEntry> performSearch(String query) {
+        List<String> plainReferences = Arrays.stream( query.split( "\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+" ) )
+              .map(String::trim)
+              .filter(str -> !str.isBlank())
+              .collect(Collectors.toCollection(ArrayList::new));
+        if (plainReferences.isEmpty()) {
+            return Collections.emptyList();
+        } else {
+            return plainReferences.stream()
+                  .map(reference -> parseBibToBibEntry(parseUsingGrobid(reference)))
+                  .flatMap(Optional::stream)
+                  .collect(Collectors.toList());
+        }
+    }
+
+    @Override
+    public String getName() {
+        return "GROBID";
+    }
+}

src/main/java/org/jabref/logic/importer/util/GrobidService.java

@@ -0,0 +1,57 @@
+package org.jabref.logic.importer.util;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+
+import org.jabref.logic.net.URLDownload;
+
+/**
+ * Implements an API to a GROBID server, as described at
+ * https://grobid.readthedocs.io/en/latest/Grobid-service/#grobid-web-services
+ * <p>
+ * Note: Currently a custom GROBID server is used...
+ * https://github.com/NikodemKch/grobid
+ * <p>
+ * The methods are structured to match the GROBID server api.
+ * Each method corresponds to a GROBID service request. Only the ones already used are already implemented.
+ */
+public class GrobidService {
+
+    public enum ConsolidateCitations {
+        NO(0), WITH_METADATA(1), WITH_DOI_ONLY(2);
+        private int code;
+
+        ConsolidateCitations(int code) {
+            this.code = code;
+        }
+
+        public int getCode() {
+            return this.code;
+        }
+    }
+
+    String grobidServerURL;
+
+    public GrobidService(String grobidServerURL) {
+        this.grobidServerURL = grobidServerURL;
+    }
+
+    /**
+     * @return A BibTeX-String if extraction is successful and an IOException otherwise.
+     */
+    public String processCitation(String rawCitation, ConsolidateCitations consolidateCitations) throws IOException {
+        rawCitation = URLEncoder.encode(rawCitation, StandardCharsets.UTF_8);
+        URLDownload urlDownload = new URLDownload(grobidServerURL
+                + "/api/processCitation");
+        //urlDownload.addHeader("Accept", "application/x-bibtex"); //TODO: Uncomment as soon as the default GROBID server is used.
+        urlDownload.setPostData("citations=" + rawCitation + "&consolidateCitations=" + consolidateCitations);
+        String httpResponse = urlDownload.asString();
+
+        if (httpResponse == null || httpResponse.equals("@misc{-1,\n\n}\n")) { //This filters empty BibTeX entries
+            throw new IOException("The GROBID server response does not contain anything.");
+        }
+
+        return httpResponse;
+    }
+}