Feature/enable lucene query parsing

CHANGELOG.md

@@ -11,6 +11,8 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 
 ### Added
 
+- We added a query parser and mapping layer to enable conversion of queries formulated in simplified lucene syntax by the user into api queries. [#6799](https://github.com/JabRef/jabref/pull/6799)
+
 ### Changed
 
 ### Fixed

build.gradle

@@ -137,6 +137,10 @@ dependencies {
     antlr4 'org.antlr:antlr4:4.8-1'
     implementation 'org.antlr:antlr4-runtime:4.8-1'
 
+    implementation (group: 'org.apache.lucene', name: 'lucene-queryparser', version: '8.6.1') {
+        exclude group: 'org.apache.lucene', module: 'lucene-sandbox'
+    }
+
     implementation group: 'org.mariadb.jdbc', name: 'mariadb-java-client', version: '2.6.2'
 
     implementation 'org.postgresql:postgresql:42.2.16'

docs/adr/0015-support-an-abstract-query-syntax-for-query-conversion.md

@@ -0,0 +1,56 @@
+# Query syntax design
+
+## Context and Problem Statement
+
+All libraries use their own query syntax for advanced search options. To increase usability, users should be able to formulate their (abstract) search queries in a query syntax that can be mapped to the library specific search queries. To achieve this, the query has to be parsed into an AST.
+
+Which query syntax should be used for the abstract queries?
+Which features should the syntax support?
+
+## Considered Options
+
+* Use a simplified syntax that is derived of the [lucene](https://lucene.apache.org/core/8_6_1/queryparser/org/apache/lucene/queryparser/classic/package-summary.html) query syntax
+* Formulate a own query syntax
+
+## Decision Outcome
+
+Chosen option: "Use a syntax that is derived of the lucene query syntax", because only option that is already known, and easy to implemenent.
+Furthermore parsers for lucene already exist and are tested.
+For simplicitly, and lack of universal capabilities across fetchers, only basic query features and therefor syntax is supported:
+
+* All terms in the query are whitespace separated and will be ANDed
+* Default and certain fielded terms are supported
+* Fielded Terms:
+  * author
+  * title
+  * journal
+  * year (for single year)
+  * year-range (for range e.g. year-range:2012-2015)
+* The journal, year, and year-range fields should only be populated once in each query
+* Example:
+  * \[author:"Igor Steinmacher" author:"Christoph Treude" year:2017\] will be converted to
+  * \[author:"Igor Steinmacher" AND author:"Christoph Treude" AND year:2017\]
+
+### Positive Consequences
+
+* Already tested
+* Well Known
+* Easy to implement
+* Can use an existing parser
+
+## Pros and Cons of the Options
+
+### Use a syntax that is derived of the lucene query syntax
+
+* Good, because already exists
+* Good, because already well known
+* Good, because there already exists a [parser for lucene syntax](https://lucene.apache.org/core/8_0_0/queryparser/org/apache/lucene/queryparser/flexible/standard/StandardQueryParser.html)
+* Good, because capabilities of query conversion can easily be extended using the [flexible lucene framework](https://lucene.apache.org/core/8_0_0/queryparser/org/apache/lucene/queryparser/flexible/core/package-summary.html)
+
+### Formulate a own query syntax
+
+* Good, because allows for flexibility
+* Bad, because needs a new parser (has to be decided whether to use [ANTLR](https://www.antlr.org/), [JavaCC](https://javacc.github.io/javacc/), or [LogicNG](https://github.com/logic-ng/LogicNG))
+* Bad, because has to be tested
+* Bad, because syntax is not well known
+* Bad, because the design should be easily extensible, requires an appropriate design (high effort)

src/main/java/module-info.java

@@ -89,4 +89,6 @@
     requires flexmark.util.ast;
     requires flexmark.util.data;
     requires com.h2database.mvstore;
+    requires lucene.queryparser;
+    requires lucene.core;
 }

src/main/java/org/jabref/logic/importer/QueryConverter.java

@@ -0,0 +1,48 @@
+package org.jabref.logic.importer;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.jabref.logic.importer.fetcher.ComplexSearchQuery;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
+import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+
+/**
+ * This class converts a query string written in lucene syntax into a complex search query.
+ *
+ * For simplicity this is limited to fielded data and the boolean AND operator.
+ */
+public class QueryConverter {
+
+    /**
+     * Converts the given query string into a complex query using lucene.
+     * Note: For unique fields, the alphabetically first instance in the query string is used in the complex query.
+     *
+     * @param queryString The given query string
+     * @return A complex query containing all fields of the query string
+     * @throws QueryNodeException Error during parsing
+     */
+    public ComplexSearchQuery convertQueryStringIntoComplexQuery(String queryString) throws QueryNodeException {
+        ComplexSearchQuery.ComplexSearchQueryBuilder builder = ComplexSearchQuery.builder();
+
+        StandardQueryParser parser = new StandardQueryParser();
+        Query luceneQuery = parser.parse(queryString, "default");
+        Set<Term> terms = new HashSet<>();
+        // This implementation collects all terms from the leaves of the query tree independent of the internal boolean structure
+        // If further capabilities are required in the future the visitor and ComplexSearchQuery has to be adapted accordingly.
+        QueryVisitor visitor = QueryVisitor.termCollector(terms);
+        luceneQuery.visit(visitor);
+
+        List<Term> sortedTerms = new ArrayList<>(terms);
+        sortedTerms.sort(Comparator.comparing(Term::text));
+        builder.terms(sortedTerms);
+        return builder.build();
+    }
+}

src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java

@@ -1,6 +1,7 @@
 package org.jabref.logic.importer;
 
 import java.util.List;
+import java.util.Optional;
 
 import org.jabref.logic.importer.fetcher.ComplexSearchQuery;
 import org.jabref.model.entry.BibEntry;
@@ -26,7 +27,8 @@ public interface SearchBasedFetcher extends WebFetcher {
      * @return a list of {@link BibEntry}, which are matched by the query (may be empty)
      */
     default List<BibEntry> performComplexSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException {
-        // Default Implementation behaves like perform search using the default field as query
-        return performSearch(complexSearchQuery.getDefaultField().orElse(""));
+        // Default Implementation behaves like perform search using the default field phrases as query
+        Optional<List<String>> defaultPhrases = complexSearchQuery.getDefaultFieldPhrases();
+        return performSearch(defaultPhrases.map(strings -> String.join(" ", strings)).orElse(""));
     }
 }

src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java

@@ -7,6 +7,7 @@
 import java.net.URL;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 
 import org.jabref.logic.importer.fetcher.ComplexSearchQuery;
 import org.jabref.model.cleanup.Formatter;
@@ -83,8 +84,9 @@ private List<BibEntry> getBibEntries(URL urlForQuery) throws FetcherException {
     }
 
     default URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery) throws URISyntaxException, MalformedURLException, FetcherException {
-        // Default Implementation behaves like getURLForQuery using the default field as query
-        return this.getURLForQuery(complexSearchQuery.getDefaultField().orElse(""));
+        // Default Implementation behaves like getURLForQuery using the default field phrases as query
+        Optional<List<String>> defaultPhrases = complexSearchQuery.getDefaultFieldPhrases();
+        return this.getURLForQuery(defaultPhrases.map(strings -> String.join(" ", strings)).orElse(""));
     }
 
     /**

src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java

@@ -269,7 +269,7 @@ public List<BibEntry> performComplexSearch(ComplexSearchQuery complexSearchQuery
         complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("jr:" + journal));
         // Since ArXiv API does not support year search, we ignore the year related terms
         complexSearchQuery.getToYear().ifPresent(year -> searchTerms.add(year.toString()));
-        complexSearchQuery.getDefaultField().ifPresent(defaultField -> searchTerms.add(defaultField));
+        complexSearchQuery.getDefaultFieldPhrases().ifPresent(searchTerms::addAll);
         String complexQueryString = String.join(" AND ", searchTerms);
         return performSearch(complexQueryString);
     }

src/main/java/org/jabref/logic/importer/fetcher/ComplexSearchQuery.java

@@ -1,23 +1,26 @@
 package org.jabref.logic.importer.fetcher;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 
 import org.jabref.model.strings.StringUtil;
 
+import org.apache.lucene.index.Term;
+
 public class ComplexSearchQuery {
     // Field for non-fielded search
-    private final String defaultField;
+    private final List<String> defaultField;
     private final List<String> authors;
     private final List<String> titlePhrases;
     private final Integer fromYear;
     private final Integer toYear;
     private final Integer singleYear;
     private final String journal;
 
-    private ComplexSearchQuery(String defaultField, List<String> authors, List<String> titlePhrases, Integer fromYear, Integer toYear, Integer singleYear, String journal) {
+    private ComplexSearchQuery(List<String> defaultField, List<String> authors, List<String> titlePhrases, Integer fromYear, Integer toYear, Integer singleYear, String journal) {
         this.defaultField = defaultField;
         this.authors = authors;
         this.titlePhrases = titlePhrases;
@@ -28,7 +31,7 @@ private ComplexSearchQuery(String defaultField, List<String> authors, List<Strin
         this.singleYear = singleYear;
     }
 
-    public Optional<String> getDefaultField() {
+    public Optional<List<String>> getDefaultFieldPhrases() {
         return Optional.ofNullable(defaultField);
     }
 
@@ -61,22 +64,26 @@ public static ComplexSearchQueryBuilder builder() {
     }
 
     public static class ComplexSearchQueryBuilder {
-        private String defaultField;
+        private List<String> defaultFieldPhrases;
         private List<String> authors;
         private List<String> titlePhrases;
         private String journal;
         private Integer fromYear;
         private Integer toYear;
         private Integer singleYear;
 
-        public ComplexSearchQueryBuilder() {
+        private ComplexSearchQueryBuilder() {
         }
 
-        public ComplexSearchQueryBuilder defaultField(String defaultField) {
-            if (Objects.requireNonNull(defaultField).isBlank()) {
+        public ComplexSearchQueryBuilder defaultFieldPhrase(String defaultFieldPhrase) {
+            if (Objects.requireNonNull(defaultFieldPhrase).isBlank()) {
                 throw new IllegalArgumentException("Parameter must not be blank");
             }
-            this.defaultField = defaultField;
+            if (Objects.isNull(defaultFieldPhrases)) {
+                this.defaultFieldPhrases = new ArrayList<>();
+            }
+            // Strip all quotes before wrapping
+            this.defaultFieldPhrases.add(String.format("\"%s\"", defaultFieldPhrase.replace("\"", "")));
             return this;
         }
 
@@ -135,6 +142,21 @@ public ComplexSearchQueryBuilder journal(String journal) {
             return this;
         }
 
+        public ComplexSearchQueryBuilder terms(Collection<Term> terms) {
+            terms.forEach(term -> {
+                String termText = term.text();
+                switch (term.field().toLowerCase()) {
+                    case "author" -> this.author(termText);
+                    case "title" -> this.titlePhrase(termText);
+                    case "journal" -> this.journal(termText);
+                    case "year" -> this.singleYear(Integer.valueOf(termText));
+                    case "year-range" -> this.fromYearAndToYear(Integer.valueOf(termText.split("-")[0]), Integer.valueOf(termText.split("-")[1]));
+                    case "default" -> this.defaultFieldPhrase(termText);
+                }
+            });
+            return this;
+        }
+
         /**
          * Instantiates the AdvancesSearchConfig from the provided Builder parameters
          * If all text fields are empty an empty optional is returned
@@ -147,11 +169,11 @@ public ComplexSearchQuery build() throws IllegalStateException {
             if (textSearchFieldsAndYearFieldsAreEmpty()) {
                 throw new IllegalStateException("At least one text field has to be set");
             }
-            return new ComplexSearchQuery(defaultField, authors, titlePhrases, fromYear, toYear, singleYear, journal);
+            return new ComplexSearchQuery(defaultFieldPhrases, authors, titlePhrases, fromYear, toYear, singleYear, journal);
         }
 
         private boolean textSearchFieldsAndYearFieldsAreEmpty() {
-            return StringUtil.isBlank(defaultField) && this.stringListIsBlank(titlePhrases) &&
+            return this.stringListIsBlank(defaultFieldPhrases) && this.stringListIsBlank(titlePhrases) &&
                     this.stringListIsBlank(authors) && StringUtil.isBlank(journal) && yearFieldsAreEmpty();
         }
 

src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java

@@ -205,7 +205,7 @@ public List<BibEntry> performComplexSearch(ComplexSearchQuery complexSearchQuery
 
     private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery) {
         List<String> searchTerms = new ArrayList<>();
-        complexSearchQuery.getDefaultField().ifPresent(defaultField -> searchTerms.add(defaultField));
+        complexSearchQuery.getDefaultFieldPhrases().ifPresent(searchTerms::addAll);
         complexSearchQuery.getAuthors().ifPresent(authors -> authors.forEach(author -> searchTerms.add("author:" + author)));
         complexSearchQuery.getTitlePhrases().ifPresent(phrases -> searchTerms.add("allintitle:" + String.join(" ", phrases)));
         complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("source:" + journal));

src/main/java/org/jabref/logic/importer/fetcher/IEEE.java

@@ -236,7 +236,7 @@ public Optional<HelpFile> getHelpPage() {
     public URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery) throws URISyntaxException, MalformedURLException {
         URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles");
         uriBuilder.addParameter("apikey", API_KEY);
-        complexSearchQuery.getDefaultField().ifPresent(defaultField -> uriBuilder.addParameter("querytext", defaultField));
+        complexSearchQuery.getDefaultFieldPhrases().ifPresent(defaultFieldPhrases -> uriBuilder.addParameter("querytext", String.join(" AND ", defaultFieldPhrases)));
         complexSearchQuery.getAuthors().ifPresent(authors ->
                 uriBuilder.addParameter("author", String.join(" AND ", authors)));
         complexSearchQuery.getTitlePhrases().ifPresent(articleTitlePhrases ->

src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java

@@ -176,7 +176,7 @@ private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery
         complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("journal:" + journal));
         // Since Springer API does not support year range search, we ignore formYear and toYear and use "singleYear" only
         complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add("year:" + year.toString()));
-        complexSearchQuery.getDefaultField().ifPresent(defaultField -> searchTerms.add(defaultField));
+        complexSearchQuery.getDefaultFieldPhrases().ifPresent(searchTerms::addAll);
         return String.join(" AND ", searchTerms);
     }
 

src/test/java/org/jabref/logic/importer/QueryConverterTest.java

@@ -0,0 +1,59 @@
+package org.jabref.logic.importer;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.jabref.logic.importer.fetcher.ComplexSearchQuery;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class QueryConverterTest {
+    QueryConverter converter = new QueryConverter();
+
+    @Test
+    public void convertAuthorField() throws Exception {
+        ComplexSearchQuery searchQuery = new QueryConverter().convertQueryStringIntoComplexQuery("author:\"Igor Steinmacher\"");
+        assertEquals(List.of("\"Igor Steinmacher\""), searchQuery.getAuthors().get());
+    }
+
+    @Test
+    public void convertDefaultField() throws Exception {
+        ComplexSearchQuery searchQuery = new QueryConverter().convertQueryStringIntoComplexQuery("\"default value\"");
+        assertEquals(List.of("\"default value\""), searchQuery.getDefaultFieldPhrases().get());
+    }
+
+    @Test
+    public void convertExplicitDefaultField() throws Exception {
+        ComplexSearchQuery searchQuery = new QueryConverter().convertQueryStringIntoComplexQuery("default:\"default value\"");
+        assertEquals(List.of("\"default value\""), searchQuery.getDefaultFieldPhrases().get());
+    }
+
+    @Test
+    public void convertJournalField() throws Exception {
+        ComplexSearchQuery searchQuery = new QueryConverter().convertQueryStringIntoComplexQuery("journal:\"Nature\"");
+        assertEquals("\"Nature\"", searchQuery.getJournal().get());
+    }
+
+    @Test
+    public void convertYearField() throws Exception {
+        ComplexSearchQuery searchQuery = new QueryConverter().convertQueryStringIntoComplexQuery("year:2015");
+        assertEquals(2015, searchQuery.getSingleYear().get());
+    }
+
+    @Test
+    public void convertYearRangeField() throws Exception {
+        ComplexSearchQuery searchQuery = new QueryConverter().convertQueryStringIntoComplexQuery("year-range:2012-2015");
+        assertEquals(2012, searchQuery.getFromYear().get());
+        assertEquals(2015, searchQuery.getToYear().get());
+    }
+
+    @Test
+    public void convertMultipleValuesWithTheSameField() throws Exception {
+        ComplexSearchQuery searchQuery = new QueryConverter().convertQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\"");
+        List<String> sortedAuthors = searchQuery.getAuthors().get();
+        Collections.sort(sortedAuthors);
+        assertEquals(List.of("\"Christoph Treude\"", "\"Igor Steinmacher\""), sortedAuthors);
+    }
+}