Jstor Fetcher

CHANGELOG.md

@@ -15,6 +15,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 - We added a query parser and mapping layer to enable conversion of queries formulated in simplified lucene syntax by the user into api queries. [#6799](https://github.com/JabRef/jabref/pull/6799)
 - We added some basic functionality to customise the look of JabRef by importing a css theme file. [#5790](https://github.com/JabRef/jabref/issues/5790)
 - We added connection check function in network preference setting [#6560](https://github.com/JabRef/jabref/issues/6560)
+- We added a new fetcher to enable users to search jstor.org [#6627](https://github.com/JabRef/jabref/issues/6627)
 
 ### Changed
 

src/main/java/org/jabref/logic/importer/WebFetchers.java

@@ -1,12 +1,5 @@
 package org.jabref.logic.importer;
 
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
 import org.jabref.logic.importer.fetcher.ACS;
 import org.jabref.logic.importer.fetcher.ApsFetcher;
 import org.jabref.logic.importer.fetcher.ArXiv;
@@ -26,6 +19,7 @@
 import org.jabref.logic.importer.fetcher.INSPIREFetcher;
 import org.jabref.logic.importer.fetcher.IacrEprintFetcher;
 import org.jabref.logic.importer.fetcher.IsbnFetcher;
+import org.jabref.logic.importer.fetcher.JstorFetcher;
 import org.jabref.logic.importer.fetcher.LibraryOfCongress;
 import org.jabref.logic.importer.fetcher.MathSciNet;
 import org.jabref.logic.importer.fetcher.MedlineFetcher;
@@ -42,6 +36,13 @@
 import org.jabref.model.entry.identifier.DOI;
 import org.jabref.model.entry.identifier.Identifier;
 
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
 import static org.jabref.model.entry.field.StandardField.EPRINT;
 import static org.jabref.model.entry.field.StandardField.ISBN;
 
@@ -104,6 +105,7 @@ public static SortedSet<SearchBasedFetcher> getSearchBasedFetchers(ImportFormatP
         set.add(new IEEE(importFormatPreferences));
         set.add(new CompositeSearchBasedFetcher(set, 30));
         set.add(new CollectionOfComputerScienceBibliographiesFetcher(importFormatPreferences));
+        set.add(new JstorFetcher(importFormatPreferences));
         return set;
     }
 

src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java

@@ -0,0 +1,76 @@
+package org.jabref.logic.importer.fetcher;
+
+import com.google.common.base.Charsets;
+import com.microsoft.applicationinsights.core.dependencies.apachecommons.io.IOUtils;
+import org.apache.http.client.utils.URIBuilder;
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.ParseException;
+import org.jabref.logic.importer.Parser;
+import org.jabref.logic.importer.SearchBasedParserFetcher;
+import org.jabref.logic.importer.fileformat.BibtexParser;
+import org.jabref.logic.util.OS;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.util.DummyFileUpdateMonitor;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Fetcher for jstor.org
+ **/
+public class JstorFetcher implements SearchBasedParserFetcher {
+
+    private static final String SEARCH_HOST = "https://www.jstor.org/open/search/";
+    private static final String CITE_HOST = "https://www.jstor.org/citation/text";
+
+    private final ImportFormatPreferences importFormatPreferences;
+
+    public JstorFetcher(ImportFormatPreferences importFormatPreferences) {
+        this.importFormatPreferences = importFormatPreferences;
+    }
+
+    @Override
+    public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException {
+        URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST);
+        uriBuilder.addParameter("Query", query);
+        return uriBuilder.build().toURL();
+    }
+
+    @Override
+    public Parser getParser() {
+        return inputStream -> {
+            String response = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining(OS.NEWLINE));
+            List<BibEntry> entries = new ArrayList<>();
+
+            Document doc = Jsoup.parse(response);
+
+            List<Element> elements = doc.body().getElementsByClass("cite-this-item");
+            for (Element element : elements) {
+                BibtexParser parser = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor());
+                String id = element.attr("href").replace("citation/info/", "");
+                try {
+                    String data = IOUtils.toString(new URL(CITE_HOST + id), Charsets.UTF_8);
+                    entries.addAll(parser.parseEntries(data));
+                } catch (IOException e) {
+                    throw new ParseException("could not download data from jstor.org", e);
+                }
+            }
+            return entries;
+        };
+    }
+
+    @Override
+    public String getName() {
+        return "JSTOR";
+    }
+}

src/test/java/org/jabref/logic/importer/fetcher/JstorFetcherTest.java

@@ -0,0 +1,40 @@
+package org.jabref.logic.importer.fetcher;
+
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.StandardField;
+import org.jabref.model.entry.types.StandardEntryType;
+import org.jabref.testutils.category.FetcherTest;
+import org.junit.jupiter.api.Test;
+import org.mockito.Answers;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+
+@FetcherTest
+public class JstorFetcherTest {
+
+    private final JstorFetcher fetcher = new JstorFetcher(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS));
+
+    @Test
+    void searchByTitle() throws Exception {
+        BibEntry expected = new BibEntry(StandardEntryType.Article)
+                .withCitationKey("10.2307/90002164")
+                .withField(StandardField.AUTHOR, "Yang Yanxia")
+                .withField(StandardField.TITLE, "Test Anxiety Analysis of Chinese College Students in Computer-based Spoken English Test")
+                .withField(StandardField.ISSN, "11763647, 14364522")
+                .withField(StandardField.JOURNAL, "Journal of Educational Technology & Society")
+                .withField(StandardField.ABSTRACT, "ABSTRACT Test anxiety was a commonly known or assumed factor that could greatly influence performance of test takers. With the employment of designed questionnaires and computer-based spoken English test, this paper explored test anxiety manifestation of Chinese college students from both macro and micro aspects, and found out that the major anxiety in computer-based spoken English test was spoken English test anxiety, which consisted of test anxiety and communication apprehension. Regard to proximal test anxiety, the causes listed in proper order as low spoken English abilities, lack of speaking techniques, anxiety from the evaluative process and inadaptability with computer-based spoken English test format. As to distal anxiety causes, attitude toward learning spoken English and self-evaluation of speaking abilities were significantly negatively correlated with test anxiety. Besides, as test anxiety significantly associated often with test performance, a look at pedagogical implications has been discussed in this paper.")
+                .withField(StandardField.PUBLISHER, "International Forum of Educational Technology & Society")
+                .withField(StandardField.NUMBER, "2")
+                .withField(StandardField.PAGES, "63--73")
+                .withField(StandardField.VOLUME, "20")
+                .withField(StandardField.URL, "http://www.jstor.org/stable/90002164")
+                .withField(StandardField.YEAR, "2017");
+
+        List<BibEntry> entries = fetcher.performSearch("Test Anxiety Analysis of Chinese College Students in Computer-based Spoken English Test");
+        assertTrue(entries.contains(expected));
+    }
+}