From cd8a19d1080875672e1922c999c1899c78048523 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Tue, 22 Oct 2019 20:04:32 +0200 Subject: [PATCH 01/34] Changed performSearch to default implementation Removes method as it is essentially the same as the default implementation. --- CHANGELOG.md | 1 + .../fetcher/AstrophysicsDataSystem.java | 47 ++----------------- 2 files changed, 6 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796c8e28ff5..4169fb01b44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `# - We added an option to show the preview as an extra tab in the entry editor (instead of in a split view). [#5244](https://github.com/JabRef/jabref/issues/5244) - A custom Open/LibreOffice jstyle file now requires a layout line for the entry type `default` [#5452](https://github.com/JabRef/jabref/issues/5452) - The entry editor is now open by default when JabRef starts up. [#5460](https://github.com/JabRef/jabref/issues/5460) +- We changed the ADS fetcher to use the new ADS API [#4949](https://github.com/JabRef/jabref/issues/4949) ### Fixed diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 708039914e3..2ac10652bc9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -1,13 +1,8 @@ package org.jabref.logic.importer.fetcher; -import java.io.IOException; -import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.net.URLConnection; -import java.util.Collections; -import java.util.List; import java.util.Objects; import java.util.Optional; @@ -21,29 +16,24 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.logic.l10n.Localization; -import org.jabref.logic.net.URLDownload; import org.jabref.model.cleanup.FieldFormatterCleanup; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; /** * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) - * - * Search query-based: http://adsabs.harvard.edu/basic_search.html - * Entry -based: http://adsabs.harvard.edu/abstract_service.html - * - * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON - * (or at least needs multiple calls to get BibTeX, status: September 2016) + *

+ * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html + *

+ * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple + * calls to get BibTeX, status: September 2016) */ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { @@ -126,33 +116,6 @@ public Parser getParser() { return new BibtexParser(preferences, new DummyFileUpdateMonitor()); } - @Override - public List performSearch(String query) throws FetcherException { - if (StringUtil.isBlank(query)) { - return Collections.emptyList(); - } - - try { - URLConnection connection = getURLForQuery(query).openConnection(); - connection.setRequestProperty("User-Agent", URLDownload.USER_AGENT); - try (InputStream stream = connection.getInputStream()) { - List fetchedEntries = getParser().parseEntries(stream); - - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - return fetchedEntries; - } catch (IOException e) { - throw new FetcherException("An I/O exception occurred", e); - } - } catch (URISyntaxException | MalformedURLException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("An I/O exception occurred", e); - } catch (ParseException e) { - throw new FetcherException("Error occurred when parsing entry", Localization.lang("Error occurred when parsing entry"), e); - } - } - @Override public void doPostCleanup(BibEntry entry) { new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); From bbe261828b0f4556d26324eb308f15a2d772f0ff Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Tue, 22 Oct 2019 23:25:30 +0200 Subject: [PATCH 02/34] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- CHANGELOG.md | 2 +- src/main/java/module-info.java | 1 + .../jabref/logic/importer/WebFetchers.java | 4 + .../fetcher/NewAstrophysicsDataSystem.java | 180 ++++++++++++++++++ 4 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4169fb01b44..ed9808660c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `# - We added an option to show the preview as an extra tab in the entry editor (instead of in a split view). [#5244](https://github.com/JabRef/jabref/issues/5244) - A custom Open/LibreOffice jstyle file now requires a layout line for the entry type `default` [#5452](https://github.com/JabRef/jabref/issues/5452) - The entry editor is now open by default when JabRef starts up. [#5460](https://github.com/JabRef/jabref/issues/5460) -- We changed the ADS fetcher to use the new ADS API [#4949](https://github.com/JabRef/jabref/issues/4949) +- We add a new ADS fetcher to use the new ADS API [#4949](https://github.com/JabRef/jabref/issues/4949) ### Fixed diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 791e2e50698..bfb3a41fdcc 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -65,4 +65,5 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; + requires json; } diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index 66319f5e6b4..4161786d0c4 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -27,6 +27,7 @@ import org.jabref.logic.importer.fetcher.LibraryOfCongress; import org.jabref.logic.importer.fetcher.MathSciNet; import org.jabref.logic.importer.fetcher.MedlineFetcher; +import org.jabref.logic.importer.fetcher.NewAstrophysicsDataSystem; import org.jabref.logic.importer.fetcher.OpenAccessDoi; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; @@ -88,6 +89,7 @@ public static SortedSet getSearchBasedFetchers(ImportFormatP set.add(new GvkFetcher()); set.add(new MedlineFetcher()); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); set.add(new ZbMATH(importFormatPreferences)); set.add(new ACMPortalFetcher(importFormatPreferences)); @@ -108,6 +110,7 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new ArXiv(importFormatPreferences)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new DiVA(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); @@ -127,6 +130,7 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) { SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java new file mode 100644 index 00000000000..8d93f6ee4aa --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -0,0 +1,180 @@ +package org.jabref.logic.importer.fetcher; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.EntryBasedParserFetcher; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.IdBasedParserFetcher; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.logic.net.URLDownload; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.strings.StringUtil; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.apache.http.client.utils.URIBuilder; +import org.json.JSONObject; + +//TODO Replace Old ADS after the new one is mature + +/** + * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) + *

+ * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html + *

+ * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple + * calls to get BibTeX, status: September 2016) + */ +public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { + + private static String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; + + private static String API_KEY = ""; //TODO Add API Token + + private final ImportFormatPreferences preferences; + + public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { + this.preferences = Objects.requireNonNull(preferences); + } + + private String buildPostData(String... bibcodes) { + JSONObject obj = new JSONObject(); + obj.put("bibcode", bibcodes); + return obj.toString(); + } + + @Override + public String getName() { + return "New SAO/NASA Astrophysics Data System"; + } + + @Override + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); + } + + @Override + public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { + StringBuilder stringBuilder = new StringBuilder(); + + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); + Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author" + a); + + if (title.isPresent()) { + stringBuilder.append(title.get()) + .append(author.map(s -> " OR " + s) + .orElse("")); + } else { + stringBuilder.append(author.orElse("")); + } + String query = stringBuilder.toString().trim(); + + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); + } + + @Override + public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + + @Override + public Optional getHelpPage() { + return Optional.of(HelpFile.FETCHER_ADS); + } + + @Override + public Parser getParser() { + return new BibtexParser(preferences, new DummyFileUpdateMonitor()); + } + + @Override + public void doPostCleanup(BibEntry entry) { + + } + + @Override + public List performSearch(BibEntry entry) throws FetcherException { + return Collections.emptyList(); + } + + @Override + public Optional performSearchById(String identifier) throws FetcherException { + + Optional> results = performSearchByIds(identifier); + if (results.isEmpty()) { + return Optional.empty(); + } + + List fetchedEntries = results.get(); + + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } + + private Optional> performSearchByIds(String... identifiers) throws FetcherException { + + long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); + if (idCount == 0) { + return Optional.empty(); + } + + try { + + String postData = buildPostData(identifiers); + URLDownload download = new URLDownload(getURLForID("")); + download.addHeader("Authorization", "Bearer " + API_KEY); + download.addHeader("ContentType", "application/json"); + download.setPostData(postData); + download.asString(); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + List fetchedEntries = getParser().parseEntries(obj.getString("export")); + + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return Optional.of(fetchedEntries); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } catch (ParseException e) { + throw new FetcherException("An internal parser error occurred", e); + } + } + + @Override + public List performSearch(String query) throws FetcherException { + + return Collections.emptyList(); + } +} From 167a617828eab8dea6c9fcef94b71b949fd0d87c Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 20:02:29 +0200 Subject: [PATCH 03/34] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- .../fetcher/NewAstrophysicsDataSystem.java | 103 +++++++++++++----- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 8d93f6ee4aa..0ca438eff28 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -4,6 +4,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -26,6 +27,8 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; //TODO Replace Old ADS after the new one is mature @@ -40,8 +43,8 @@ */ public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - private static String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; - private static String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; + private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; private static String API_KEY = ""; //TODO Add API Token @@ -75,7 +78,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR StringBuilder stringBuilder = new StringBuilder(); Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); - Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author" + a); + Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author:" + a); if (title.isPresent()) { stringBuilder.append(title.get()) @@ -89,6 +92,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); builder.addParameter("fl", "bibcode"); + builder.addParameter("rows", "20"); return builder.build().toURL(); } @@ -114,18 +118,69 @@ public void doPostCleanup(BibEntry entry) { @Override public List performSearch(BibEntry entry) throws FetcherException { - return Collections.emptyList(); + + if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForEntry(entry)); + String[] bibcodeArray = new String[bibcodes.size()]; + return performSearchByIds(bibcodes.toArray(bibcodeArray)); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } } @Override - public Optional performSearchById(String identifier) throws FetcherException { + public List performSearch(String query) throws FetcherException { - Optional> results = performSearchByIds(identifier); - if (results.isEmpty()) { - return Optional.empty(); + if (StringUtil.isBlank(query)) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForQuery(query)); + String[] bibcodeArray = new String[bibcodes.size()]; + return performSearchByIds(bibcodes.toArray(bibcodeArray)); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + private List fetchBibcodes(URL url) throws FetcherException { + + try { + URLDownload download = new URLDownload(url); + download.addHeader("Authorization", "Bearer " + API_KEY); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + try { + JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); + List bibcodes = new ArrayList<>(); + for (int i = 0; i < codes.length(); i++) { + bibcodes.add(codes.getJSONObject(i).getString("bibcode")); + } + return bibcodes; + } catch (JSONException e) { + return Collections.emptyList(); + } + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); } + } - List fetchedEntries = results.get(); + @Override + public Optional performSearchById(String identifier) throws FetcherException { + List fetchedEntries = performSearchByIds(identifier); + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } if (fetchedEntries.size() > 1) { LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier @@ -136,33 +191,33 @@ public Optional performSearchById(String identifier) throws FetcherExc return Optional.of(entry); } - private Optional> performSearchByIds(String... identifiers) throws FetcherException { + private List performSearchByIds(String... identifiers) throws FetcherException { long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); if (idCount == 0) { - return Optional.empty(); + return Collections.emptyList(); } - try { - String postData = buildPostData(identifiers); URLDownload download = new URLDownload(getURLForID("")); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); download.setPostData(postData); - download.asString(); String content = download.asString(); JSONObject obj = new JSONObject(content); - List fetchedEntries = getParser().parseEntries(obj.getString("export")); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); + try { + List fetchedEntries = getParser().parseEntries(obj.optString("export")); + if (fetchedEntries.isEmpty()) { + return Collections.emptyList(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return fetchedEntries; + } catch (JSONException e) { + return Collections.emptyList(); } - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - - return Optional.of(fetchedEntries); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -172,9 +227,5 @@ private Optional> performSearchByIds(String... identifiers) throw } } - @Override - public List performSearch(String query) throws FetcherException { - return Collections.emptyList(); - } } From 7279e1b24777777c3c982746bb7c3195ea91e82b Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 20:36:04 +0200 Subject: [PATCH 04/34] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- .../logic/importer/fetcher/NewAstrophysicsDataSystem.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 0ca438eff28..824cf89c6e2 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -226,6 +226,4 @@ private List performSearchByIds(String... identifiers) throws FetcherE throw new FetcherException("An internal parser error occurred", e); } } - - } From 329d1dd533c70a49f6eea07c1d49527f627318ec Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 21:10:29 +0200 Subject: [PATCH 05/34] added cleanup --- .../fetcher/NewAstrophysicsDataSystem.java | 19 +- .../NewAstrophysicsDataSystemTest.java | 206 ++++++++++++++++++ 2 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 824cf89c6e2..9481aace5bc 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -11,6 +11,11 @@ import java.util.Objects; import java.util.Optional; +import org.jabref.logic.cleanup.MoveFieldCleanup; +import org.jabref.logic.formatter.bibtexfields.ClearFormatter; +import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; +import org.jabref.logic.formatter.bibtexfields.RemoveBracesFormatter; +import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.EntryBasedParserFetcher; import org.jabref.logic.importer.FetcherException; @@ -21,8 +26,10 @@ import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; +import org.jabref.model.cleanup.FieldFormatterCleanup; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; @@ -113,7 +120,17 @@ public Parser getParser() { @Override public void doPostCleanup(BibEntry entry) { - + new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveNewlinesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); + + // Remove ADS note + new FieldFormatterCleanup(new UnknownField("adsnote"), new ClearFormatter()).cleanup(entry); + // Move adsurl to url field + new MoveFieldCleanup(new UnknownField("adsurl"), StandardField.URL).cleanup(entry); + // The fetcher adds some garbage (number of found entries etc before) + entry.setCommentsBeforeEntry(""); } @Override diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java new file mode 100644 index 00000000000..649a39a9054 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -0,0 +1,206 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.jabref.logic.bibtex.FieldContentParserPreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.testutils.category.FetcherTest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@FetcherTest +public class NewAstrophysicsDataSystemTest { + + private NewAstrophysicsDataSystem fetcher; + private BibEntry diezSliceTheoremEntry, famaeyMcGaughEntry, sunWelchEntry, xiongSunEntry, ingersollPollardEntry, luceyPaulEntry; + + @BeforeEach + public void setUp() throws Exception { + ImportFormatPreferences importFormatPreferences = mock(ImportFormatPreferences.class); + when(importFormatPreferences.getFieldContentParserPreferences()).thenReturn( + mock(FieldContentParserPreferences.class)); + fetcher = new NewAstrophysicsDataSystem(importFormatPreferences); + + diezSliceTheoremEntry = new BibEntry(); + diezSliceTheoremEntry.setType(StandardEntryType.Article); + diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); + diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); + diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.ABSTRACT, + "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " + + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " + + "fundamental tool to generalize the result of Palais to this " + + "infinite-dimensional setting. The presented slice theorem is illustrated " + + "by its application to gauge theories: the action of the gauge " + + "transformation group admits smooth slices at every point and thus the " + + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " + + "covariant and symplectic formulation of classical field theory is " + + "proposed and extensively discussed. At the root of this novel framework " + + "is the incorporation of field degrees of freedom F and spacetime M into " + + "the product manifold F * M. The induced bigrading of differential forms " + + "is used in order to carry over the usual symplectic theory to this new " + + "setting. The examples of the Klein-Gordon field and general Yang-Mills " + + "theory illustrate that the presented approach conveniently handles the " + + "occurring symmetries."); + + famaeyMcGaughEntry = new BibEntry(); + famaeyMcGaughEntry.setType(StandardEntryType.Article); + famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); + famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); + famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); + famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); + famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); + famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); + famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + + sunWelchEntry = new BibEntry(); + sunWelchEntry.setType(StandardEntryType.Article); + sunWelchEntry.setCiteKey("2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); + sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); + sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.PAGES, "44-48"); + sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); + sunWelchEntry.setField(StandardField.VOLUME, "11"); + sunWelchEntry.setField(StandardField.YEAR, "2012"); + sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + + xiongSunEntry = new BibEntry(); + xiongSunEntry.setType(StandardEntryType.Article); + xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); + xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); + xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.PAGES, "879-889"); + xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); + xiongSunEntry.setField(StandardField.VOLUME, "45"); + xiongSunEntry.setField(StandardField.YEAR, "2007"); + xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + + ingersollPollardEntry = new BibEntry(); + ingersollPollardEntry.setType(StandardEntryType.Article); + ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); + ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); + ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); + ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.VOLUME, "52"); + ingersollPollardEntry.setField(StandardField.YEAR, "1982"); + ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + + luceyPaulEntry = new BibEntry(); + luceyPaulEntry.setType(StandardEntryType.Article); + luceyPaulEntry.setCiteKey("2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); + luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); + luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); + luceyPaulEntry.setField(StandardField.PAGES, "20297-20306"); + luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); + luceyPaulEntry.setField(StandardField.VOLUME, "105"); + luceyPaulEntry.setField(StandardField.YEAR, "2000"); + luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + } + + @Test + public void testHelpPage() { + assertEquals("ADS", fetcher.getHelpPage().get().getPageName()); + } + + @Test + public void testGetName() { + assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); + } + + @Test + public void searchByQueryFindsEntry() throws Exception { + List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); + assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + } + + @Test + public void searchByEntryFindsEntry() throws Exception { + BibEntry searchEntry = new BibEntry(); + searchEntry.setField(StandardField.TITLE, "slice theorem"); + searchEntry.setField(StandardField.AUTHOR, "Diez"); + + List fetchedEntries = fetcher.performSearch(searchEntry); + assertFalse(fetchedEntries.isEmpty()); + assertEquals(diezSliceTheoremEntry, fetchedEntries.get(0)); + } + + @Test + public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); + fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright + assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByIdEmptyDOI() throws Exception { + Optional fetchedEntry = fetcher.performSearchById(""); + assertEquals(Optional.empty(), fetchedEntry); + } + + @Test + public void testPerformSearchByIdInvalidDoi() throws Exception { + assertEquals(Optional.empty(), fetcher.performSearchById("this.doi.will.fail")); + } + + @Test + public void testPerformSearchBySunWelchEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1038/nmat3160"); + fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT)); //Remove abstract due to copyright + assertEquals(Optional.of(sunWelchEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByXiongSunEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1109/TGRS.2006.890567"); + assertEquals(Optional.of(xiongSunEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByIngersollPollardEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1016/0019-1035(82)90169-5"); + assertEquals(Optional.of(ingersollPollardEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByLuceyPaulEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); + assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); + } +} From 605519db040dae259ca9d563a9830ca37dc025b3 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 21:45:32 +0200 Subject: [PATCH 06/34] support doi and ads bibcodes --- .../fetcher/NewAstrophysicsDataSystem.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 9481aace5bc..5183a3d8e67 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -67,6 +67,10 @@ private String buildPostData(String... bibcodes) { return obj.toString(); } + private URL getURLforExport() throws URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + @Override public String getName() { return "New SAO/NASA Astrophysics Data System"; @@ -89,7 +93,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR if (title.isPresent()) { stringBuilder.append(title.get()) - .append(author.map(s -> " OR " + s) + .append(author.map(s -> " AND " + s) .orElse("")); } else { stringBuilder.append(author.orElse("")); @@ -105,7 +109,11 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - return new URIBuilder(API_EXPORT_URL).build().toURL(); + String query = "doi:" + identifier + " OR " + "bibcode:" + identifier; + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); } @Override @@ -194,18 +202,29 @@ private List fetchBibcodes(URL url) throws FetcherException { @Override public Optional performSearchById(String identifier) throws FetcherException { - List fetchedEntries = performSearchByIds(identifier); - if (fetchedEntries.isEmpty()) { + if (StringUtil.isBlank(identifier)) { return Optional.empty(); } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } + try { + List bibcodes = fetchBibcodes(getURLForID(identifier)); + String[] bibcodeArray = new String[bibcodes.size()]; + List fetchedEntries = performSearchByIds(bibcodes.toArray(bibcodeArray)); - BibEntry entry = fetchedEntries.get(0); - return Optional.of(entry); + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } } private List performSearchByIds(String... identifiers) throws FetcherException { @@ -216,7 +235,7 @@ private List performSearchByIds(String... identifiers) throws FetcherE } try { String postData = buildPostData(identifiers); - URLDownload download = new URLDownload(getURLForID("")); + URLDownload download = new URLDownload(getURLforExport()); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); download.setPostData(postData); From cba78290dee278deee0bdb6fced9fa570dd291f2 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:07:22 +0200 Subject: [PATCH 07/34] fixes wrong search syntax --- .../importer/fetcher/NewAstrophysicsDataSystem.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 5183a3d8e67..510e213ebe1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -88,8 +88,8 @@ public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLE public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { StringBuilder stringBuilder = new StringBuilder(); - Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); - Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author:" + a); + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); + Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); if (title.isPresent()) { stringBuilder.append(title.get()) @@ -109,7 +109,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - String query = "doi:" + identifier + " OR " + "bibcode:" + identifier; + String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); builder.addParameter("fl", "bibcode"); @@ -151,7 +151,8 @@ public List performSearch(BibEntry entry) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForEntry(entry)); String[] bibcodeArray = new String[bibcodes.size()]; - return performSearchByIds(bibcodes.toArray(bibcodeArray)); + bibcodes.toArray(bibcodeArray); + return performSearchByIds(bibcodeArray); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { From bd6d84551450807762f11b13484e24a1f10de283 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:08:17 +0200 Subject: [PATCH 08/34] fixed test cases to API response --- .../NewAstrophysicsDataSystemTest.java | 80 +++++++++---------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java index 649a39a9054..561d316a0d0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -35,95 +35,89 @@ public void setUp() throws Exception { diezSliceTheoremEntry = new BibEntry(); diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); - diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); - diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); + diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(new UnknownField("eid"), "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); + diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " - + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " - + "fundamental tool to generalize the result of Palais to this " - + "infinite-dimensional setting. The presented slice theorem is illustrated " - + "by its application to gauge theories: the action of the gauge " - + "transformation group admits smooth slices at every point and thus the " - + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " - + "covariant and symplectic formulation of classical field theory is " - + "proposed and extensively discussed. At the root of this novel framework " - + "is the incorporation of field degrees of freedom F and spacetime M into " - + "the product manifold F * M. The induced bigrading of differential forms " - + "is used in order to carry over the usual symplectic theory to this new " - + "setting. The examples of the Klein-Gordon field and general Yang-Mills " - + "theory illustrate that the presented approach conveniently handles the " - + "occurring symmetries."); + "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); famaeyMcGaughEntry = new BibEntry(); famaeyMcGaughEntry.setType(StandardEntryType.Article); famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); + famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); + famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); - famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); + famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); sunWelchEntry = new BibEntry(); sunWelchEntry.setType(StandardEntryType.Article); sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.MONTH, "Jan"); + sunWelchEntry.setField(StandardField.NUMBER, "1"); sunWelchEntry.setField(StandardField.PAGES, "44-48"); sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); sunWelchEntry.setField(StandardField.VOLUME, "11"); sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); xiongSunEntry = new BibEntry(); xiongSunEntry.setType(StandardEntryType.Article); xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.MONTH, "Apr"); + xiongSunEntry.setField(StandardField.NUMBER, "4"); xiongSunEntry.setField(StandardField.PAGES, "879-889"); xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); xiongSunEntry.setField(StandardField.VOLUME, "45"); xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); ingersollPollardEntry = new BibEntry(); ingersollPollardEntry.setType(StandardEntryType.Article); ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); - ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); + ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); + ingersollPollardEntry.setField(StandardField.NUMBER, "1"); ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); ingersollPollardEntry.setField(StandardField.VOLUME, "52"); ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); luceyPaulEntry = new BibEntry(); luceyPaulEntry.setType(StandardEntryType.Article); luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); @@ -131,7 +125,9 @@ public void setUp() throws Exception { luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); luceyPaulEntry.setField(StandardField.VOLUME, "105"); luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.MONTH, "Jan"); + luceyPaulEntry.setField(StandardField.NUMBER, "E8"); } @Test @@ -165,7 +161,7 @@ public void searchByEntryFindsEntry() throws Exception { public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); + assertEquals(famaeyMcGaughEntry, fetchedEntry.get()); } @Test From 27243ed1b82f4e31f91e898ad8537c56b2a1d832 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:28:18 +0200 Subject: [PATCH 09/34] fixed bug where equals doesnt work with unknown field --- .../fetcher/NewAstrophysicsDataSystemTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java index 561d316a0d0..403bfaefa55 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -8,7 +7,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; @@ -17,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,7 +44,7 @@ public void setUp() throws Exception { diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(new UnknownField("eid"), "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, @@ -64,7 +63,7 @@ public void setUp() throws Exception { famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); - famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.EID, "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); @@ -143,7 +142,7 @@ public void testGetName() { @Test public void searchByQueryFindsEntry() throws Exception { List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); } @Test @@ -161,7 +160,7 @@ public void searchByEntryFindsEntry() throws Exception { public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(famaeyMcGaughEntry, fetchedEntry.get()); + assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); } @Test From 8910676fcc5f784d4760be10847f7a0f6ed7f5df Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:44:14 +0200 Subject: [PATCH 10/34] changed array method parameters to Collections --- .../fetcher/NewAstrophysicsDataSystem.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 510e213ebe1..37dedfb6ca9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -5,11 +5,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import org.jabref.logic.cleanup.MoveFieldCleanup; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; @@ -61,7 +62,7 @@ public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } - private String buildPostData(String... bibcodes) { + private String buildPostData(Collection bibcodes) { JSONObject obj = new JSONObject(); obj.put("bibcode", bibcodes); return obj.toString(); @@ -150,9 +151,7 @@ public List performSearch(BibEntry entry) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForEntry(entry)); - String[] bibcodeArray = new String[bibcodes.size()]; - bibcodes.toArray(bibcodeArray); - return performSearchByIds(bibcodeArray); + return performSearchByIds(bibcodes); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -169,8 +168,7 @@ public List performSearch(String query) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForQuery(query)); - String[] bibcodeArray = new String[bibcodes.size()]; - return performSearchByIds(bibcodes.toArray(bibcodeArray)); + return performSearchByIds(bibcodes); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -209,8 +207,7 @@ public Optional performSearchById(String identifier) throws FetcherExc try { List bibcodes = fetchBibcodes(getURLForID(identifier)); - String[] bibcodeArray = new String[bibcodes.size()]; - List fetchedEntries = performSearchByIds(bibcodes.toArray(bibcodeArray)); + List fetchedEntries = performSearchByIds(bibcodes); if (fetchedEntries.isEmpty()) { return Optional.empty(); @@ -228,14 +225,14 @@ public Optional performSearchById(String identifier) throws FetcherExc } } - private List performSearchByIds(String... identifiers) throws FetcherException { + private List performSearchByIds(Collection identifiers) throws FetcherException { - long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); - if (idCount == 0) { + List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); + if (ids.isEmpty()) { return Collections.emptyList(); } try { - String postData = buildPostData(identifiers); + String postData = buildPostData(ids); URLDownload download = new URLDownload(getURLforExport()); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); From b811c0bd7d8ff807bc261d199ba200e9da16b612 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:45:02 +0200 Subject: [PATCH 11/34] removes json module dependency --- src/main/java/module-info.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index bfb3a41fdcc..791e2e50698 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -65,5 +65,4 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; - requires json; } From 638dd947ca28e68f051c0e1dd45879579c49ed9b Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:52:28 +0200 Subject: [PATCH 12/34] replace old fetcher with the new one --- .../jabref/logic/importer/WebFetchers.java | 4 - .../fetcher/AstrophysicsDataSystem.java | 219 ++++++++++++--- .../fetcher/NewAstrophysicsDataSystem.java | 263 ------------------ .../fetcher/AstrophysicsDataSystemTest.java | 89 +++--- .../NewAstrophysicsDataSystemTest.java | 201 ------------- 5 files changed, 215 insertions(+), 561 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java delete mode 100644 src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index 4161786d0c4..66319f5e6b4 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -27,7 +27,6 @@ import org.jabref.logic.importer.fetcher.LibraryOfCongress; import org.jabref.logic.importer.fetcher.MathSciNet; import org.jabref.logic.importer.fetcher.MedlineFetcher; -import org.jabref.logic.importer.fetcher.NewAstrophysicsDataSystem; import org.jabref.logic.importer.fetcher.OpenAccessDoi; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; @@ -89,7 +88,6 @@ public static SortedSet getSearchBasedFetchers(ImportFormatP set.add(new GvkFetcher()); set.add(new MedlineFetcher()); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); set.add(new ZbMATH(importFormatPreferences)); set.add(new ACMPortalFetcher(importFormatPreferences)); @@ -110,7 +108,6 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new ArXiv(importFormatPreferences)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new DiVA(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); @@ -130,7 +127,6 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) { SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 2ac10652bc9..faa99419c0c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -1,10 +1,16 @@ package org.jabref.logic.importer.fetcher; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import org.jabref.logic.cleanup.MoveFieldCleanup; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; @@ -16,16 +22,22 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.logic.net.URLDownload; import org.jabref.model.cleanup.FieldFormatterCleanup; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; /** * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) @@ -37,73 +49,69 @@ */ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - private static String API_QUERY_URL = "http://adsabs.harvard.edu/cgi-bin/nph-basic_connect"; - private static String API_ENTRY_URL = "http://adsabs.harvard.edu/cgi-bin/nph-abs_connect"; - private static String API_DOI_URL = "http://adsabs.harvard.edu/doi/"; + private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; - private final String patternRemoveDOI = "^(doi:|DOI:)"; + private static final String API_KEY = ""; //TODO Add API Token private final ImportFormatPreferences preferences; public AstrophysicsDataSystem(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } + private String buildPostData(Collection bibcodes) { + JSONObject obj = new JSONObject(); + obj.put("bibcode", bibcodes); + return obj.toString(); + } + + private URL getURLforExport() throws URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + @Override public String getName() { return "SAO/NASA Astrophysics Data System"; } - private URIBuilder getBaseUrl(String apiUrl) throws URISyntaxException { - URIBuilder uriBuilder = new URIBuilder(apiUrl); - uriBuilder.addParameter("data_type", "BIBTEXPLUS"); - uriBuilder.addParameter("start_nr", String.valueOf(1)); - uriBuilder.addParameter("nr_to_return", String.valueOf(200)); - return uriBuilder; - } - @Override public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = getBaseUrl(API_QUERY_URL); - uriBuilder.addParameter("qsearch", query); - return uriBuilder.build().toURL(); + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); } @Override public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = getBaseUrl(API_ENTRY_URL); - - // Search astronomy + physics + arXiv db - uriBuilder.addParameter("db_key", "AST"); - uriBuilder.addParameter("db_key", "PHY"); - uriBuilder.addParameter("db_key", "PRE"); - - // Add title search - entry.getFieldOrAlias(StandardField.TITLE).ifPresent(title -> { - uriBuilder.addParameter("ttl_logic", "OR"); - uriBuilder.addParameter("title", title); - uriBuilder.addParameter("ttl_syn", "YES"); // Synonym replacement - uriBuilder.addParameter("ttl_wt", "0.3"); // Weight - uriBuilder.addParameter("ttl_wgt", "YES"); // Consider Weight - }); - - // Add author search - entry.getFieldOrAlias(StandardField.AUTHOR).ifPresent(author -> { - uriBuilder.addParameter("aut_logic", "OR"); - uriBuilder.addParameter("author", author); - uriBuilder.addParameter("aut_syn", "YES"); // Synonym replacement - uriBuilder.addParameter("aut_wt", "1.0"); // Weight - uriBuilder.addParameter("aut_wgt", "YES"); // Consider weight - }); - - return uriBuilder.build().toURL(); + StringBuilder stringBuilder = new StringBuilder(); + + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); + Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); + + if (title.isPresent()) { + stringBuilder.append(title.get()) + .append(author.map(s -> " AND " + s) + .orElse("")); + } else { + stringBuilder.append(author.orElse("")); + } + String query = stringBuilder.toString().trim(); + + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + builder.addParameter("rows", "20"); + return builder.build().toURL(); } @Override - public URL getURLForID(String identifier) throws URISyntaxException, MalformedURLException, FetcherException { - String key = identifier.replaceAll(patternRemoveDOI, ""); - URIBuilder uriBuilder = new URIBuilder(API_DOI_URL + key); - uriBuilder.addParameter("data_type", "BIBTEXPLUS"); - return uriBuilder.build().toURL(); + public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { + String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); } @Override @@ -130,4 +138,123 @@ public void doPostCleanup(BibEntry entry) { // The fetcher adds some garbage (number of found entries etc before) entry.setCommentsBeforeEntry(""); } + + @Override + public List performSearch(BibEntry entry) throws FetcherException { + + if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForEntry(entry)); + return performSearchByIds(bibcodes); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + @Override + public List performSearch(String query) throws FetcherException { + + if (StringUtil.isBlank(query)) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForQuery(query)); + return performSearchByIds(bibcodes); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + private List fetchBibcodes(URL url) throws FetcherException { + + try { + URLDownload download = new URLDownload(url); + download.addHeader("Authorization", "Bearer " + API_KEY); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + try { + JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); + List bibcodes = new ArrayList<>(); + for (int i = 0; i < codes.length(); i++) { + bibcodes.add(codes.getJSONObject(i).getString("bibcode")); + } + return bibcodes; + } catch (JSONException e) { + return Collections.emptyList(); + } + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + @Override + public Optional performSearchById(String identifier) throws FetcherException { + if (StringUtil.isBlank(identifier)) { + return Optional.empty(); + } + + try { + List bibcodes = fetchBibcodes(getURLForID(identifier)); + List fetchedEntries = performSearchByIds(bibcodes); + + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + private List performSearchByIds(Collection identifiers) throws FetcherException { + + List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); + if (ids.isEmpty()) { + return Collections.emptyList(); + } + try { + String postData = buildPostData(ids); + URLDownload download = new URLDownload(getURLforExport()); + download.addHeader("Authorization", "Bearer " + API_KEY); + download.addHeader("ContentType", "application/json"); + download.setPostData(postData); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + try { + List fetchedEntries = getParser().parseEntries(obj.optString("export")); + if (fetchedEntries.isEmpty()) { + return Collections.emptyList(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return fetchedEntries; + } catch (JSONException e) { + return Collections.emptyList(); + } + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } catch (ParseException e) { + throw new FetcherException("An internal parser error occurred", e); + } + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java deleted file mode 100644 index 37dedfb6ca9..00000000000 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ /dev/null @@ -1,263 +0,0 @@ -package org.jabref.logic.importer.fetcher; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.jabref.logic.cleanup.MoveFieldCleanup; -import org.jabref.logic.formatter.bibtexfields.ClearFormatter; -import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; -import org.jabref.logic.formatter.bibtexfields.RemoveBracesFormatter; -import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; -import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.EntryBasedParserFetcher; -import org.jabref.logic.importer.FetcherException; -import org.jabref.logic.importer.IdBasedParserFetcher; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParseException; -import org.jabref.logic.importer.Parser; -import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.logic.net.URLDownload; -import org.jabref.model.cleanup.FieldFormatterCleanup; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.strings.StringUtil; -import org.jabref.model.util.DummyFileUpdateMonitor; - -import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -//TODO Replace Old ADS after the new one is mature - -/** - * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) - *

- * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html - *

- * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple - * calls to get BibTeX, status: September 2016) - */ -public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - - private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; - private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; - - private static String API_KEY = ""; //TODO Add API Token - - private final ImportFormatPreferences preferences; - - public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { - this.preferences = Objects.requireNonNull(preferences); - } - - private String buildPostData(Collection bibcodes) { - JSONObject obj = new JSONObject(); - obj.put("bibcode", bibcodes); - return obj.toString(); - } - - private URL getURLforExport() throws URISyntaxException, MalformedURLException { - return new URIBuilder(API_EXPORT_URL).build().toURL(); - } - - @Override - public String getName() { - return "New SAO/NASA Astrophysics Data System"; - } - - @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - return builder.build().toURL(); - } - - @Override - public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { - StringBuilder stringBuilder = new StringBuilder(); - - Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); - Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); - - if (title.isPresent()) { - stringBuilder.append(title.get()) - .append(author.map(s -> " AND " + s) - .orElse("")); - } else { - stringBuilder.append(author.orElse("")); - } - String query = stringBuilder.toString().trim(); - - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - builder.addParameter("rows", "20"); - return builder.build().toURL(); - } - - @Override - public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - return builder.build().toURL(); - } - - @Override - public Optional getHelpPage() { - return Optional.of(HelpFile.FETCHER_ADS); - } - - @Override - public Parser getParser() { - return new BibtexParser(preferences, new DummyFileUpdateMonitor()); - } - - @Override - public void doPostCleanup(BibEntry entry) { - new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveNewlinesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); - - // Remove ADS note - new FieldFormatterCleanup(new UnknownField("adsnote"), new ClearFormatter()).cleanup(entry); - // Move adsurl to url field - new MoveFieldCleanup(new UnknownField("adsurl"), StandardField.URL).cleanup(entry); - // The fetcher adds some garbage (number of found entries etc before) - entry.setCommentsBeforeEntry(""); - } - - @Override - public List performSearch(BibEntry entry) throws FetcherException { - - if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { - return Collections.emptyList(); - } - - try { - List bibcodes = fetchBibcodes(getURLForEntry(entry)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - @Override - public List performSearch(String query) throws FetcherException { - - if (StringUtil.isBlank(query)) { - return Collections.emptyList(); - } - - try { - List bibcodes = fetchBibcodes(getURLForQuery(query)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - private List fetchBibcodes(URL url) throws FetcherException { - - try { - URLDownload download = new URLDownload(url); - download.addHeader("Authorization", "Bearer " + API_KEY); - String content = download.asString(); - JSONObject obj = new JSONObject(content); - - try { - JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); - List bibcodes = new ArrayList<>(); - for (int i = 0; i < codes.length(); i++) { - bibcodes.add(codes.getJSONObject(i).getString("bibcode")); - } - return bibcodes; - } catch (JSONException e) { - return Collections.emptyList(); - } - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - @Override - public Optional performSearchById(String identifier) throws FetcherException { - if (StringUtil.isBlank(identifier)) { - return Optional.empty(); - } - - try { - List bibcodes = fetchBibcodes(getURLForID(identifier)); - List fetchedEntries = performSearchByIds(bibcodes); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); - } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } - BibEntry entry = fetchedEntries.get(0); - return Optional.of(entry); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - private List performSearchByIds(Collection identifiers) throws FetcherException { - - List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); - if (ids.isEmpty()) { - return Collections.emptyList(); - } - try { - String postData = buildPostData(ids); - URLDownload download = new URLDownload(getURLforExport()); - download.addHeader("Authorization", "Bearer " + API_KEY); - download.addHeader("ContentType", "application/json"); - download.setPostData(postData); - String content = download.asString(); - JSONObject obj = new JSONObject(content); - - try { - List fetchedEntries = getParser().parseEntries(obj.optString("export")); - if (fetchedEntries.isEmpty()) { - return Collections.emptyList(); - } - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - - return fetchedEntries; - } catch (JSONException e) { - return Collections.emptyList(); - } - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); - } - } -} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java index 4ece520646d..f7f393c49dd 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -8,7 +7,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; @@ -17,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -35,95 +34,89 @@ public void setUp() throws Exception { diezSliceTheoremEntry = new BibEntry(); diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); - diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); - diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); + diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); + diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " - + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " - + "fundamental tool to generalize the result of Palais to this " - + "infinite-dimensional setting. The presented slice theorem is illustrated " - + "by its application to gauge theories: the action of the gauge " - + "transformation group admits smooth slices at every point and thus the " - + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " - + "covariant and symplectic formulation of classical field theory is " - + "proposed and extensively discussed. At the root of this novel framework " - + "is the incorporation of field degrees of freedom F and spacetime M into " - + "the product manifold F * M. The induced bigrading of differential forms " - + "is used in order to carry over the usual symplectic theory to this new " - + "setting. The examples of the Klein-Gordon field and general Yang-Mills " - + "theory illustrate that the presented approach conveniently handles the " - + "occurring symmetries."); + "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); famaeyMcGaughEntry = new BibEntry(); famaeyMcGaughEntry.setType(StandardEntryType.Article); famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); + famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); - famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); + famaeyMcGaughEntry.setField(StandardField.EID, "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); - famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); + famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); sunWelchEntry = new BibEntry(); sunWelchEntry.setType(StandardEntryType.Article); sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.MONTH, "Jan"); + sunWelchEntry.setField(StandardField.NUMBER, "1"); sunWelchEntry.setField(StandardField.PAGES, "44-48"); sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); sunWelchEntry.setField(StandardField.VOLUME, "11"); sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); xiongSunEntry = new BibEntry(); xiongSunEntry.setType(StandardEntryType.Article); xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.MONTH, "Apr"); + xiongSunEntry.setField(StandardField.NUMBER, "4"); xiongSunEntry.setField(StandardField.PAGES, "879-889"); xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); xiongSunEntry.setField(StandardField.VOLUME, "45"); xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); ingersollPollardEntry = new BibEntry(); ingersollPollardEntry.setType(StandardEntryType.Article); ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); - ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); + ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); + ingersollPollardEntry.setField(StandardField.NUMBER, "1"); ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); ingersollPollardEntry.setField(StandardField.VOLUME, "52"); ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); luceyPaulEntry = new BibEntry(); luceyPaulEntry.setType(StandardEntryType.Article); luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); @@ -131,7 +124,9 @@ public void setUp() throws Exception { luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); luceyPaulEntry.setField(StandardField.VOLUME, "105"); luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.MONTH, "Jan"); + luceyPaulEntry.setField(StandardField.NUMBER, "E8"); } @Test @@ -141,13 +136,13 @@ public void testHelpPage() { @Test public void testGetName() { - assertEquals("SAO/NASA Astrophysics Data System", fetcher.getName()); + assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); } @Test public void searchByQueryFindsEntry() throws Exception { List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); } @Test @@ -200,7 +195,7 @@ public void testPerformSearchByIngersollPollardEntry() throws Exception { @Test public void testPerformSearchByLuceyPaulEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1029/1999JE001117"); + Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java deleted file mode 100644 index 403bfaefa55..00000000000 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.jabref.logic.importer.fetcher; - -import java.util.List; -import java.util.Optional; - -import org.jabref.logic.bibtex.FieldContentParserPreferences; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.testutils.category.FetcherTest; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@FetcherTest -public class NewAstrophysicsDataSystemTest { - - private NewAstrophysicsDataSystem fetcher; - private BibEntry diezSliceTheoremEntry, famaeyMcGaughEntry, sunWelchEntry, xiongSunEntry, ingersollPollardEntry, luceyPaulEntry; - - @BeforeEach - public void setUp() throws Exception { - ImportFormatPreferences importFormatPreferences = mock(ImportFormatPreferences.class); - when(importFormatPreferences.getFieldContentParserPreferences()).thenReturn( - mock(FieldContentParserPreferences.class)); - fetcher = new NewAstrophysicsDataSystem(importFormatPreferences); - - diezSliceTheoremEntry = new BibEntry(); - diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); - diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); - diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); - diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); - diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); - - famaeyMcGaughEntry = new BibEntry(); - famaeyMcGaughEntry.setType(StandardEntryType.Article); - famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); - famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); - famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); - famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); - famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); - famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); - famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); - famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); - famaeyMcGaughEntry.setField(StandardField.EID, "10"); - famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); - famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); - famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); - - sunWelchEntry = new BibEntry(); - sunWelchEntry.setType(StandardEntryType.Article); - sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); - sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); - sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "Jan"); - sunWelchEntry.setField(StandardField.NUMBER, "1"); - sunWelchEntry.setField(StandardField.PAGES, "44-48"); - sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); - sunWelchEntry.setField(StandardField.VOLUME, "11"); - sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); - - xiongSunEntry = new BibEntry(); - xiongSunEntry.setType(StandardEntryType.Article); - xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); - xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); - xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "Apr"); - xiongSunEntry.setField(StandardField.NUMBER, "4"); - xiongSunEntry.setField(StandardField.PAGES, "879-889"); - xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); - xiongSunEntry.setField(StandardField.VOLUME, "45"); - xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); - - ingersollPollardEntry = new BibEntry(); - ingersollPollardEntry.setType(StandardEntryType.Article); - ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); - ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); - ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); - ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); - ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); - ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); - ingersollPollardEntry.setField(StandardField.NUMBER, "1"); - ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); - ingersollPollardEntry.setField(StandardField.VOLUME, "52"); - ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); - - luceyPaulEntry = new BibEntry(); - luceyPaulEntry.setType(StandardEntryType.Article); - luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); - luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); - luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); - luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); - luceyPaulEntry.setField(StandardField.PAGES, "20297-20306"); - luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); - luceyPaulEntry.setField(StandardField.VOLUME, "105"); - luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.MONTH, "Jan"); - luceyPaulEntry.setField(StandardField.NUMBER, "E8"); - } - - @Test - public void testHelpPage() { - assertEquals("ADS", fetcher.getHelpPage().get().getPageName()); - } - - @Test - public void testGetName() { - assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); - } - - @Test - public void searchByQueryFindsEntry() throws Exception { - List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); - } - - @Test - public void searchByEntryFindsEntry() throws Exception { - BibEntry searchEntry = new BibEntry(); - searchEntry.setField(StandardField.TITLE, "slice theorem"); - searchEntry.setField(StandardField.AUTHOR, "Diez"); - - List fetchedEntries = fetcher.performSearch(searchEntry); - assertFalse(fetchedEntries.isEmpty()); - assertEquals(diezSliceTheoremEntry, fetchedEntries.get(0)); - } - - @Test - public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); - fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByIdEmptyDOI() throws Exception { - Optional fetchedEntry = fetcher.performSearchById(""); - assertEquals(Optional.empty(), fetchedEntry); - } - - @Test - public void testPerformSearchByIdInvalidDoi() throws Exception { - assertEquals(Optional.empty(), fetcher.performSearchById("this.doi.will.fail")); - } - - @Test - public void testPerformSearchBySunWelchEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1038/nmat3160"); - fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT)); //Remove abstract due to copyright - assertEquals(Optional.of(sunWelchEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByXiongSunEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1109/TGRS.2006.890567"); - assertEquals(Optional.of(xiongSunEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByIngersollPollardEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1016/0019-1035(82)90169-5"); - assertEquals(Optional.of(ingersollPollardEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByLuceyPaulEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); - assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); - } -} From c9f46df5ec648ac82c96b2bb42c2420a40d3dc7f Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:57:47 +0200 Subject: [PATCH 13/34] replace old fetcher with the new one --- .../logic/importer/fetcher/AstrophysicsDataSystemTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java index f7f393c49dd..f3141019524 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java @@ -136,7 +136,7 @@ public void testHelpPage() { @Test public void testGetName() { - assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); + assertEquals("SAO/NASA Astrophysics Data System", fetcher.getName()); } @Test From 9319bb139f354f1522f59c8b360b1ff3d00a1343 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 19:15:32 +0200 Subject: [PATCH 14/34] added Javadoc --- .../fetcher/AstrophysicsDataSystem.java | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index faa99419c0c..f7d8c7f9c82 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -40,12 +40,7 @@ import org.json.JSONObject; /** - * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) - *

- * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html - *

- * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple - * calls to get BibTeX, status: September 2016) + * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/) */ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { @@ -59,13 +54,19 @@ public AstrophysicsDataSystem(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } - private String buildPostData(Collection bibcodes) { + /** + * @param bibcodes collection of bibcodes for which a JSON object should be created + */ + private static String buildPostData(Collection bibcodes) { JSONObject obj = new JSONObject(); obj.put("bibcode", bibcodes); return obj.toString(); } - private URL getURLforExport() throws URISyntaxException, MalformedURLException { + /** + * @return export URL endpoint + */ + private static URL getURLforExport() throws URISyntaxException, MalformedURLException { return new URIBuilder(API_EXPORT_URL).build().toURL(); } @@ -74,6 +75,10 @@ public String getName() { return "SAO/NASA Astrophysics Data System"; } + /** + * @param query query string, matching the apache solr format + * @return URL which points to a search request for given query + */ @Override public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); @@ -82,6 +87,10 @@ public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLE return builder.build().toURL(); } + /** + * @param entry BibEntry for which a search URL is created + * @return URL which points to a search request for given entry + */ @Override public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { StringBuilder stringBuilder = new StringBuilder(); @@ -105,6 +114,10 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR return builder.build().toURL(); } + /** + * @param identifier bibcode or doi for which a search URL is created + * @return URL which points to a search URL for given identifier + */ @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; @@ -173,6 +186,10 @@ public List performSearch(String query) throws FetcherException { } } + /** + * @param url search ul for which bibcode will be returned + * @return list of bibcodes matching the search request. May be empty + */ private List fetchBibcodes(URL url) throws FetcherException { try { @@ -180,19 +197,16 @@ private List fetchBibcodes(URL url) throws FetcherException { download.addHeader("Authorization", "Bearer " + API_KEY); String content = download.asString(); JSONObject obj = new JSONObject(content); - - try { - JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); - List bibcodes = new ArrayList<>(); - for (int i = 0; i < codes.length(); i++) { - bibcodes.add(codes.getJSONObject(i).getString("bibcode")); - } - return bibcodes; - } catch (JSONException e) { - return Collections.emptyList(); + JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); + List bibcodes = new ArrayList<>(); + for (int i = 0; i < codes.length(); i++) { + bibcodes.add(codes.getJSONObject(i).getString("bibcode")); } + return bibcodes; } catch (IOException e) { throw new FetcherException("A network error occurred", e); + } catch (JSONException e) { + return Collections.emptyList(); } } @@ -222,6 +236,11 @@ public Optional performSearchById(String identifier) throws FetcherExc } } + /** + * @param identifiers bibcodes for which bibentries ahould be fetched + * @return list of bibentries matching the bibcodes. Can be empty and differ in size to the size of requested + * bibcodes + */ private List performSearchByIds(Collection identifiers) throws FetcherException { List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); From 6e81ed07ea9e86232c783f48adfecd75e187d4d2 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Thu, 24 Oct 2019 20:20:48 +0200 Subject: [PATCH 15/34] Add token --- .../jabref/logic/importer/fetcher/AstrophysicsDataSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index f7d8c7f9c82..70f81f095dd 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -47,7 +47,7 @@ public class AstrophysicsDataSystem implements IdBasedParserFetcher, SearchBased private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; - private static final String API_KEY = ""; //TODO Add API Token + private static final String API_KEY = "tDueGIu6zl96OqkcCS5LOHboWbTgEEx8yAR7Etta"; private final ImportFormatPreferences preferences; public AstrophysicsDataSystem(ImportFormatPreferences preferences) { From c26612fd347026900baebbd5b968f01de19e4e28 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 22:00:07 +0200 Subject: [PATCH 16/34] updated unirest version and removes org.json dependency resolves #3703 --- build.gradle | 7 +++- external-libraries.txt | 4 +- src/main/java/module-info.java | 3 +- src/main/java/org/jabref/Globals.java | 7 +--- .../fetcher/AstrophysicsDataSystem.java | 6 +-- .../logic/importer/fetcher/CrossRef.java | 6 +-- .../logic/importer/fetcher/DOAJFetcher.java | 42 +++++++++---------- .../jabref/logic/importer/fetcher/IEEE.java | 4 +- .../fetcher/IsbnViaChimboriFetcher.java | 19 +++++---- .../logic/importer/fetcher/OpenAccessDoi.java | 14 +++---- .../logic/importer/fetcher/ScienceDirect.java | 28 ++++++------- .../importer/fetcher/SpringerFetcher.java | 4 +- .../logic/importer/fetcher/SpringerLink.java | 16 +++---- .../importer/fileformat/MrDLibImporter.java | 4 +- .../logic/importer/util/JsonReader.java | 2 +- .../logic/importer/util/ShortDOIService.java | 6 +-- .../org/jabref/logic/net/URLDownload.java | 4 +- .../java/org/jabref/logic/util/Version.java | 4 +- .../importer/fetcher/DOAJFetcherTest.java | 2 +- .../importer/fetcher/SpringerFetcherTest.java | 2 +- 20 files changed, 94 insertions(+), 90 deletions(-) diff --git a/build.gradle b/build.gradle index 5e3b16b3270..99dce1158e2 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,12 @@ dependencies { compile 'org.controlsfx:controlsfx:11.0.0' compile 'org.jsoup:jsoup:1.12.1' - compile 'com.mashape.unirest:unirest-java:1.4.9' + compile ('com.konghq:unirest-java:3.1.02'){ + // Use lib directly from maven, because current unirest build uses an older build of this library. + // There has been a bugfix for the automatic name manifest property of httpcore-nio this library depends on. + exclude module: "httpcore-nio" + } + compile ('org.apache.httpcomponents:httpcore-nio:4.4.12') compile 'org.slf4j:slf4j-api:2.0.0-alpha1' compile group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '3.0.0-SNAPSHOT' diff --git a/external-libraries.txt b/external-libraries.txt index 107ae83f6e4..bf0cb321046 100644 --- a/external-libraries.txt +++ b/external-libraries.txt @@ -50,9 +50,9 @@ Project: pgjdbc-ng URL: http://impossibl.github.io/pgjdbc-ng License: BSD-3-Clause -Id: com.mashape.unirest +Id: com.konghq.unirest Project: Unirest for Java -URL: https://github.com/Mashape/unirest-java +URL: https://github.com/Kong/unirest-java License: MIT Id: com.microsoft.azure:applicationinsights-core diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 791e2e50698..327ce26d480 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -55,7 +55,6 @@ requires org.apache.pdfbox; requires reactfx; requires commons.cli; - requires httpclient; requires com.github.tomtung.latex2unicode; requires jbibtex; requires citeproc.java; @@ -65,4 +64,6 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; + requires org.apache.httpcomponents.httpclient; + requires org.jsoup; } diff --git a/src/main/java/org/jabref/Globals.java b/src/main/java/org/jabref/Globals.java index a4399b13207..058b1cfe4b7 100644 --- a/src/main/java/org/jabref/Globals.java +++ b/src/main/java/org/jabref/Globals.java @@ -1,7 +1,6 @@ package org.jabref; import java.awt.GraphicsEnvironment; -import java.io.IOException; import java.util.Optional; import java.util.UUID; @@ -26,11 +25,11 @@ import org.jabref.preferences.JabRefPreferences; import com.google.common.base.StandardSystemProperty; -import com.mashape.unirest.http.Unirest; import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.TelemetryConfiguration; import com.microsoft.applicationinsights.internal.shutdown.SDKShutdownActivity; import com.microsoft.applicationinsights.telemetry.SessionState; +import kong.unirest.Unirest; public class Globals { @@ -129,9 +128,7 @@ public static void shutdownThreadPools() { public static void stopBackgroundTasks() { stopTelemetryClient(); - try { - Unirest.shutdown(); - } catch (IOException ignore) { } + Unirest.shutDown(); } public static Optional getTelemetryClient() { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 70f81f095dd..fcf2750dd79 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -34,10 +34,10 @@ import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; /** * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 50e9da651ac..aca7a6b3397 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -27,10 +27,10 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.util.OptionalUtil; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; /** * A class for fetching DOIs from CrossRef diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 074c47cff62..96f9d4f4140 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -23,9 +23,9 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.strings.StringUtil; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -153,25 +153,6 @@ public static BibEntry parseBibJSONtoBibtex(JSONObject bibJsonEntry, Character k return entry; } - @Override - public String getName() { - return "DOAJ"; - } - - @Override - public Optional getHelpPage() { - return Optional.of(HelpFile.FETCHER_DOAJ); - } - - @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, query); - uriBuilder.addParameter("pageSize", "30"); // Number of results - //uriBuilder.addParameter("page", "1"); // Page (not needed so far) - return uriBuilder.build().toURL(); - } - /** * @implNote slightly altered version based on https://gist.github.com/enginer/230e2dc2f1d213a825d5 */ @@ -196,6 +177,25 @@ private static String appendSegmentToPath(String path, String segment) { return path + "/" + segment; } + @Override + public String getName() { + return "DOAJ"; + } + + @Override + public Optional getHelpPage() { + return Optional.of(HelpFile.FETCHER_DOAJ); + } + + @Override + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); + DOAJFetcher.addPath(uriBuilder, query); + uriBuilder.addParameter("pageSize", "30"); // Number of results + //uriBuilder.addParameter("page", "1"); // Page (not needed so far) + return uriBuilder.build().toURL(); + } + @Override public Parser getParser() { return inputStream -> { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index 79b03b83d71..0ffd58376ae 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -28,9 +28,9 @@ import org.jabref.model.entry.identifier.DOI; import org.jabref.model.entry.types.StandardEntryType; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java index e3f84536f86..068f039c8b6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java @@ -13,14 +13,15 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.strings.StringUtil; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; +import kong.unirest.RawResponse; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; /** * Fetcher for ISBN using https://bibtex.chimbori.com/, which in turn uses Amazon's API. */ public class IsbnViaChimboriFetcher extends AbstractIsbnFetcher { + private RawResponse postResponse; public IsbnViaChimboriFetcher(ImportFormatPreferences importFormatPreferences) { super(importFormatPreferences); @@ -47,21 +48,21 @@ public Optional performSearchById(String identifier) throws FetcherExc this.ensureThatIsbnIsValid(identifier); - HttpResponse postResponse; + postResponse = null; try { - postResponse = Unirest.post("https://bibtex.chimbori.com/isbn-bibtex") - .field("isbn", identifier) - .asString(); + Unirest.post("https://bibtex.chimbori.com/isbn-bibtex") + .field("isbn", identifier) + .thenConsume(rawResponse -> postResponse = rawResponse); } catch (UnirestException e) { throw new FetcherException("Could not retrieve data from chimbori.com", e); } if (postResponse.getStatus() != 200) { - throw new FetcherException("Error while retrieving data from chimbori.com: " + postResponse.getBody()); + throw new FetcherException("Error while retrieving data from chimbori.com: " + postResponse.getContentAsString()); } List fetchedEntries; try { - fetchedEntries = getParser().parseEntries(postResponse.getRawBody()); + fetchedEntries = getParser().parseEntries(postResponse.getContent()); } catch (ParseException e) { throw new FetcherException("An internal parser error occurred", e); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java b/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java index 64e3ab046e6..7bb7a5125e1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java @@ -11,11 +11,11 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.JsonNode; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import org.json.JSONObject; +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; +import kong.unirest.json.JSONObject; /** * A fulltext fetcher that uses oaDOI. @@ -49,8 +49,8 @@ public TrustLevel getTrustLevel() { public Optional findFullText(DOI doi) throws UnirestException, MalformedURLException { HttpResponse jsonResponse = Unirest.get(API_URL + doi.getDOI() + "?email=developers@jabref.org") - .header("accept", "application/json") - .asJson(); + .header("accept", "application/json") + .asJson(); JSONObject root = jsonResponse.getBody().getObject(); Optional url = Optional.ofNullable(root.optJSONObject("best_oa_location")) .map(location -> location.optString("url")); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java b/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java index 4b66f6d49e4..56e27018273 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java @@ -11,13 +11,13 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.JsonNode; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -28,7 +28,7 @@ /** * FulltextFetcher implementation that attempts to find a PDF URL at ScienceDirect. * - * @see http://dev.elsevier.com/ + * @see 'https://dev.elsevier.com/' */ public class ScienceDirect implements FulltextFetcher { private static final Logger LOGGER = LoggerFactory.getLogger(ScienceDirect.class); @@ -50,9 +50,9 @@ public Optional findFullText(BibEntry entry) throws IOException { // scrape the web page not as mobile client! if (!sciLink.isEmpty()) { Document html = Jsoup.connect(sciLink) - .userAgent(URLDownload.USER_AGENT) - .referrer("http://www.google.com") - .ignoreHttpErrors(true).get(); + .userAgent(URLDownload.USER_AGENT) + .referrer("http://www.google.com") + .ignoreHttpErrors(true).get(); // Retrieve PDF link from meta data (most recent) Elements metaLinks = html.getElementsByAttributeValue("name", "citation_pdf_url"); @@ -98,9 +98,9 @@ private String getUrlByDoi(String doi) throws UnirestException { try { String request = API_URL + doi; HttpResponse jsonResponse = Unirest.get(request) - .header("X-ELS-APIKey", API_KEY) - .queryString("httpAccept", "application/json") - .asJson(); + .header("X-ELS-APIKey", API_KEY) + .queryString("httpAccept", "application/json") + .asJson(); JSONObject json = jsonResponse.getBody().getObject(); JSONArray links = json.getJSONObject("full-text-retrieval-response").getJSONObject("coredata").getJSONArray("link"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index fd38b2e61b1..8d9221709a0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -22,9 +22,9 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java index 73722330ccd..28f2357efc5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java @@ -10,11 +10,11 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.JsonNode; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import org.json.JSONObject; +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; +import kong.unirest.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,9 +42,9 @@ public Optional findFullText(BibEntry entry) throws IOException { // Available in catalog? try { HttpResponse jsonResponse = Unirest.get(API_URL) - .queryString("api_key", API_KEY) - .queryString("q", String.format("doi:%s", doi.get().getDOI())) - .asJson(); + .queryString("api_key", API_KEY) + .queryString("q", String.format("doi:%s", doi.get().getDOI())) + .asJson(); JSONObject json = jsonResponse.getBody().getObject(); int results = json.getJSONArray("result").getJSONObject(0).getInt("total"); diff --git a/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java index 659ee8da831..37c7676307b 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java @@ -17,8 +17,8 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.json.JSONException; -import org.json.JSONObject; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/logic/importer/util/JsonReader.java b/src/main/java/org/jabref/logic/importer/util/JsonReader.java index ea46bb7f756..7bfd02986b1 100644 --- a/src/main/java/org/jabref/logic/importer/util/JsonReader.java +++ b/src/main/java/org/jabref/logic/importer/util/JsonReader.java @@ -8,7 +8,7 @@ import org.jabref.logic.importer.ParseException; -import org.json.JSONObject; +import kong.unirest.json.JSONObject; /** * Converts an {@link InputStream} into a {@link JSONObject}. diff --git a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java index bbb3dc7b4c6..a74745f0021 100644 --- a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java +++ b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java @@ -10,14 +10,14 @@ import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.identifier.DOI; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONException; -import org.json.JSONObject; /** * Class for obtaining shortened DOI names. * - * @see http://shortdoi.org + * @see 'https://shortdoi.org' */ public class ShortDOIService { diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 0eb2508cf13..34fc9537f51 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -41,7 +41,7 @@ import org.jabref.logic.util.io.FileUtil; import org.jabref.model.util.FileHelper; -import com.mashape.unirest.http.Unirest; +import kong.unirest.Unirest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,7 +131,7 @@ public URL getSource() { } public String getMimeType() { - Unirest.setDefaultHeader("User-Agent", "Mozilla/5.0 (Windows; U; WindowsNT 5.1; en-US; rv1.8.1.6) Gecko/20070725 Firefox/2.0.0.6"); + Unirest.config().setDefaultHeader("User-Agent", "Mozilla/5.0 (Windows; U; WindowsNT 5.1; en-US; rv1.8.1.6) Gecko/20070725 Firefox/2.0.0.6"); String contentType; // Try to use HEAD request to avoid downloading the whole file diff --git a/src/main/java/org/jabref/logic/util/Version.java b/src/main/java/org/jabref/logic/util/Version.java index e288c718064..0775fd6a0fb 100644 --- a/src/main/java/org/jabref/logic/util/Version.java +++ b/src/main/java/org/jabref/logic/util/Version.java @@ -12,8 +12,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.json.JSONArray; -import org.json.JSONObject; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java index a5c81f98570..68ea36fafa0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java @@ -10,8 +10,8 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java index 063b561a4cf..c4db21ec256 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java @@ -9,7 +9,7 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; -import org.json.JSONObject; +import kong.unirest.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; From d5764fbca34f1a7cb1a545e2af8c9f18e4d4bf56 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Fri, 25 Oct 2019 11:34:47 +0200 Subject: [PATCH 17/34] updated unirest version and removes org.json dependency resolves #3703 [dependency fix in unirest](https://github.com/Kong/unirest-java/issues/311) --- build.gradle | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 99dce1158e2..d17f57fef8b 100644 --- a/build.gradle +++ b/build.gradle @@ -159,12 +159,7 @@ dependencies { compile 'org.controlsfx:controlsfx:11.0.0' compile 'org.jsoup:jsoup:1.12.1' - compile ('com.konghq:unirest-java:3.1.02'){ - // Use lib directly from maven, because current unirest build uses an older build of this library. - // There has been a bugfix for the automatic name manifest property of httpcore-nio this library depends on. - exclude module: "httpcore-nio" - } - compile ('org.apache.httpcomponents:httpcore-nio:4.4.12') + compile ('com.konghq:unirest-java:3.1.03') compile 'org.slf4j:slf4j-api:2.0.0-alpha1' compile group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '3.0.0-SNAPSHOT' From f4b164d75213205033183fedcb83e63eeaef5a8b Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Fri, 25 Oct 2019 11:47:15 +0200 Subject: [PATCH 18/34] updated unirest version and removes org.json dependency resolves #3703 [dependency fix in unirest](https://github.com/Kong/unirest-java/issues/311) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d17f57fef8b..bb680358efe 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,7 @@ dependencies { compile 'org.controlsfx:controlsfx:11.0.0' compile 'org.jsoup:jsoup:1.12.1' - compile ('com.konghq:unirest-java:3.1.03') + compile 'com.konghq:unirest-java:3.1.03' compile 'org.slf4j:slf4j-api:2.0.0-alpha1' compile group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '3.0.0-SNAPSHOT' From 5405c53c2e575a580a39e7a3fa78cc3f33eb05a8 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Tue, 22 Oct 2019 20:04:32 +0200 Subject: [PATCH 19/34] Changed performSearch to default implementation Removes method as it is essentially the same as the default implementation. --- .../jabref/logic/importer/fetcher/AstrophysicsDataSystem.java | 2 -- src/main/resources/csl-styles | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 70f81f095dd..0e1adceaa9a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -22,7 +22,6 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; @@ -31,7 +30,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; diff --git a/src/main/resources/csl-styles b/src/main/resources/csl-styles index c11d9747ac2..c3fd4bdeadb 160000 --- a/src/main/resources/csl-styles +++ b/src/main/resources/csl-styles @@ -1 +1 @@ -Subproject commit c11d9747ac24fc675ce220fffe8004c8b9de2a70 +Subproject commit c3fd4bdeadbfc4a713284ad15cca64c8198a7dc7 From 9d5b0b7868ee64c9d5318e5e0b573e844c2b4318 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Tue, 22 Oct 2019 23:25:30 +0200 Subject: [PATCH 20/34] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- src/main/java/module-info.java | 1 + .../jabref/logic/importer/WebFetchers.java | 4 + .../fetcher/NewAstrophysicsDataSystem.java | 180 ++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 791e2e50698..bfb3a41fdcc 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -65,4 +65,5 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; + requires json; } diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index 66319f5e6b4..4161786d0c4 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -27,6 +27,7 @@ import org.jabref.logic.importer.fetcher.LibraryOfCongress; import org.jabref.logic.importer.fetcher.MathSciNet; import org.jabref.logic.importer.fetcher.MedlineFetcher; +import org.jabref.logic.importer.fetcher.NewAstrophysicsDataSystem; import org.jabref.logic.importer.fetcher.OpenAccessDoi; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; @@ -88,6 +89,7 @@ public static SortedSet getSearchBasedFetchers(ImportFormatP set.add(new GvkFetcher()); set.add(new MedlineFetcher()); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); set.add(new ZbMATH(importFormatPreferences)); set.add(new ACMPortalFetcher(importFormatPreferences)); @@ -108,6 +110,7 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new ArXiv(importFormatPreferences)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new DiVA(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); @@ -127,6 +130,7 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) { SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); + set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java new file mode 100644 index 00000000000..8d93f6ee4aa --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -0,0 +1,180 @@ +package org.jabref.logic.importer.fetcher; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.EntryBasedParserFetcher; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.IdBasedParserFetcher; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.logic.net.URLDownload; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.strings.StringUtil; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.apache.http.client.utils.URIBuilder; +import org.json.JSONObject; + +//TODO Replace Old ADS after the new one is mature + +/** + * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) + *

+ * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html + *

+ * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple + * calls to get BibTeX, status: September 2016) + */ +public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { + + private static String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; + + private static String API_KEY = ""; //TODO Add API Token + + private final ImportFormatPreferences preferences; + + public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { + this.preferences = Objects.requireNonNull(preferences); + } + + private String buildPostData(String... bibcodes) { + JSONObject obj = new JSONObject(); + obj.put("bibcode", bibcodes); + return obj.toString(); + } + + @Override + public String getName() { + return "New SAO/NASA Astrophysics Data System"; + } + + @Override + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); + } + + @Override + public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { + StringBuilder stringBuilder = new StringBuilder(); + + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); + Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author" + a); + + if (title.isPresent()) { + stringBuilder.append(title.get()) + .append(author.map(s -> " OR " + s) + .orElse("")); + } else { + stringBuilder.append(author.orElse("")); + } + String query = stringBuilder.toString().trim(); + + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); + } + + @Override + public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + + @Override + public Optional getHelpPage() { + return Optional.of(HelpFile.FETCHER_ADS); + } + + @Override + public Parser getParser() { + return new BibtexParser(preferences, new DummyFileUpdateMonitor()); + } + + @Override + public void doPostCleanup(BibEntry entry) { + + } + + @Override + public List performSearch(BibEntry entry) throws FetcherException { + return Collections.emptyList(); + } + + @Override + public Optional performSearchById(String identifier) throws FetcherException { + + Optional> results = performSearchByIds(identifier); + if (results.isEmpty()) { + return Optional.empty(); + } + + List fetchedEntries = results.get(); + + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } + + private Optional> performSearchByIds(String... identifiers) throws FetcherException { + + long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); + if (idCount == 0) { + return Optional.empty(); + } + + try { + + String postData = buildPostData(identifiers); + URLDownload download = new URLDownload(getURLForID("")); + download.addHeader("Authorization", "Bearer " + API_KEY); + download.addHeader("ContentType", "application/json"); + download.setPostData(postData); + download.asString(); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + List fetchedEntries = getParser().parseEntries(obj.getString("export")); + + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return Optional.of(fetchedEntries); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } catch (ParseException e) { + throw new FetcherException("An internal parser error occurred", e); + } + } + + @Override + public List performSearch(String query) throws FetcherException { + + return Collections.emptyList(); + } +} From 0db18f9dbd7fcbdda3ed37022b9fe1f005a3800c Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 20:02:29 +0200 Subject: [PATCH 21/34] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- .../fetcher/NewAstrophysicsDataSystem.java | 103 +++++++++++++----- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 8d93f6ee4aa..0ca438eff28 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -4,6 +4,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -26,6 +27,8 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; //TODO Replace Old ADS after the new one is mature @@ -40,8 +43,8 @@ */ public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - private static String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; - private static String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; + private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; + private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; private static String API_KEY = ""; //TODO Add API Token @@ -75,7 +78,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR StringBuilder stringBuilder = new StringBuilder(); Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); - Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author" + a); + Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author:" + a); if (title.isPresent()) { stringBuilder.append(title.get()) @@ -89,6 +92,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); builder.addParameter("fl", "bibcode"); + builder.addParameter("rows", "20"); return builder.build().toURL(); } @@ -114,18 +118,69 @@ public void doPostCleanup(BibEntry entry) { @Override public List performSearch(BibEntry entry) throws FetcherException { - return Collections.emptyList(); + + if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForEntry(entry)); + String[] bibcodeArray = new String[bibcodes.size()]; + return performSearchByIds(bibcodes.toArray(bibcodeArray)); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } } @Override - public Optional performSearchById(String identifier) throws FetcherException { + public List performSearch(String query) throws FetcherException { - Optional> results = performSearchByIds(identifier); - if (results.isEmpty()) { - return Optional.empty(); + if (StringUtil.isBlank(query)) { + return Collections.emptyList(); + } + + try { + List bibcodes = fetchBibcodes(getURLForQuery(query)); + String[] bibcodeArray = new String[bibcodes.size()]; + return performSearchByIds(bibcodes.toArray(bibcodeArray)); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } + } + + private List fetchBibcodes(URL url) throws FetcherException { + + try { + URLDownload download = new URLDownload(url); + download.addHeader("Authorization", "Bearer " + API_KEY); + String content = download.asString(); + JSONObject obj = new JSONObject(content); + + try { + JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); + List bibcodes = new ArrayList<>(); + for (int i = 0; i < codes.length(); i++) { + bibcodes.add(codes.getJSONObject(i).getString("bibcode")); + } + return bibcodes; + } catch (JSONException e) { + return Collections.emptyList(); + } + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); } + } - List fetchedEntries = results.get(); + @Override + public Optional performSearchById(String identifier) throws FetcherException { + List fetchedEntries = performSearchByIds(identifier); + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } if (fetchedEntries.size() > 1) { LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier @@ -136,33 +191,33 @@ public Optional performSearchById(String identifier) throws FetcherExc return Optional.of(entry); } - private Optional> performSearchByIds(String... identifiers) throws FetcherException { + private List performSearchByIds(String... identifiers) throws FetcherException { long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); if (idCount == 0) { - return Optional.empty(); + return Collections.emptyList(); } - try { - String postData = buildPostData(identifiers); URLDownload download = new URLDownload(getURLForID("")); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); download.setPostData(postData); - download.asString(); String content = download.asString(); JSONObject obj = new JSONObject(content); - List fetchedEntries = getParser().parseEntries(obj.getString("export")); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); + try { + List fetchedEntries = getParser().parseEntries(obj.optString("export")); + if (fetchedEntries.isEmpty()) { + return Collections.emptyList(); + } + // Post-cleanup + fetchedEntries.forEach(this::doPostCleanup); + + return fetchedEntries; + } catch (JSONException e) { + return Collections.emptyList(); } - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - - return Optional.of(fetchedEntries); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -172,9 +227,5 @@ private Optional> performSearchByIds(String... identifiers) throw } } - @Override - public List performSearch(String query) throws FetcherException { - return Collections.emptyList(); - } } From c82c4ff9541b42016ab27e20a2b6691ceff7b1bd Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 20:36:04 +0200 Subject: [PATCH 22/34] Added new ADS fetcher Added a new ADS fetcher that uses the new ADS API, ref #4949 Preserve the old fetcher until the new fetcher is stable/ well tested --- .../logic/importer/fetcher/NewAstrophysicsDataSystem.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 0ca438eff28..824cf89c6e2 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -226,6 +226,4 @@ private List performSearchByIds(String... identifiers) throws FetcherE throw new FetcherException("An internal parser error occurred", e); } } - - } From 8435c6ccb5a1fdc938a8bb01c30815bdffa4792a Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 21:10:29 +0200 Subject: [PATCH 23/34] added cleanup --- .../fetcher/NewAstrophysicsDataSystem.java | 19 +- .../NewAstrophysicsDataSystemTest.java | 206 ++++++++++++++++++ 2 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 824cf89c6e2..9481aace5bc 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -11,6 +11,11 @@ import java.util.Objects; import java.util.Optional; +import org.jabref.logic.cleanup.MoveFieldCleanup; +import org.jabref.logic.formatter.bibtexfields.ClearFormatter; +import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; +import org.jabref.logic.formatter.bibtexfields.RemoveBracesFormatter; +import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.EntryBasedParserFetcher; import org.jabref.logic.importer.FetcherException; @@ -21,8 +26,10 @@ import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; +import org.jabref.model.cleanup.FieldFormatterCleanup; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; @@ -113,7 +120,17 @@ public Parser getParser() { @Override public void doPostCleanup(BibEntry entry) { - + new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveNewlinesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); + new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); + + // Remove ADS note + new FieldFormatterCleanup(new UnknownField("adsnote"), new ClearFormatter()).cleanup(entry); + // Move adsurl to url field + new MoveFieldCleanup(new UnknownField("adsurl"), StandardField.URL).cleanup(entry); + // The fetcher adds some garbage (number of found entries etc before) + entry.setCommentsBeforeEntry(""); } @Override diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java new file mode 100644 index 00000000000..649a39a9054 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -0,0 +1,206 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.jabref.logic.bibtex.FieldContentParserPreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.testutils.category.FetcherTest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@FetcherTest +public class NewAstrophysicsDataSystemTest { + + private NewAstrophysicsDataSystem fetcher; + private BibEntry diezSliceTheoremEntry, famaeyMcGaughEntry, sunWelchEntry, xiongSunEntry, ingersollPollardEntry, luceyPaulEntry; + + @BeforeEach + public void setUp() throws Exception { + ImportFormatPreferences importFormatPreferences = mock(ImportFormatPreferences.class); + when(importFormatPreferences.getFieldContentParserPreferences()).thenReturn( + mock(FieldContentParserPreferences.class)); + fetcher = new NewAstrophysicsDataSystem(importFormatPreferences); + + diezSliceTheoremEntry = new BibEntry(); + diezSliceTheoremEntry.setType(StandardEntryType.Article); + diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); + diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); + diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.ABSTRACT, + "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " + + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " + + "fundamental tool to generalize the result of Palais to this " + + "infinite-dimensional setting. The presented slice theorem is illustrated " + + "by its application to gauge theories: the action of the gauge " + + "transformation group admits smooth slices at every point and thus the " + + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " + + "covariant and symplectic formulation of classical field theory is " + + "proposed and extensively discussed. At the root of this novel framework " + + "is the incorporation of field degrees of freedom F and spacetime M into " + + "the product manifold F * M. The induced bigrading of differential forms " + + "is used in order to carry over the usual symplectic theory to this new " + + "setting. The examples of the Klein-Gordon field and general Yang-Mills " + + "theory illustrate that the presented approach conveniently handles the " + + "occurring symmetries."); + + famaeyMcGaughEntry = new BibEntry(); + famaeyMcGaughEntry.setType(StandardEntryType.Article); + famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); + famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); + famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); + famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); + famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); + famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); + famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + + sunWelchEntry = new BibEntry(); + sunWelchEntry.setType(StandardEntryType.Article); + sunWelchEntry.setCiteKey("2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); + sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); + sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.PAGES, "44-48"); + sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); + sunWelchEntry.setField(StandardField.VOLUME, "11"); + sunWelchEntry.setField(StandardField.YEAR, "2012"); + sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + + xiongSunEntry = new BibEntry(); + xiongSunEntry.setType(StandardEntryType.Article); + xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); + xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); + xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.PAGES, "879-889"); + xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); + xiongSunEntry.setField(StandardField.VOLUME, "45"); + xiongSunEntry.setField(StandardField.YEAR, "2007"); + xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + + ingersollPollardEntry = new BibEntry(); + ingersollPollardEntry.setType(StandardEntryType.Article); + ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); + ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); + ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); + ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.VOLUME, "52"); + ingersollPollardEntry.setField(StandardField.YEAR, "1982"); + ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + + luceyPaulEntry = new BibEntry(); + luceyPaulEntry.setType(StandardEntryType.Article); + luceyPaulEntry.setCiteKey("2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); + luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); + luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); + luceyPaulEntry.setField(StandardField.PAGES, "20297-20306"); + luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); + luceyPaulEntry.setField(StandardField.VOLUME, "105"); + luceyPaulEntry.setField(StandardField.YEAR, "2000"); + luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + } + + @Test + public void testHelpPage() { + assertEquals("ADS", fetcher.getHelpPage().get().getPageName()); + } + + @Test + public void testGetName() { + assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); + } + + @Test + public void searchByQueryFindsEntry() throws Exception { + List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); + assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + } + + @Test + public void searchByEntryFindsEntry() throws Exception { + BibEntry searchEntry = new BibEntry(); + searchEntry.setField(StandardField.TITLE, "slice theorem"); + searchEntry.setField(StandardField.AUTHOR, "Diez"); + + List fetchedEntries = fetcher.performSearch(searchEntry); + assertFalse(fetchedEntries.isEmpty()); + assertEquals(diezSliceTheoremEntry, fetchedEntries.get(0)); + } + + @Test + public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); + fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright + assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByIdEmptyDOI() throws Exception { + Optional fetchedEntry = fetcher.performSearchById(""); + assertEquals(Optional.empty(), fetchedEntry); + } + + @Test + public void testPerformSearchByIdInvalidDoi() throws Exception { + assertEquals(Optional.empty(), fetcher.performSearchById("this.doi.will.fail")); + } + + @Test + public void testPerformSearchBySunWelchEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1038/nmat3160"); + fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT)); //Remove abstract due to copyright + assertEquals(Optional.of(sunWelchEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByXiongSunEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1109/TGRS.2006.890567"); + assertEquals(Optional.of(xiongSunEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByIngersollPollardEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("10.1016/0019-1035(82)90169-5"); + assertEquals(Optional.of(ingersollPollardEntry), fetchedEntry); + } + + @Test + public void testPerformSearchByLuceyPaulEntry() throws Exception { + Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); + assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); + } +} From 339df67e30a152bcee5c637404ca99f55a6bd5be Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 21:45:32 +0200 Subject: [PATCH 24/34] support doi and ads bibcodes --- .../fetcher/NewAstrophysicsDataSystem.java | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 9481aace5bc..5183a3d8e67 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -67,6 +67,10 @@ private String buildPostData(String... bibcodes) { return obj.toString(); } + private URL getURLforExport() throws URISyntaxException, MalformedURLException { + return new URIBuilder(API_EXPORT_URL).build().toURL(); + } + @Override public String getName() { return "New SAO/NASA Astrophysics Data System"; @@ -89,7 +93,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR if (title.isPresent()) { stringBuilder.append(title.get()) - .append(author.map(s -> " OR " + s) + .append(author.map(s -> " AND " + s) .orElse("")); } else { stringBuilder.append(author.orElse("")); @@ -105,7 +109,11 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - return new URIBuilder(API_EXPORT_URL).build().toURL(); + String query = "doi:" + identifier + " OR " + "bibcode:" + identifier; + URIBuilder builder = new URIBuilder(API_SEARCH_URL); + builder.addParameter("q", query); + builder.addParameter("fl", "bibcode"); + return builder.build().toURL(); } @Override @@ -194,18 +202,29 @@ private List fetchBibcodes(URL url) throws FetcherException { @Override public Optional performSearchById(String identifier) throws FetcherException { - List fetchedEntries = performSearchByIds(identifier); - if (fetchedEntries.isEmpty()) { + if (StringUtil.isBlank(identifier)) { return Optional.empty(); } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } + try { + List bibcodes = fetchBibcodes(getURLForID(identifier)); + String[] bibcodeArray = new String[bibcodes.size()]; + List fetchedEntries = performSearchByIds(bibcodes.toArray(bibcodeArray)); - BibEntry entry = fetchedEntries.get(0); - return Optional.of(entry); + if (fetchedEntries.isEmpty()) { + return Optional.empty(); + } + if (fetchedEntries.size() > 1) { + LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier + + ". We will use the first entry."); + } + BibEntry entry = fetchedEntries.get(0); + return Optional.of(entry); + } catch (URISyntaxException e) { + throw new FetcherException("Search URI is malformed", e); + } catch (IOException e) { + throw new FetcherException("A network error occurred", e); + } } private List performSearchByIds(String... identifiers) throws FetcherException { @@ -216,7 +235,7 @@ private List performSearchByIds(String... identifiers) throws FetcherE } try { String postData = buildPostData(identifiers); - URLDownload download = new URLDownload(getURLForID("")); + URLDownload download = new URLDownload(getURLforExport()); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); download.setPostData(postData); From b6a8feab39ba2a9d95d5e7f781a9e8a83f80306e Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:07:22 +0200 Subject: [PATCH 25/34] fixes wrong search syntax --- .../importer/fetcher/NewAstrophysicsDataSystem.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 5183a3d8e67..510e213ebe1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -88,8 +88,8 @@ public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLE public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { StringBuilder stringBuilder = new StringBuilder(); - Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:" + t); - Optional author = entry.getFieldOrAlias(StandardField.TITLE).map(a -> "author:" + a); + Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); + Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); if (title.isPresent()) { stringBuilder.append(title.get()) @@ -109,7 +109,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR @Override public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - String query = "doi:" + identifier + " OR " + "bibcode:" + identifier; + String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", query); builder.addParameter("fl", "bibcode"); @@ -151,7 +151,8 @@ public List performSearch(BibEntry entry) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForEntry(entry)); String[] bibcodeArray = new String[bibcodes.size()]; - return performSearchByIds(bibcodes.toArray(bibcodeArray)); + bibcodes.toArray(bibcodeArray); + return performSearchByIds(bibcodeArray); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { From d2edeedbd4e099ac72a5b53e5694e5dae4031811 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:08:17 +0200 Subject: [PATCH 26/34] fixed test cases to API response --- .../NewAstrophysicsDataSystemTest.java | 80 +++++++++---------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java index 649a39a9054..561d316a0d0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -35,95 +35,89 @@ public void setUp() throws Exception { diezSliceTheoremEntry = new BibEntry(); diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2014arXiv1405.2249D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, T."); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem for Fr$\\backslash$'echet group actions and covariant symplectic field theory"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2014"); + diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); + diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); + diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); + diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1405.2249"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "ArXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematical Physics, Mathematics - Differential Geometry, Mathematics - Symplectic Geometry, 58B99, 58Z05, 58B25, 22E65, 58D19, 53D20, 53D42"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "#may#"); - diezSliceTheoremEntry.setField(new UnknownField("primaryclass"), "math-ph"); - diezSliceTheoremEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2014arXiv1405.2249D"); + diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); + diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); + diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); + diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); + diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(new UnknownField("eid"), "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); + diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "A general slice theorem for the action of a Fr$\\backslash$'echet Lie group on a " - + "Fr$\\backslash$'echet manifolds is established. The Nash-Moser theorem provides the " - + "fundamental tool to generalize the result of Palais to this " - + "infinite-dimensional setting. The presented slice theorem is illustrated " - + "by its application to gauge theories: the action of the gauge " - + "transformation group admits smooth slices at every point and thus the " - + "gauge orbit space is stratified by Fr$\\backslash$'echet manifolds. Furthermore, a " - + "covariant and symplectic formulation of classical field theory is " - + "proposed and extensively discussed. At the root of this novel framework " - + "is the incorporation of field degrees of freedom F and spacetime M into " - + "the product manifold F * M. The induced bigrading of differential forms " - + "is used in order to carry over the usual symplectic theory to this new " - + "setting. The examples of the Klein-Gordon field and general Yang-Mills " - + "theory illustrate that the presented approach conveniently handles the " - + "occurring symmetries."); + "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); famaeyMcGaughEntry = new BibEntry(); famaeyMcGaughEntry.setType(StandardEntryType.Article); famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, B. and McGaugh, S. S."); + famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "#sep#"); + famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); + famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); + famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics"); - famaeyMcGaughEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012LRR....15...10F"); + famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); + famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); sunWelchEntry = new BibEntry(); sunWelchEntry.setType(StandardEntryType.Article); sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Y. and Welch, G. C. and Leong, W. L. and Takacs, C. J. and Bazan, G. C. and Heeger, A. J."); + sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "#jan#"); + sunWelchEntry.setField(StandardField.MONTH, "Jan"); + sunWelchEntry.setField(StandardField.NUMBER, "1"); sunWelchEntry.setField(StandardField.PAGES, "44-48"); sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); sunWelchEntry.setField(StandardField.VOLUME, "11"); sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2012NatMa..11...44S"); + sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); xiongSunEntry = new BibEntry(); xiongSunEntry.setType(StandardEntryType.Article); xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, X. and Sun, J. and Barnes, W. and Salomonson, V. and Esposito, J. and Erives, H. and Guenther, B."); + xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "#apr#"); + xiongSunEntry.setField(StandardField.MONTH, "Apr"); + xiongSunEntry.setField(StandardField.NUMBER, "4"); xiongSunEntry.setField(StandardField.PAGES, "879-889"); xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); xiongSunEntry.setField(StandardField.VOLUME, "45"); xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2007ITGRS..45..879X"); + xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); ingersollPollardEntry = new BibEntry(); ingersollPollardEntry.setType(StandardEntryType.Article); ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS"); - ingersollPollardEntry.setField(StandardField.MONTH, "#oct#"); + ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); + ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); + ingersollPollardEntry.setField(StandardField.NUMBER, "1"); ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn - Scale analysis, anelastic equations, barotropic stability criterion"); + ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); ingersollPollardEntry.setField(StandardField.VOLUME, "52"); ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/1982Icar...52...62I"); + ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); luceyPaulEntry = new BibEntry(); luceyPaulEntry.setType(StandardEntryType.Article); luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, P. G. and Blewett, D. T. and Jolliff, B. L."); + luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); @@ -131,7 +125,9 @@ public void setUp() throws Exception { luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); luceyPaulEntry.setField(StandardField.VOLUME, "105"); luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "http://adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); + luceyPaulEntry.setField(StandardField.MONTH, "Jan"); + luceyPaulEntry.setField(StandardField.NUMBER, "E8"); } @Test @@ -165,7 +161,7 @@ public void searchByEntryFindsEntry() throws Exception { public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); + assertEquals(famaeyMcGaughEntry, fetchedEntry.get()); } @Test From bbcac7be720df53d14d058d1fe0aa1baaa4e245b Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Wed, 23 Oct 2019 23:28:18 +0200 Subject: [PATCH 27/34] fixed bug where equals doesnt work with unknown field --- .../fetcher/NewAstrophysicsDataSystemTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java index 561d316a0d0..403bfaefa55 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java @@ -1,6 +1,5 @@ package org.jabref.logic.importer.fetcher; -import java.util.Collections; import java.util.List; import java.util.Optional; @@ -8,7 +7,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; @@ -17,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -45,7 +44,7 @@ public void setUp() throws Exception { diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(new UnknownField("eid"), "arXiv:1812.04698"); + diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); diezSliceTheoremEntry.setField(StandardField.ABSTRACT, @@ -64,7 +63,7 @@ public void setUp() throws Exception { famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); - famaeyMcGaughEntry.setField(new UnknownField("eid"), "10"); + famaeyMcGaughEntry.setField(StandardField.EID, "10"); famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); @@ -143,7 +142,7 @@ public void testGetName() { @Test public void searchByQueryFindsEntry() throws Exception { List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertEquals(Collections.singletonList(diezSliceTheoremEntry), fetchedEntries); + assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); } @Test @@ -161,7 +160,7 @@ public void searchByEntryFindsEntry() throws Exception { public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(famaeyMcGaughEntry, fetchedEntry.get()); + assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); } @Test From 3871673bd5e87a8b2d9a93104f3877e27614eb50 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:44:14 +0200 Subject: [PATCH 28/34] changed array method parameters to Collections --- .../fetcher/NewAstrophysicsDataSystem.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java index 510e213ebe1..37dedfb6ca9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java @@ -5,11 +5,12 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import org.jabref.logic.cleanup.MoveFieldCleanup; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; @@ -61,7 +62,7 @@ public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); } - private String buildPostData(String... bibcodes) { + private String buildPostData(Collection bibcodes) { JSONObject obj = new JSONObject(); obj.put("bibcode", bibcodes); return obj.toString(); @@ -150,9 +151,7 @@ public List performSearch(BibEntry entry) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForEntry(entry)); - String[] bibcodeArray = new String[bibcodes.size()]; - bibcodes.toArray(bibcodeArray); - return performSearchByIds(bibcodeArray); + return performSearchByIds(bibcodes); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -169,8 +168,7 @@ public List performSearch(String query) throws FetcherException { try { List bibcodes = fetchBibcodes(getURLForQuery(query)); - String[] bibcodeArray = new String[bibcodes.size()]; - return performSearchByIds(bibcodes.toArray(bibcodeArray)); + return performSearchByIds(bibcodes); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { @@ -209,8 +207,7 @@ public Optional performSearchById(String identifier) throws FetcherExc try { List bibcodes = fetchBibcodes(getURLForID(identifier)); - String[] bibcodeArray = new String[bibcodes.size()]; - List fetchedEntries = performSearchByIds(bibcodes.toArray(bibcodeArray)); + List fetchedEntries = performSearchByIds(bibcodes); if (fetchedEntries.isEmpty()) { return Optional.empty(); @@ -228,14 +225,14 @@ public Optional performSearchById(String identifier) throws FetcherExc } } - private List performSearchByIds(String... identifiers) throws FetcherException { + private List performSearchByIds(Collection identifiers) throws FetcherException { - long idCount = Arrays.stream(identifiers).filter(identifier -> !StringUtil.isBlank(identifier)).count(); - if (idCount == 0) { + List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); + if (ids.isEmpty()) { return Collections.emptyList(); } try { - String postData = buildPostData(identifiers); + String postData = buildPostData(ids); URLDownload download = new URLDownload(getURLforExport()); download.addHeader("Authorization", "Bearer " + API_KEY); download.addHeader("ContentType", "application/json"); From 1298d0f6ff6af3c8547fb5f239921efb1284a70b Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:45:02 +0200 Subject: [PATCH 29/34] removes json module dependency --- src/main/java/module-info.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index bfb3a41fdcc..791e2e50698 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -65,5 +65,4 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; - requires json; } From 50be88b46721e5061dfbf651b06686ac87d32ef6 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:52:28 +0200 Subject: [PATCH 30/34] replace old fetcher with the new one --- .../jabref/logic/importer/WebFetchers.java | 4 - .../fetcher/AstrophysicsDataSystem.java | 2 + .../fetcher/NewAstrophysicsDataSystem.java | 263 ------------------ .../fetcher/AstrophysicsDataSystemTest.java | 2 +- .../NewAstrophysicsDataSystemTest.java | 201 ------------- 5 files changed, 3 insertions(+), 469 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java delete mode 100644 src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index 4161786d0c4..66319f5e6b4 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -27,7 +27,6 @@ import org.jabref.logic.importer.fetcher.LibraryOfCongress; import org.jabref.logic.importer.fetcher.MathSciNet; import org.jabref.logic.importer.fetcher.MedlineFetcher; -import org.jabref.logic.importer.fetcher.NewAstrophysicsDataSystem; import org.jabref.logic.importer.fetcher.OpenAccessDoi; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; @@ -89,7 +88,6 @@ public static SortedSet getSearchBasedFetchers(ImportFormatP set.add(new GvkFetcher()); set.add(new MedlineFetcher()); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); set.add(new ZbMATH(importFormatPreferences)); set.add(new ACMPortalFetcher(importFormatPreferences)); @@ -110,7 +108,6 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new ArXiv(importFormatPreferences)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new DiVA(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); @@ -130,7 +127,6 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) { SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); - set.add(new NewAstrophysicsDataSystem(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 0e1adceaa9a..70f81f095dd 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -22,6 +22,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.IdBasedParserFetcher; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; import org.jabref.logic.importer.fileformat.BibtexParser; @@ -30,6 +31,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java deleted file mode 100644 index 37dedfb6ca9..00000000000 --- a/src/main/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystem.java +++ /dev/null @@ -1,263 +0,0 @@ -package org.jabref.logic.importer.fetcher; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.jabref.logic.cleanup.MoveFieldCleanup; -import org.jabref.logic.formatter.bibtexfields.ClearFormatter; -import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter; -import org.jabref.logic.formatter.bibtexfields.RemoveBracesFormatter; -import org.jabref.logic.formatter.bibtexfields.RemoveNewlinesFormatter; -import org.jabref.logic.help.HelpFile; -import org.jabref.logic.importer.EntryBasedParserFetcher; -import org.jabref.logic.importer.FetcherException; -import org.jabref.logic.importer.IdBasedParserFetcher; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParseException; -import org.jabref.logic.importer.Parser; -import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.logic.net.URLDownload; -import org.jabref.model.cleanup.FieldFormatterCleanup; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.field.UnknownField; -import org.jabref.model.strings.StringUtil; -import org.jabref.model.util.DummyFileUpdateMonitor; - -import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -//TODO Replace Old ADS after the new one is mature - -/** - * Fetches data from the SAO/NASA Astrophysics Data System (http://www.adsabs.harvard.edu/) - *

- * Search query-based: http://adsabs.harvard.edu/basic_search.html Entry -based: http://adsabs.harvard.edu/abstract_service.html - *

- * There is also a new API (https://github.com/adsabs/adsabs-dev-api) but it returns JSON (or at least needs multiple - * calls to get BibTeX, status: September 2016) - */ -public class NewAstrophysicsDataSystem implements IdBasedParserFetcher, SearchBasedParserFetcher, EntryBasedParserFetcher { - - private static final String API_SEARCH_URL = "https://api.adsabs.harvard.edu/v1/search/query"; - private static final String API_EXPORT_URL = "https://api.adsabs.harvard.edu/v1/export/bibtexabs"; - - private static String API_KEY = ""; //TODO Add API Token - - private final ImportFormatPreferences preferences; - - public NewAstrophysicsDataSystem(ImportFormatPreferences preferences) { - this.preferences = Objects.requireNonNull(preferences); - } - - private String buildPostData(Collection bibcodes) { - JSONObject obj = new JSONObject(); - obj.put("bibcode", bibcodes); - return obj.toString(); - } - - private URL getURLforExport() throws URISyntaxException, MalformedURLException { - return new URIBuilder(API_EXPORT_URL).build().toURL(); - } - - @Override - public String getName() { - return "New SAO/NASA Astrophysics Data System"; - } - - @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - return builder.build().toURL(); - } - - @Override - public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedURLException, FetcherException { - StringBuilder stringBuilder = new StringBuilder(); - - Optional title = entry.getFieldOrAlias(StandardField.TITLE).map(t -> "title:\"" + t + "\""); - Optional author = entry.getFieldOrAlias(StandardField.AUTHOR).map(a -> "author:\"" + a + "\""); - - if (title.isPresent()) { - stringBuilder.append(title.get()) - .append(author.map(s -> " AND " + s) - .orElse("")); - } else { - stringBuilder.append(author.orElse("")); - } - String query = stringBuilder.toString().trim(); - - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - builder.addParameter("rows", "20"); - return builder.build().toURL(); - } - - @Override - public URL getURLForID(String identifier) throws FetcherException, URISyntaxException, MalformedURLException { - String query = "doi:\"" + identifier + "\" OR " + "bibcode:\"" + identifier + "\""; - URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); - builder.addParameter("fl", "bibcode"); - return builder.build().toURL(); - } - - @Override - public Optional getHelpPage() { - return Optional.of(HelpFile.FETCHER_ADS); - } - - @Override - public Parser getParser() { - return new BibtexParser(preferences, new DummyFileUpdateMonitor()); - } - - @Override - public void doPostCleanup(BibEntry entry) { - new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveBracesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.ABSTRACT, new RemoveNewlinesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); - new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); - - // Remove ADS note - new FieldFormatterCleanup(new UnknownField("adsnote"), new ClearFormatter()).cleanup(entry); - // Move adsurl to url field - new MoveFieldCleanup(new UnknownField("adsurl"), StandardField.URL).cleanup(entry); - // The fetcher adds some garbage (number of found entries etc before) - entry.setCommentsBeforeEntry(""); - } - - @Override - public List performSearch(BibEntry entry) throws FetcherException { - - if (entry.getFieldOrAlias(StandardField.TITLE).isEmpty() && entry.getFieldOrAlias(StandardField.AUTHOR).isEmpty()) { - return Collections.emptyList(); - } - - try { - List bibcodes = fetchBibcodes(getURLForEntry(entry)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - @Override - public List performSearch(String query) throws FetcherException { - - if (StringUtil.isBlank(query)) { - return Collections.emptyList(); - } - - try { - List bibcodes = fetchBibcodes(getURLForQuery(query)); - return performSearchByIds(bibcodes); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - private List fetchBibcodes(URL url) throws FetcherException { - - try { - URLDownload download = new URLDownload(url); - download.addHeader("Authorization", "Bearer " + API_KEY); - String content = download.asString(); - JSONObject obj = new JSONObject(content); - - try { - JSONArray codes = obj.getJSONObject("response").getJSONArray("docs"); - List bibcodes = new ArrayList<>(); - for (int i = 0; i < codes.length(); i++) { - bibcodes.add(codes.getJSONObject(i).getString("bibcode")); - } - return bibcodes; - } catch (JSONException e) { - return Collections.emptyList(); - } - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - @Override - public Optional performSearchById(String identifier) throws FetcherException { - if (StringUtil.isBlank(identifier)) { - return Optional.empty(); - } - - try { - List bibcodes = fetchBibcodes(getURLForID(identifier)); - List fetchedEntries = performSearchByIds(bibcodes); - - if (fetchedEntries.isEmpty()) { - return Optional.empty(); - } - if (fetchedEntries.size() > 1) { - LOGGER.info("Fetcher " + getName() + "found more than one result for identifier " + identifier - + ". We will use the first entry."); - } - BibEntry entry = fetchedEntries.get(0); - return Optional.of(entry); - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } - } - - private List performSearchByIds(Collection identifiers) throws FetcherException { - - List ids = identifiers.stream().filter(identifier -> !StringUtil.isBlank(identifier)).collect(Collectors.toList()); - if (ids.isEmpty()) { - return Collections.emptyList(); - } - try { - String postData = buildPostData(ids); - URLDownload download = new URLDownload(getURLforExport()); - download.addHeader("Authorization", "Bearer " + API_KEY); - download.addHeader("ContentType", "application/json"); - download.setPostData(postData); - String content = download.asString(); - JSONObject obj = new JSONObject(content); - - try { - List fetchedEntries = getParser().parseEntries(obj.optString("export")); - if (fetchedEntries.isEmpty()) { - return Collections.emptyList(); - } - // Post-cleanup - fetchedEntries.forEach(this::doPostCleanup); - - return fetchedEntries; - } catch (JSONException e) { - return Collections.emptyList(); - } - } catch (URISyntaxException e) { - throw new FetcherException("Search URI is malformed", e); - } catch (IOException e) { - throw new FetcherException("A network error occurred", e); - } catch (ParseException e) { - throw new FetcherException("An internal parser error occurred", e); - } - } -} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java index f3141019524..f7f393c49dd 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java @@ -136,7 +136,7 @@ public void testHelpPage() { @Test public void testGetName() { - assertEquals("SAO/NASA Astrophysics Data System", fetcher.getName()); + assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); } @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java deleted file mode 100644 index 403bfaefa55..00000000000 --- a/src/test/java/org/jabref/logic/importer/fetcher/NewAstrophysicsDataSystemTest.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.jabref.logic.importer.fetcher; - -import java.util.List; -import java.util.Optional; - -import org.jabref.logic.bibtex.FieldContentParserPreferences; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.testutils.category.FetcherTest; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@FetcherTest -public class NewAstrophysicsDataSystemTest { - - private NewAstrophysicsDataSystem fetcher; - private BibEntry diezSliceTheoremEntry, famaeyMcGaughEntry, sunWelchEntry, xiongSunEntry, ingersollPollardEntry, luceyPaulEntry; - - @BeforeEach - public void setUp() throws Exception { - ImportFormatPreferences importFormatPreferences = mock(ImportFormatPreferences.class); - when(importFormatPreferences.getFieldContentParserPreferences()).thenReturn( - mock(FieldContentParserPreferences.class)); - fetcher = new NewAstrophysicsDataSystem(importFormatPreferences); - - diezSliceTheoremEntry = new BibEntry(); - diezSliceTheoremEntry.setType(StandardEntryType.Article); - diezSliceTheoremEntry.setCiteKey("2018arXiv181204698D"); - diezSliceTheoremEntry.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); - diezSliceTheoremEntry.setField(StandardField.TITLE, "Slice theorem and orbit type stratification in infinite dimensions"); - diezSliceTheoremEntry.setField(StandardField.YEAR, "2018"); - diezSliceTheoremEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - diezSliceTheoremEntry.setField(StandardField.EPRINT, "1812.04698"); - diezSliceTheoremEntry.setField(StandardField.JOURNAL, "arXiv e-prints"); - diezSliceTheoremEntry.setField(StandardField.KEYWORDS, "Mathematics - Differential Geometry, Mathematical Physics, 58B25, (58D19, 58B20, 22E99, 58A35)"); - diezSliceTheoremEntry.setField(StandardField.MONTH, "Dec"); - diezSliceTheoremEntry.setField(StandardField.PAGES, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(StandardField.EID, "arXiv:1812.04698"); - diezSliceTheoremEntry.setField(StandardField.PRIMARYCLASS, "math.DG"); - diezSliceTheoremEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2018arXiv181204698D"); - diezSliceTheoremEntry.setField(StandardField.ABSTRACT, - "We establish a general slice theorem for the action of a locally convex Lie group on a locally convex manifold, which generalizes the classical slice theorem of Palais to infinite dimensions. We discuss two important settings under which the assumptions of this theorem are fulfilled. First, using Gl{\\\"o}ckner's inverse function theorem, we show that the linear action of a compact Lie group on a Fr{\\'e}chet space admits a slice. Second, using the Nash--Moser theorem, we establish a slice theorem for the tame action of a tame Fr{\\'e}chet Lie group on a tame Fr{\\'e}chet manifold. For this purpose, we develop the concept of a graded Riemannian metric, which allows the construction of a path-length metric compatible with the manifold topology and of a local addition. Finally, generalizing a classical result in finite dimensions, we prove that the existence of a slice implies that the decomposition of the manifold into orbit types of the group action is a stratification."); - - famaeyMcGaughEntry = new BibEntry(); - famaeyMcGaughEntry.setType(StandardEntryType.Article); - famaeyMcGaughEntry.setCiteKey("2012LRR....15...10F"); - famaeyMcGaughEntry.setField(StandardField.AUTHOR, "Famaey, Beno{\\^\\i}t and McGaugh, Stacy S."); - famaeyMcGaughEntry.setField(StandardField.TITLE, "Modified Newtonian Dynamics (MOND): Observational Phenomenology and Relativistic Extensions"); - famaeyMcGaughEntry.setField(StandardField.JOURNAL, "Living Reviews in Relativity"); - famaeyMcGaughEntry.setField(StandardField.YEAR, "2012"); - famaeyMcGaughEntry.setField(StandardField.VOLUME, "15"); - famaeyMcGaughEntry.setField(StandardField.MONTH, "Sep"); - famaeyMcGaughEntry.setField(StandardField.NUMBER, "1"); - famaeyMcGaughEntry.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - famaeyMcGaughEntry.setField(StandardField.DOI, "10.12942/lrr-2012-10"); - famaeyMcGaughEntry.setField(StandardField.PRIMARYCLASS, "astro-ph.CO"); - famaeyMcGaughEntry.setField(StandardField.EID, "10"); - famaeyMcGaughEntry.setField(StandardField.EPRINT, "1112.3960"); - famaeyMcGaughEntry.setField(StandardField.PAGES, "10"); - famaeyMcGaughEntry.setField(StandardField.KEYWORDS, "astronomical observations, Newtonian limit, equations of motion, extragalactic astronomy, cosmology, theories of gravity, fundamental physics, astrophysics, Astrophysics - Cosmology and Nongalactic Astrophysics, Astrophysics - Astrophysics of Galaxies, General Relativity and Quantum Cosmology, High Energy Physics - Phenomenology, High Energy Physics - Theory"); - famaeyMcGaughEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012LRR....15...10F"); - - sunWelchEntry = new BibEntry(); - sunWelchEntry.setType(StandardEntryType.Article); - sunWelchEntry.setCiteKey("2012NatMa..11...44S"); - sunWelchEntry.setField(StandardField.AUTHOR, "Sun, Yanming and Welch, Gregory C. and Leong, Wei Lin and Takacs, Christopher J. and Bazan, Guillermo C. and Heeger, Alan J."); - sunWelchEntry.setField(StandardField.DOI, "10.1038/nmat3160"); - sunWelchEntry.setField(StandardField.JOURNAL, "Nature Materials"); - sunWelchEntry.setField(StandardField.MONTH, "Jan"); - sunWelchEntry.setField(StandardField.NUMBER, "1"); - sunWelchEntry.setField(StandardField.PAGES, "44-48"); - sunWelchEntry.setField(StandardField.TITLE, "Solution-processed small-molecule solar cells with 6.7\\% efficiency"); - sunWelchEntry.setField(StandardField.VOLUME, "11"); - sunWelchEntry.setField(StandardField.YEAR, "2012"); - sunWelchEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2012NatMa..11...44S"); - - xiongSunEntry = new BibEntry(); - xiongSunEntry.setType(StandardEntryType.Article); - xiongSunEntry.setCiteKey("2007ITGRS..45..879X"); - xiongSunEntry.setField(StandardField.AUTHOR, "Xiong, Xiaoxiong and Sun, Junqiang and Barnes, William and Salomonson, Vincent and Esposito, Joseph and Erives, Hector and Guenther, Bruce"); - xiongSunEntry.setField(StandardField.DOI, "10.1109/TGRS.2006.890567"); - xiongSunEntry.setField(StandardField.JOURNAL, "IEEE Transactions on Geoscience and Remote Sensing"); - xiongSunEntry.setField(StandardField.MONTH, "Apr"); - xiongSunEntry.setField(StandardField.NUMBER, "4"); - xiongSunEntry.setField(StandardField.PAGES, "879-889"); - xiongSunEntry.setField(StandardField.TITLE, "Multiyear On-Orbit Calibration and Performance of Terra MODIS Reflective Solar Bands"); - xiongSunEntry.setField(StandardField.VOLUME, "45"); - xiongSunEntry.setField(StandardField.YEAR, "2007"); - xiongSunEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2007ITGRS..45..879X"); - - ingersollPollardEntry = new BibEntry(); - ingersollPollardEntry.setType(StandardEntryType.Article); - ingersollPollardEntry.setCiteKey("1982Icar...52...62I"); - ingersollPollardEntry.setField(StandardField.ABSTRACT, "If Jupiter's and Saturn's fluid interiors were inviscid and adiabatic, any steady zonal motion would take the form of differentially rotating cylinders concentric about the planetary axis of rotation. B. A. Smith et al. [ Science215, 504-537 (1982)] showed that Saturn's observed zonal wind profile extends a significant distance below cloud base. Further extension into the interior occurs if the values of the eddy viscosity and superadiabaticity are small. We estimate these values using a scaling analysis of deep convection in the presence of differential rotation. The differential rotation inhibits the convection and reduces the effective eddy viscosity. Viscous dissipation of zonal mean kinetic energy is then within the bounds set by the internal heat source. The differential rotation increases the superadiabaticity, but not so much as to eliminate the cylindrical structure of the flow. Very large departures from adiabaticity, necessary for decoupling the atmosphere and interior, do not occur. Using our scaling analysis we develop the anelastic equations that describe motions in Jupiter's and Saturn's interiors. A simple problem is solved, that of an adiabatic fluid with a steady zonal wind varying as a function of cylindrical radius. Low zonal wavenumber perturbations are two dimensional (independent of the axial coordinate) and obey a modified barotropic stability equation. The parameter analogous to {\\ensuremath{\\beta}} is negative and is three to four times larger than the {\\ensuremath{\\beta}} for thin atmospheres. Jupiter's and Saturn's observed zonal wind profiles are close to marginal stability according to this deep sphere criterion, but are several times supercritical according to the thin atmosphere criterion."); - ingersollPollardEntry.setField(StandardField.AUTHOR, "Ingersoll, A. P. and Pollard, D."); - ingersollPollardEntry.setField(StandardField.DOI, "10.1016/0019-1035(82)90169-5"); - ingersollPollardEntry.setField(StandardField.JOURNAL, "\\icarus"); - ingersollPollardEntry.setField(StandardField.KEYWORDS, "Atmospheric Circulation, Barotropic Flow, Convective Flow, Flow Stability, Jupiter Atmosphere, Rotating Fluids, Saturn Atmosphere, Adiabatic Flow, Anelasticity, Compressible Fluids, Planetary Rotation, Rotating Cylinders, Scaling Laws, Wind Profiles, PLANETS, JUPITER, SATURN, MOTION, INTERIORS, ATMOSPHERE, ANALYSIS, SCALE, BAROTROPY, CHARACTERISTICS, STRUCTURE, WINDS, VISCOSITY, DATA, CONVECTION, ROTATION, EDDY EFFECTS, ENERGY, ADIABATICITY, DIAGRAMS, REVIEW, LATITUDE, ZONES, VELOCITY, MATHEMATICAL MODELS, HEAT FLOW, EQUATIONS OF MOTION, FLUIDS, DYNAMICS, TEMPERATURE, GRADIENTS, Lunar and Planetary Exploration; Planets"); - ingersollPollardEntry.setField(StandardField.MONTH, "Oct"); - ingersollPollardEntry.setField(StandardField.NUMBER, "1"); - ingersollPollardEntry.setField(StandardField.PAGES, "62-80"); - ingersollPollardEntry.setField(StandardField.TITLE, "Motion in the interiors and atmospheres of Jupiter and Saturn: scale analysis, anelastic equations, barotropic stability criterion"); - ingersollPollardEntry.setField(StandardField.VOLUME, "52"); - ingersollPollardEntry.setField(StandardField.YEAR, "1982"); - ingersollPollardEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/1982Icar...52...62I"); - - luceyPaulEntry = new BibEntry(); - luceyPaulEntry.setType(StandardEntryType.Article); - luceyPaulEntry.setCiteKey("2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.AUTHOR, "Lucey, Paul G. and Blewett, David T. and Jolliff, Bradley L."); - luceyPaulEntry.setField(StandardField.DOI, "10.1029/1999JE001117"); - luceyPaulEntry.setField(StandardField.JOURNAL, "\\jgr"); - luceyPaulEntry.setField(StandardField.KEYWORDS, "Planetology: Solid Surface Planets: Composition, Planetology: Solid Surface Planets: Remote sensing, Planetology: Solid Surface Planets: Surface materials and properties, Planetology: Solar System Objects: Moon (1221)"); - luceyPaulEntry.setField(StandardField.PAGES, "20297-20306"); - luceyPaulEntry.setField(StandardField.TITLE, "Lunar iron and titanium abundance algorithms based on final processing of Clementine ultraviolet-visible images"); - luceyPaulEntry.setField(StandardField.VOLUME, "105"); - luceyPaulEntry.setField(StandardField.YEAR, "2000"); - luceyPaulEntry.setField(StandardField.URL, "https://ui.adsabs.harvard.edu/abs/2000JGR...10520297L"); - luceyPaulEntry.setField(StandardField.MONTH, "Jan"); - luceyPaulEntry.setField(StandardField.NUMBER, "E8"); - } - - @Test - public void testHelpPage() { - assertEquals("ADS", fetcher.getHelpPage().get().getPageName()); - } - - @Test - public void testGetName() { - assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); - } - - @Test - public void searchByQueryFindsEntry() throws Exception { - List fetchedEntries = fetcher.performSearch("Diez slice theorem Lie"); - assertTrue(fetchedEntries.contains(diezSliceTheoremEntry)); - } - - @Test - public void searchByEntryFindsEntry() throws Exception { - BibEntry searchEntry = new BibEntry(); - searchEntry.setField(StandardField.TITLE, "slice theorem"); - searchEntry.setField(StandardField.AUTHOR, "Diez"); - - List fetchedEntries = fetcher.performSearch(searchEntry); - assertFalse(fetchedEntries.isEmpty()); - assertEquals(diezSliceTheoremEntry, fetchedEntries.get(0)); - } - - @Test - public void testPerformSearchByFamaeyMcGaughEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.12942/lrr-2012-10"); - fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT));//Remove abstract due to copyright - assertEquals(Optional.of(famaeyMcGaughEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByIdEmptyDOI() throws Exception { - Optional fetchedEntry = fetcher.performSearchById(""); - assertEquals(Optional.empty(), fetchedEntry); - } - - @Test - public void testPerformSearchByIdInvalidDoi() throws Exception { - assertEquals(Optional.empty(), fetcher.performSearchById("this.doi.will.fail")); - } - - @Test - public void testPerformSearchBySunWelchEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1038/nmat3160"); - fetchedEntry.ifPresent(entry -> entry.clearField(StandardField.ABSTRACT)); //Remove abstract due to copyright - assertEquals(Optional.of(sunWelchEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByXiongSunEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1109/TGRS.2006.890567"); - assertEquals(Optional.of(xiongSunEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByIngersollPollardEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("10.1016/0019-1035(82)90169-5"); - assertEquals(Optional.of(ingersollPollardEntry), fetchedEntry); - } - - @Test - public void testPerformSearchByLuceyPaulEntry() throws Exception { - Optional fetchedEntry = fetcher.performSearchById("2000JGR...10520297L"); - assertEquals(Optional.of(luceyPaulEntry), fetchedEntry); - } -} From 972b02580f85910064eeaca18ab13c8f8ec69435 Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 17:57:47 +0200 Subject: [PATCH 31/34] replace old fetcher with the new one --- .../logic/importer/fetcher/AstrophysicsDataSystemTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java index f7f393c49dd..f3141019524 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystemTest.java @@ -136,7 +136,7 @@ public void testHelpPage() { @Test public void testGetName() { - assertEquals("New SAO/NASA Astrophysics Data System", fetcher.getName()); + assertEquals("SAO/NASA Astrophysics Data System", fetcher.getName()); } @Test From 6a948541160fc6247146ee32eb767792a41a00bb Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Thu, 24 Oct 2019 22:00:07 +0200 Subject: [PATCH 32/34] updated unirest version and removes org.json dependency resolves #3703 --- build.gradle | 7 +++- external-libraries.txt | 4 +- src/main/java/module-info.java | 3 +- src/main/java/org/jabref/Globals.java | 7 +--- .../fetcher/AstrophysicsDataSystem.java | 6 +-- .../logic/importer/fetcher/CrossRef.java | 6 +-- .../logic/importer/fetcher/DOAJFetcher.java | 42 +++++++++---------- .../jabref/logic/importer/fetcher/IEEE.java | 4 +- .../fetcher/IsbnViaChimboriFetcher.java | 19 +++++---- .../logic/importer/fetcher/OpenAccessDoi.java | 14 +++---- .../logic/importer/fetcher/ScienceDirect.java | 28 ++++++------- .../importer/fetcher/SpringerFetcher.java | 4 +- .../logic/importer/fetcher/SpringerLink.java | 16 +++---- .../importer/fileformat/MrDLibImporter.java | 4 +- .../logic/importer/util/JsonReader.java | 2 +- .../logic/importer/util/ShortDOIService.java | 6 +-- .../org/jabref/logic/net/URLDownload.java | 4 +- .../java/org/jabref/logic/util/Version.java | 4 +- .../importer/fetcher/DOAJFetcherTest.java | 2 +- .../importer/fetcher/SpringerFetcherTest.java | 2 +- 20 files changed, 94 insertions(+), 90 deletions(-) diff --git a/build.gradle b/build.gradle index 1b792794bb0..aea88b515e2 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,12 @@ dependencies { compile 'org.controlsfx:controlsfx:11.0.0' compile 'org.jsoup:jsoup:1.12.1' - compile 'com.mashape.unirest:unirest-java:1.4.9' + compile ('com.konghq:unirest-java:3.1.02'){ + // Use lib directly from maven, because current unirest build uses an older build of this library. + // There has been a bugfix for the automatic name manifest property of httpcore-nio this library depends on. + exclude module: "httpcore-nio" + } + compile ('org.apache.httpcomponents:httpcore-nio:4.4.12') compile 'org.slf4j:slf4j-api:2.0.0-alpha1' compile group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '3.0.0-SNAPSHOT' diff --git a/external-libraries.txt b/external-libraries.txt index 385d4d18217..996d126c839 100644 --- a/external-libraries.txt +++ b/external-libraries.txt @@ -50,9 +50,9 @@ Project: pgjdbc-ng URL: http://impossibl.github.io/pgjdbc-ng License: BSD-3-Clause -Id: com.mashape.unirest +Id: com.konghq.unirest Project: Unirest for Java -URL: https://github.com/Mashape/unirest-java +URL: https://github.com/Kong/unirest-java License: MIT Id: com.microsoft.azure:applicationinsights-core diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 791e2e50698..327ce26d480 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -55,7 +55,6 @@ requires org.apache.pdfbox; requires reactfx; requires commons.cli; - requires httpclient; requires com.github.tomtung.latex2unicode; requires jbibtex; requires citeproc.java; @@ -65,4 +64,6 @@ requires de.saxsys.mvvmfx.validation; requires richtextfx; requires unirest.java; + requires org.apache.httpcomponents.httpclient; + requires org.jsoup; } diff --git a/src/main/java/org/jabref/Globals.java b/src/main/java/org/jabref/Globals.java index f65daeb27e6..bf5e00d33e7 100644 --- a/src/main/java/org/jabref/Globals.java +++ b/src/main/java/org/jabref/Globals.java @@ -1,7 +1,6 @@ package org.jabref; import java.awt.GraphicsEnvironment; -import java.io.IOException; import java.util.Optional; import java.util.UUID; @@ -26,11 +25,11 @@ import org.jabref.preferences.JabRefPreferences; import com.google.common.base.StandardSystemProperty; -import com.mashape.unirest.http.Unirest; import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.TelemetryConfiguration; import com.microsoft.applicationinsights.internal.shutdown.SDKShutdownActivity; import com.microsoft.applicationinsights.telemetry.SessionState; +import kong.unirest.Unirest; public class Globals { @@ -127,9 +126,7 @@ public static void shutdownThreadPools() { public static void stopBackgroundTasks() { stopTelemetryClient(); - try { - Unirest.shutdown(); - } catch (IOException ignore) { } + Unirest.shutDown(); } public static Optional getTelemetryClient() { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 70f81f095dd..fcf2750dd79 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -34,10 +34,10 @@ import org.jabref.model.strings.StringUtil; import org.jabref.model.util.DummyFileUpdateMonitor; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; /** * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 50e9da651ac..aca7a6b3397 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -27,10 +27,10 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.util.OptionalUtil; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; /** * A class for fetching DOIs from CrossRef diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 074c47cff62..96f9d4f4140 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -23,9 +23,9 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.strings.StringUtil; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -153,25 +153,6 @@ public static BibEntry parseBibJSONtoBibtex(JSONObject bibJsonEntry, Character k return entry; } - @Override - public String getName() { - return "DOAJ"; - } - - @Override - public Optional getHelpPage() { - return Optional.of(HelpFile.FETCHER_DOAJ); - } - - @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, query); - uriBuilder.addParameter("pageSize", "30"); // Number of results - //uriBuilder.addParameter("page", "1"); // Page (not needed so far) - return uriBuilder.build().toURL(); - } - /** * @implNote slightly altered version based on https://gist.github.com/enginer/230e2dc2f1d213a825d5 */ @@ -196,6 +177,25 @@ private static String appendSegmentToPath(String path, String segment) { return path + "/" + segment; } + @Override + public String getName() { + return "DOAJ"; + } + + @Override + public Optional getHelpPage() { + return Optional.of(HelpFile.FETCHER_DOAJ); + } + + @Override + public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); + DOAJFetcher.addPath(uriBuilder, query); + uriBuilder.addParameter("pageSize", "30"); // Number of results + //uriBuilder.addParameter("page", "1"); // Page (not needed so far) + return uriBuilder.build().toURL(); + } + @Override public Parser getParser() { return inputStream -> { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index 79b03b83d71..0ffd58376ae 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -28,9 +28,9 @@ import org.jabref.model.entry.identifier.DOI; import org.jabref.model.entry.types.StandardEntryType; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java index e3f84536f86..068f039c8b6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IsbnViaChimboriFetcher.java @@ -13,14 +13,15 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.strings.StringUtil; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; +import kong.unirest.RawResponse; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; /** * Fetcher for ISBN using https://bibtex.chimbori.com/, which in turn uses Amazon's API. */ public class IsbnViaChimboriFetcher extends AbstractIsbnFetcher { + private RawResponse postResponse; public IsbnViaChimboriFetcher(ImportFormatPreferences importFormatPreferences) { super(importFormatPreferences); @@ -47,21 +48,21 @@ public Optional performSearchById(String identifier) throws FetcherExc this.ensureThatIsbnIsValid(identifier); - HttpResponse postResponse; + postResponse = null; try { - postResponse = Unirest.post("https://bibtex.chimbori.com/isbn-bibtex") - .field("isbn", identifier) - .asString(); + Unirest.post("https://bibtex.chimbori.com/isbn-bibtex") + .field("isbn", identifier) + .thenConsume(rawResponse -> postResponse = rawResponse); } catch (UnirestException e) { throw new FetcherException("Could not retrieve data from chimbori.com", e); } if (postResponse.getStatus() != 200) { - throw new FetcherException("Error while retrieving data from chimbori.com: " + postResponse.getBody()); + throw new FetcherException("Error while retrieving data from chimbori.com: " + postResponse.getContentAsString()); } List fetchedEntries; try { - fetchedEntries = getParser().parseEntries(postResponse.getRawBody()); + fetchedEntries = getParser().parseEntries(postResponse.getContent()); } catch (ParseException e) { throw new FetcherException("An internal parser error occurred", e); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java b/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java index 64e3ab046e6..7bb7a5125e1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/OpenAccessDoi.java @@ -11,11 +11,11 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.JsonNode; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import org.json.JSONObject; +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; +import kong.unirest.json.JSONObject; /** * A fulltext fetcher that uses oaDOI. @@ -49,8 +49,8 @@ public TrustLevel getTrustLevel() { public Optional findFullText(DOI doi) throws UnirestException, MalformedURLException { HttpResponse jsonResponse = Unirest.get(API_URL + doi.getDOI() + "?email=developers@jabref.org") - .header("accept", "application/json") - .asJson(); + .header("accept", "application/json") + .asJson(); JSONObject root = jsonResponse.getBody().getObject(); Optional url = Optional.ofNullable(root.optJSONObject("best_oa_location")) .map(location -> location.optString("url")); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java b/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java index 4b66f6d49e4..56e27018273 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ScienceDirect.java @@ -11,13 +11,13 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.JsonNode; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -28,7 +28,7 @@ /** * FulltextFetcher implementation that attempts to find a PDF URL at ScienceDirect. * - * @see http://dev.elsevier.com/ + * @see 'https://dev.elsevier.com/' */ public class ScienceDirect implements FulltextFetcher { private static final Logger LOGGER = LoggerFactory.getLogger(ScienceDirect.class); @@ -50,9 +50,9 @@ public Optional findFullText(BibEntry entry) throws IOException { // scrape the web page not as mobile client! if (!sciLink.isEmpty()) { Document html = Jsoup.connect(sciLink) - .userAgent(URLDownload.USER_AGENT) - .referrer("http://www.google.com") - .ignoreHttpErrors(true).get(); + .userAgent(URLDownload.USER_AGENT) + .referrer("http://www.google.com") + .ignoreHttpErrors(true).get(); // Retrieve PDF link from meta data (most recent) Elements metaLinks = html.getElementsByAttributeValue("name", "citation_pdf_url"); @@ -98,9 +98,9 @@ private String getUrlByDoi(String doi) throws UnirestException { try { String request = API_URL + doi; HttpResponse jsonResponse = Unirest.get(request) - .header("X-ELS-APIKey", API_KEY) - .queryString("httpAccept", "application/json") - .asJson(); + .header("X-ELS-APIKey", API_KEY) + .queryString("httpAccept", "application/json") + .asJson(); JSONObject json = jsonResponse.getBody().getObject(); JSONArray links = json.getJSONObject("full-text-retrieval-response").getJSONObject("coredata").getJSONArray("link"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index fd38b2e61b1..8d9221709a0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -22,9 +22,9 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java index 73722330ccd..28f2357efc5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerLink.java @@ -10,11 +10,11 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.identifier.DOI; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.JsonNode; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.http.exceptions.UnirestException; -import org.json.JSONObject; +import kong.unirest.HttpResponse; +import kong.unirest.JsonNode; +import kong.unirest.Unirest; +import kong.unirest.UnirestException; +import kong.unirest.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,9 +42,9 @@ public Optional findFullText(BibEntry entry) throws IOException { // Available in catalog? try { HttpResponse jsonResponse = Unirest.get(API_URL) - .queryString("api_key", API_KEY) - .queryString("q", String.format("doi:%s", doi.get().getDOI())) - .asJson(); + .queryString("api_key", API_KEY) + .queryString("q", String.format("doi:%s", doi.get().getDOI())) + .asJson(); JSONObject json = jsonResponse.getBody().getObject(); int results = json.getJSONArray("result").getJSONObject(0).getInt("total"); diff --git a/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java index 659ee8da831..37c7676307b 100644 --- a/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java +++ b/src/main/java/org/jabref/logic/importer/fileformat/MrDLibImporter.java @@ -17,8 +17,8 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.json.JSONException; -import org.json.JSONObject; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/jabref/logic/importer/util/JsonReader.java b/src/main/java/org/jabref/logic/importer/util/JsonReader.java index ea46bb7f756..7bfd02986b1 100644 --- a/src/main/java/org/jabref/logic/importer/util/JsonReader.java +++ b/src/main/java/org/jabref/logic/importer/util/JsonReader.java @@ -8,7 +8,7 @@ import org.jabref.logic.importer.ParseException; -import org.json.JSONObject; +import kong.unirest.json.JSONObject; /** * Converts an {@link InputStream} into a {@link JSONObject}. diff --git a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java index bbb3dc7b4c6..a74745f0021 100644 --- a/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java +++ b/src/main/java/org/jabref/logic/importer/util/ShortDOIService.java @@ -10,14 +10,14 @@ import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.identifier.DOI; +import kong.unirest.json.JSONException; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONException; -import org.json.JSONObject; /** * Class for obtaining shortened DOI names. * - * @see http://shortdoi.org + * @see 'https://shortdoi.org' */ public class ShortDOIService { diff --git a/src/main/java/org/jabref/logic/net/URLDownload.java b/src/main/java/org/jabref/logic/net/URLDownload.java index 0eb2508cf13..34fc9537f51 100644 --- a/src/main/java/org/jabref/logic/net/URLDownload.java +++ b/src/main/java/org/jabref/logic/net/URLDownload.java @@ -41,7 +41,7 @@ import org.jabref.logic.util.io.FileUtil; import org.jabref.model.util.FileHelper; -import com.mashape.unirest.http.Unirest; +import kong.unirest.Unirest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -131,7 +131,7 @@ public URL getSource() { } public String getMimeType() { - Unirest.setDefaultHeader("User-Agent", "Mozilla/5.0 (Windows; U; WindowsNT 5.1; en-US; rv1.8.1.6) Gecko/20070725 Firefox/2.0.0.6"); + Unirest.config().setDefaultHeader("User-Agent", "Mozilla/5.0 (Windows; U; WindowsNT 5.1; en-US; rv1.8.1.6) Gecko/20070725 Firefox/2.0.0.6"); String contentType; // Try to use HEAD request to avoid downloading the whole file diff --git a/src/main/java/org/jabref/logic/util/Version.java b/src/main/java/org/jabref/logic/util/Version.java index e288c718064..0775fd6a0fb 100644 --- a/src/main/java/org/jabref/logic/util/Version.java +++ b/src/main/java/org/jabref/logic/util/Version.java @@ -12,8 +12,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.json.JSONArray; -import org.json.JSONObject; +import kong.unirest.json.JSONArray; +import kong.unirest.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java index a5c81f98570..68ea36fafa0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java @@ -10,8 +10,8 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; +import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; -import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java index 063b561a4cf..c4db21ec256 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java @@ -9,7 +9,7 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; -import org.json.JSONObject; +import kong.unirest.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; From eecbb0e7ca020ed5fe871e42584a0edf86cb422f Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Fri, 25 Oct 2019 11:34:47 +0200 Subject: [PATCH 33/34] updated unirest version and removes org.json dependency resolves #3703 [dependency fix in unirest](https://github.com/Kong/unirest-java/issues/311) --- build.gradle | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index aea88b515e2..4341779dc8d 100644 --- a/build.gradle +++ b/build.gradle @@ -159,12 +159,7 @@ dependencies { compile 'org.controlsfx:controlsfx:11.0.0' compile 'org.jsoup:jsoup:1.12.1' - compile ('com.konghq:unirest-java:3.1.02'){ - // Use lib directly from maven, because current unirest build uses an older build of this library. - // There has been a bugfix for the automatic name manifest property of httpcore-nio this library depends on. - exclude module: "httpcore-nio" - } - compile ('org.apache.httpcomponents:httpcore-nio:4.4.12') + compile ('com.konghq:unirest-java:3.1.03') compile 'org.slf4j:slf4j-api:2.0.0-alpha1' compile group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '3.0.0-SNAPSHOT' From 0d9fa0a461efa532b69e7e12dc656bb065b09dce Mon Sep 17 00:00:00 2001 From: Johannes Hupe Date: Fri, 25 Oct 2019 11:47:15 +0200 Subject: [PATCH 34/34] updated unirest version and removes org.json dependency resolves #3703 [dependency fix in unirest](https://github.com/Kong/unirest-java/issues/311) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 4341779dc8d..ee78e564bd3 100644 --- a/build.gradle +++ b/build.gradle @@ -159,7 +159,7 @@ dependencies { compile 'org.controlsfx:controlsfx:11.0.0' compile 'org.jsoup:jsoup:1.12.1' - compile ('com.konghq:unirest-java:3.1.03') + compile 'com.konghq:unirest-java:3.1.03' compile 'org.slf4j:slf4j-api:2.0.0-alpha1' compile group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '3.0.0-SNAPSHOT'