new URI instead of new URL.

RipMeApp · Mar 21, 2024 · 9cdfe51 · 9cdfe51
1 parent c2d1472
commit 9cdfe51
Show file tree

Hide file tree

Showing 17 changed files with 84 additions and 90 deletions.
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ArtStationRipper.java
@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -50,7 +52,7 @@ public String getGID(URL url) throws MalformedURLException {
             try {
                 // groupData = Http.url(albumURL.getLocation()).getJSON();
                 groupData = getJson(albumURL.getLocation());
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 throw new MalformedURLException("Couldn't load JSON from " + albumURL.getLocation());
             }
             return groupData.getString("title");
@@ -62,7 +64,7 @@ public String getGID(URL url) throws MalformedURLException {
             try {
 //                 groupData = Http.url(userInfoURL).getJSON();
                 groupData = getJson(userInfoURL);
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 throw new MalformedURLException("Couldn't load JSON from " + userInfoURL);
             }
             return groupData.getString("full_name");
@@ -74,7 +76,7 @@ public String getGID(URL url) throws MalformedURLException {
     }
 
     @Override
-    protected JSONObject getFirstPage() throws IOException {
+    protected JSONObject getFirstPage() throws IOException, URISyntaxException {
         if (albumURL.getType() == URL_TYPE.SINGLE_PROJECT) {
             // URL points to JSON of a single project, just return it
             // return Http.url(albumURL.getLocation()).getJSON();
@@ -90,7 +92,7 @@ protected JSONObject getFirstPage() throws IOException {
             if (albumContent.getInt("total_count") > 0) {
                 // Get JSON of the first project and return it
                 JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(0);
-                ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
+                ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL());
                 // return Http.url(projectURL.getLocation()).getJSON();
                 return getJson(projectURL.getLocation());
             }
@@ -100,7 +102,7 @@ protected JSONObject getFirstPage() throws IOException {
     }
 
     @Override
-    protected JSONObject getNextPage(JSONObject doc) throws IOException {
+    protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
         if (albumURL.getType() == URL_TYPE.USER_PORTFOLIO) {
             // Initialize the page number if it hasn't been initialized already
             if (projectPageNumber == null) {
@@ -117,15 +119,15 @@ protected JSONObject getNextPage(JSONObject doc) throws IOException {
                 projectIndex = 0;
             }
 
-            Integer currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
+            int currentProject = ((projectPageNumber - 1) * 50) + (projectIndex + 1);
             // JSONObject albumContent = Http.url(albumURL.getLocation() + "?page=" +
             // projectPageNumber).getJSON();
             JSONObject albumContent = getJson(albumURL.getLocation() + "?page=" + projectPageNumber);
 
             if (albumContent.getInt("total_count") > currentProject) {
                 // Get JSON of the next project and return it
                 JSONObject projectInfo = albumContent.getJSONArray("data").getJSONObject(projectIndex);
-                ParsedURL projectURL = parseURL(new URL(projectInfo.getString("permalink")));
+                ParsedURL projectURL = parseURL(new URI(projectInfo.getString("permalink")).toURL());
                 projectIndex++;
                 // return Http.url(projectURL.getLocation()).getJSON();
                 return getJson(projectURL.getLocation());
@@ -320,8 +322,8 @@ private JSONObject getJson(URL url) throws IOException {
         throw new IOException("Error fetching json. Status code:" + status);
     }
 
-    private JSONObject getJson(String url) throws IOException {
-        return getJson(new URL(url));
+    private JSONObject getJson(String url) throws IOException, URISyntaxException {
+        return getJson(new URI(url).toURL());
     }
 
 }
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -72,7 +72,7 @@ public static List<ChanSite> getChansFromConfig(String rawChanString) {
         );
 
     private ChanSite chanSite;
-    private Boolean generalChanSite = true;
+    private boolean generalChanSite = true;
 
     public ChanRipper(URL url) throws IOException {
         super(url);

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java
@@ -94,13 +94,6 @@ public String getGID(URL url) throws MalformedURLException {
                         + " Got: " + url);
     }
 
-    /**
-     * Attempts to get page, checks for IP ban, waits.
-     *
-     * @param url
-     * @return Page document
-     * @throws IOException If page loading errors, or if retries are exhausted
-     */
     private Document getPageWithRetries(URL url) throws IOException {
         Document doc;
         int retries = 3;
@@ -251,16 +244,16 @@ private void fetchImage() {
                         savePath += String.format("%03d_", index);
                     }
                     savePath += m.group(1);
-                    addURLToDownload(new URL(imgsrc), Paths.get(savePath));
+                    addURLToDownload(new URI(imgsrc).toURL(), Paths.get(savePath));
                 } else {
                     // Provide prefix and let the AbstractRipper "guess" the filename
                     String prefix = "";
                     if (Utils.getConfigBoolean("download.save_order", true)) {
                         prefix = String.format("%03d_", index);
                     }
-                    addURLToDownload(new URL(imgsrc), prefix);
+                    addURLToDownload(new URI(imgsrc).toURL(), prefix);
                 }
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 LOGGER.error("[!] Exception while loading/parsing " + this.url, e);
             }
         }

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java
@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -85,8 +86,8 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce
     }
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
-        return new URL(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com"));
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
+        return new URI(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com")).toURL();
     }
 
 

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
@@ -1,9 +1,7 @@
 package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.*;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
@@ -164,8 +162,8 @@ private String getUserID(String username) throws IOException {
     }
 
     @Override
-    public JSONObject getFirstPage() throws IOException {
-        URL apiURL = new URL(baseURL + "&consumer_key=" + CONSUMER_KEY);
+    public JSONObject getFirstPage() throws IOException, URISyntaxException {
+        URL apiURL = new URI(baseURL + "&consumer_key=" + CONSUMER_KEY).toURL();
         LOGGER.debug("apiURL: " + apiURL);
         JSONObject json = Http.url(apiURL).getJSON();
 
@@ -232,7 +230,7 @@ else if (baseURL.contains("/blogs?")) {
     }
 
     @Override
-    public JSONObject getNextPage(JSONObject json) throws IOException {
+    public JSONObject getNextPage(JSONObject json) throws IOException, URISyntaxException {
         if (isThisATest()) {
             return null;
         }
@@ -249,9 +247,9 @@ public JSONObject getNextPage(JSONObject json) throws IOException {
 
         sleep(500);
         ++page;
-        URL apiURL = new URL(baseURL
+        URL apiURL = new URI(baseURL
                              + "&page=" + page
-                             + "&consumer_key=" + CONSUMER_KEY);
+                             + "&consumer_key=" + CONSUMER_KEY).toURL();
         return Http.url(apiURL).getJSON();
     }
 
@@ -296,28 +294,23 @@ public List<String> getURLsFromJSON(JSONObject json) {
                     }
                 }
             }
-            if (imageURL == null) {
-                LOGGER.error("Failed to find image for photo " + photo.toString());
-            }
-            else {
-                imageURLs.add(imageURL);
-                if (isThisATest()) {
-                    break;
-                }
+            imageURLs.add(imageURL);
+            if (isThisATest()) {
+                break;
             }
         }
         return imageURLs;
     }
 
     private boolean urlExists(String url) {
         try {
-            HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
+            HttpURLConnection connection = (HttpURLConnection) new URI(url).toURL().openConnection();
             connection.setRequestMethod("HEAD");
             if (connection.getResponseCode() != 200) {
                 throw new IOException("Couldn't find full-size image at " + url);
             }
             return true;
-        } catch (IOException e) {
+        } catch (IOException | URISyntaxException e) {
             return false;
         }
     }

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java
@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -40,15 +42,15 @@ public String getDomain() {
     }
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
         String u = url.toExternalForm();
         if (u.contains("/thumbs/")) {
             u = u.replace("/thumbs/", "/full/");
         }
         if (u.contains("/expanded/")) {
             u = u.replaceAll("/expanded/", "/full/");
         }
-        return new URL(u);
+        return new URI(u).toURL();
     }
 
     @Override

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java
@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Base64;
@@ -80,16 +82,15 @@ protected List<String> getURLsFromJSON(JSONObject json) throws JSONException {
     }
 
     @Override
-    protected JSONObject getFirstPage() throws IOException {
+    protected JSONObject getFirstPage() throws IOException, URISyntaxException {
         String jsonEncodedString = getJsonEncodedStringFromPage();
         String jsonDecodedString = decodeJsonString(jsonEncodedString);
         return new JSONObject(jsonDecodedString);
     }
 
-    public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException
-    {
+    public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException, URISyntaxException {
         // Image data only appears on the /read/ page and not on the /view/ one.
-        URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url)));
+        URL readUrl = new URI(String.format("http://hentainexus.com/read/%s",getGID(url))).toURL();
         Document document = Http.url(readUrl).response().parse();
 
         for (Element scripts : document.getElementsByTag("script")) {
@@ -143,7 +144,7 @@ The following code is a Java adaptation of the initRender() JavaScript function
         }
 
         magicByte = (byte) (magicByte & 0x7);
-        ArrayList<Integer> newArray = new ArrayList();
+        ArrayList<Integer> newArray = new ArrayList<>();
 
         for (int i = 0x0; i < 0x100; i++) {
             newArray.add(i);

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -14,7 +15,6 @@
 
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
-import org.jsoup.nodes.Element;
 
 public class HitomiRipper extends AbstractHTMLRipper {
 
@@ -47,9 +47,9 @@ public String getGID(URL url) throws MalformedURLException {
     }
 
     @Override
-    public Document getFirstPage() throws IOException {
+    public Document getFirstPage() throws IOException, URISyntaxException {
         // if we go to /GALLERYID.js we get a nice json array of all images in the gallery
-        return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get();
+        return Http.url(new URI(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js")).toURL()).ignoreContentType().get();
     }
 
 

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java
@@ -8,6 +8,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -34,7 +36,7 @@ public String getHost() {
 
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
         String u = url.toExternalForm();
         // https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat
         // https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat
@@ -44,15 +46,15 @@ public URL sanitizeURL(URL url) throws MalformedURLException {
             throw new MalformedURLException("Invalid URL: " + url);
         }
 
-        return new URL(u);
+        return new URI(u).toURL();
     }
 
     String getUser() throws MalformedURLException {
         return getGID(url);
     }
 
-    URL getPage(int page) throws MalformedURLException {
-        return new URL("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser());
+    URL getPage(int page) throws MalformedURLException, URISyntaxException {
+        return new URI("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()).toURL();
     }
 
 
@@ -71,18 +73,18 @@ public String getGID(URL url) throws MalformedURLException {
     int currentPage = 1;
 
     @Override
-    protected JSONObject getFirstPage() throws IOException {
+    protected JSONObject getFirstPage() throws IOException, URISyntaxException {
         return Http.url(getPage(1)).getJSON();
     }
 
     List<String> descriptions = new ArrayList<>();
 
     @Override
-    protected JSONObject getNextPage(JSONObject doc) throws IOException {
+    protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
         currentPage++;
         JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON();
         JSONArray items = nextPage.getJSONArray("items");
-        if (items.length() == 0) {
+        if (items.isEmpty()) {
             throw new IOException("No more pages");
         }
         return nextPage;
@@ -120,7 +122,7 @@ protected List<String> getURLsFromJSON(JSONObject json) {
 
                     return new ApiEntry(srcUrl, o.getString("author"), o.getString("title"));
                 })
-                .collect(Collectors.toList());
+                .toList();
 
         data.forEach(e -> descriptions.add(e.title));
         return data.stream().map(e -> e.srcUrl).collect(Collectors.toList());