[YouTube] Detect deleted/nonexistent/invalid channels and playlists

- Added tests for these cases.
TeamNewPipe · Mar 1, 2020 · e65333c · PeterHindes · Mar 12, 2020 · e65333c
1 parent 98e3594
commit e65333c
Show file tree

Hide file tree

Showing 5 changed files with 85 additions and 1 deletion.
diff --git a/...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
@@ -58,8 +58,8 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
         final String url = super.getUrl() + "/videos?pbj=1&view=0&flow=grid";
 
         final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
-
         initialData = ajaxJson.getObject(1).getObject("response");
+        YoutubeParsingHelper.defaultAlertsCheck(initialData);
     }
 
 

diff --git a/...va/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/...va/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java
@@ -10,6 +10,7 @@
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
+import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Utils;
@@ -39,6 +40,8 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
         final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
 
         initialData = ajaxJson.getObject(1).getObject("response");
+        YoutubeParsingHelper.defaultAlertsCheck(initialData);
+
         playlistInfo = getPlaylistInfo();
     }
 

diff --git a/.../java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java b/.../java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
@@ -362,10 +362,45 @@ public static JsonArray getJsonResponse(String url, Localization localization) t
             throw new ParsingException("JSON response is too short");
         }
 
+        // Check if the request was redirected to the error page.
+        final URL latestUrl = new URL(response.latestUrl());
+        if (latestUrl.getHost().equalsIgnoreCase("www.youtube.com")) {
+            final String path = latestUrl.getPath();
+            if (path.equalsIgnoreCase("/oops") || path.equalsIgnoreCase("/error")) {
+                throw new ContentNotAvailableException("Content unavailable");
+            }
+        }
+
+        final String responseContentType = response.getHeader("Content-Type");
+        if (responseContentType != null && responseContentType.toLowerCase().contains("text/html")) {
+            throw new ParsingException("Got HTML document, expected JSON response" +
+                    " (latest url was: \"" + response.latestUrl() + "\")");
+        }
+
         try {
             return JsonParser.array().from(responseBody);
         } catch (JsonParserException e) {
             throw new ParsingException("Could not parse JSON", e);
         }
     }
+
+    /**
+     * Shared alert detection function, multiple endpoints return the error similarly structured.
+     * <p>
+     * Will check if the object has an alert of the type "ERROR".
+     *
+     * @param initialData the object which will be checked if an alert is present
+     * @throws ContentNotAvailableException if an alert is detected
+     */
+    public static void defaultAlertsCheck(JsonObject initialData) throws ContentNotAvailableException {
+        final JsonArray alerts = initialData.getArray("alerts");
+        if (alerts != null && !alerts.isEmpty()) {
+            final JsonObject alertRenderer = alerts.getObject(0).getObject("alertRenderer");
+            final String alertText = alertRenderer.getObject("text").getString("simpleText");
+            final String alertType = alertRenderer.getString("type");
+            if (alertType.equalsIgnoreCase("ERROR")) {
+                throw new ContentNotAvailableException("Got error: \"" + alertText + "\"");
+            }
+        }
+    }
 }
diff --git a/.../test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java b/.../test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java
@@ -6,6 +6,7 @@
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.ServiceList;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
@@ -19,6 +20,28 @@
  * Test for {@link ChannelExtractor}
  */
 public class YoutubeChannelExtractorTest {
+
+    public static class NotAvailable {
+        @BeforeClass
+        public static void setUp() {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void deletedFetch() throws Exception {
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/UCAUc4iz6edWerIjlnL8OSSw");
+            extractor.fetchPage();
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void nonExistentFetch() throws Exception {
+            final ChannelExtractor extractor =
+                    YouTube.getChannelExtractor("https://www.youtube.com/channel/DOESNT-EXIST");
+            extractor.fetchPage();
+        }
+    }
+
     public static class Gronkh implements BaseChannelExtractorTest {
         private static YoutubeChannelExtractor extractor;
 

diff --git a/...test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java b/...test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java
@@ -7,6 +7,7 @@
 import org.schabi.newpipe.extractor.ListExtractor;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.ServiceList;
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
 import org.schabi.newpipe.extractor.services.BasePlaylistExtractorTest;
@@ -23,6 +24,28 @@
  * Test for {@link YoutubePlaylistExtractor}
  */
 public class YoutubePlaylistExtractorTest {
+
+    public static class NotAvailable {
+        @BeforeClass
+        public static void setUp() {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void nonExistentFetch() throws Exception {
+            final PlaylistExtractor extractor =
+                    YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL11111111111111111111111111111111");
+            extractor.fetchPage();
+        }
+
+        @Test(expected = ContentNotAvailableException.class)
+        public void invalidId() throws Exception {
+            final PlaylistExtractor extractor =
+                    YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID");
+            extractor.fetchPage();
+        }
+    }
+
     public static class TimelessPopHits implements BasePlaylistExtractorTest {
         private static YoutubePlaylistExtractor extractor;