From f063dd15ec6183d6319ebc119fda370c949a9f9f Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Fri, 2 Jun 2023 22:08:53 +0100 Subject: [PATCH 1/4] Fail-fast extraction if the initial player response is invalid. --- .../youtube/extractors/YoutubeStreamExtractor.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 17462aac35..86ae8533f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -868,6 +868,10 @@ public void onFetchPage(@Nonnull final Downloader downloader) playerMicroFormatRenderer = youtubePlayerResponse.getObject("microformat") .getObject("playerMicroformatRenderer"); + if (isPlayerResponseNotValid(playerResponse, videoId)) { + throw new ExtractionException("Initial player response is not valid"); + } + final byte[] body = JsonWriter.string( prepareDesktopJsonBuilder(localization, contentCountry) .value(VIDEO_ID, videoId) @@ -1058,6 +1062,11 @@ private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCo final JsonObject tvHtml5EmbedPlayerResponse = getJsonPostResponse(PLAYER, createDesktopPlayerBody(localization, contentCountry, videoId, sts, true, html5Cpn), localization); + + if (isPlayerResponseNotValid(tvHtml5EmbedPlayerResponse, videoId)) { + return; + } + final JsonObject streamingData = tvHtml5EmbedPlayerResponse.getObject( STREAMING_DATA); if (!isNullOrEmpty(streamingData)) { From e3480ba2aeb171729dbb1429d29317a1044f1611 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Fri, 2 Jun 2023 22:54:26 +0100 Subject: [PATCH 2/4] Update the JavaDoc for the rate-limiting issue. --- .../youtube/extractors/YoutubeStreamExtractor.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 86ae8533f3..03a8794936 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1079,12 +1079,23 @@ private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCo * Checks whether an additional player response is not valid. * *

- * If YouTube detect that requests come from a third party client, they may replace the real + * If YouTube detects that requests come from a third party client, they may replace the real * player response by another one of a video saying that this content is not available on this * app and to watch it on the latest version of YouTube. *

* *

+ * YouTube may also sometimes for currently unknown reasons rate-limit an IP, and replace the real one + * by a player response with a video that says that the video is unavailable. + *

+ * + *

+ * This behaviour has been observed in on the {@code ANDROID} and {@code WEB} clients, see + * + * https://github.com/TeamPiped/Piped/issues/2487. + *

+ * + *

* We can detect this by checking whether the video ID of the player response returned is the * same as the one requested by the extractor. *

From 9aa8c4fa4f2b2af9115b08567883808503fe795b Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Fri, 2 Jun 2023 22:56:10 +0100 Subject: [PATCH 3/4] Update parameter and JavaDoc. --- .../youtube/extractors/YoutubeStreamExtractor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 03a8794936..4a089eb4e5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1106,15 +1106,14 @@ private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCo * https://github.com/TeamNewPipe/NewPipe/issues/8713. *

* - * @param additionalPlayerResponse an additional response to the one of the {@code HTML5} - * client used + * @param playerResponse A player response from any client * @param videoId the video ID of the content requested * @return whether the video ID of the player response is not equal to the one requested */ private static boolean isPlayerResponseNotValid( - @Nonnull final JsonObject additionalPlayerResponse, + @Nonnull final JsonObject playerResponse, @Nonnull final String videoId) { - return !videoId.equals(additionalPlayerResponse.getObject("videoDetails") + return !videoId.equals(playerResponse.getObject("videoDetails") .getString("videoId", "")); } From eaadb960b22ca5411b22160391a348adf708decc Mon Sep 17 00:00:00 2001 From: Audric V <74829229+AudricV@users.noreply.github.com> Date: Sun, 18 Jun 2023 20:58:44 +0200 Subject: [PATCH 4/4] Remove empty string video ID fallback and improve Javadoc --- .../extractors/YoutubeStreamExtractor.java | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 4a089eb4e5..113f2d4a3d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1076,21 +1076,22 @@ private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCo } /** - * Checks whether an additional player response is not valid. + * Checks whether a player response is invalid. * *

- * If YouTube detects that requests come from a third party client, they may replace the real + * If YouTube detect that requests come from a third party client, they may replace the real * player response by another one of a video saying that this content is not available on this - * app and to watch it on the latest version of YouTube. - *

- * - *

- * YouTube may also sometimes for currently unknown reasons rate-limit an IP, and replace the real one - * by a player response with a video that says that the video is unavailable. + * app and to watch it on the latest version of YouTube. This behavior has been observed on the + * {@code ANDROID} client, see + * + * https://github.com/TeamNewPipe/NewPipe/issues/8713. *

* *

- * This behaviour has been observed in on the {@code ANDROID} and {@code WEB} clients, see + * YouTube may also sometimes for currently unknown reasons rate-limit an IP, and replace the + * real one by a player response with a video that says that the requested video is + * unavailable. This behaviour has been observed in Piped on the InnerTube clients used by the + * extractor ({@code ANDROID} and {@code WEB} clients) which should apply for all clients, see * * https://github.com/TeamPiped/Piped/issues/2487. *

@@ -1100,21 +1101,15 @@ private void fetchTvHtml5EmbedJsonPlayer(@Nonnull final ContentCountry contentCo * same as the one requested by the extractor. *

* - *

- * This behavior has been already observed on the {@code ANDROID} client, see - * - * https://github.com/TeamNewPipe/NewPipe/issues/8713. - *

- * - * @param playerResponse A player response from any client - * @param videoId the video ID of the content requested + * @param playerResponse a player response from any client + * @param videoId the video ID of the content requested * @return whether the video ID of the player response is not equal to the one requested */ private static boolean isPlayerResponseNotValid( @Nonnull final JsonObject playerResponse, @Nonnull final String videoId) { return !videoId.equals(playerResponse.getObject("videoDetails") - .getString("videoId", "")); + .getString("videoId")); } private static void storePlayerJs() throws ParsingException {