From 4627dbbe1c86ae50a59e149b631586a20247bbde Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:28:15 -0400 Subject: [PATCH 1/2] Fix: Search not showing next page button when a next page exists --- src/invidious/routes/api/v1/channels.cr | 2 +- src/invidious/routes/api/v1/search.cr | 2 +- src/invidious/routes/playlists.cr | 6 ++++-- src/invidious/routes/search.cr | 4 ++-- src/invidious/search/processors.cr | 12 ++++++------ src/invidious/search/query.cr | 12 +++++++----- src/invidious/yt_backend/extractors.cr | 7 +++++-- 7 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 67018660b..4d4032bc1 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -440,7 +440,7 @@ module Invidious::Routes::API::V1::Channels query.channel = env.params.url["ucid"] begin - search_results = query.process + search_results, _ = query.process rescue ex return error_json(400, ex) end diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 9fb283c23..8cad6dac6 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Search query = Invidious::Search::Query.new(env.params.query, :regular, region) begin - search_results = query.process + search_results, _ = query.process rescue ex return error_json(400, ex) end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 9c6843e99..8d83b4256 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -254,9 +254,11 @@ module Invidious::Routes::Playlists begin query = Invidious::Search::Query.new(env.params.query, :playlist, region) - items = query.process.select(SearchVideo).map(&.as(SearchVideo)) + processed_query, has_continuation = query.process + items = processed_query.select(SearchVideo).map(&.as(SearchVideo)) rescue ex items = [] of SearchVideo + has_continuation = false end # Pagination @@ -264,7 +266,7 @@ module Invidious::Routes::Playlists page_nav_html = Frontend::Pagination.nav_numeric(locale, base_url: "/add_playlist_items?list=#{playlist.id}&q=#{query_encoded}", current_page: page, - show_next: (items.size >= 20) + show_next: has_continuation ) env.set "add_playlist_items", plid diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 5be335339..7fa88fa98 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -52,7 +52,7 @@ module Invidious::Routes::Search user = env.get? "user" begin - items = query.process + items, has_continuation = query.process rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex @@ -65,7 +65,7 @@ module Invidious::Routes::Search page_nav_html = Frontend::Pagination.nav_numeric(locale, base_url: "/search?#{query.to_http_params}", current_page: query.page, - show_next: (items.size >= 20) + show_next: has_continuation ) if query.type == Invidious::Search::Query::Type::Channel diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index 25edb9362..ae9ef1e92 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -3,19 +3,19 @@ module Invidious::Search extend self # Regular search (`/search` endpoint) - def regular(query : Query) : Array(SearchItem) + def regular(query : Query) : {Array(SearchItem), Bool} search_params = query.filters.to_yt_params(page: query.page) client_config = YoutubeAPI::ClientConfig.new(region: query.region) initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) - items, _ = extract_items(initial_data) - return items.reject!(Category) + items, next_continuation = extract_items(initial_data) + return items.reject!(Category), next_continuation != nil end # Search a youtube channel # TODO: clean code, and rely more on YoutubeAPI - def channel(query : Query) : Array(SearchItem) + def channel(query : Query) : {Array(SearchItem), Bool} response = YT_POOL.client &.get("/channel/#{query.channel}") if response.status_code == 404 @@ -31,8 +31,8 @@ module Invidious::Search continuation = produce_channel_search_continuation(ucid, query.text, query.page) response_json = YoutubeAPI.browse(continuation) - items, _ = extract_items(response_json, "", ucid) - return items.reject!(Category) + items, next_continuation = extract_items(response_json, "", ucid) + return items.reject!(Category), next_continuation != nil end # Search inside of user subscriptions diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index e38845d96..8fdba74a9 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -105,26 +105,28 @@ module Invidious::Search # Run the search query using the corresponding search processor. # Returns either the results or an empty array of `SearchItem`. - def process(user : Invidious::User? = nil) : Array(SearchItem) | Array(ChannelVideo) + def process(user : Invidious::User? = nil) : {Array(SearchItem) | Array(ChannelVideo), Bool} items = [] of SearchItem + has_continuation = false # Don't bother going further if search query is empty - return items if self.empty_raw_query? + return items, has_continuation if self.empty_raw_query? case @type when .regular?, .playlist? - items = Processors.regular(self) + items, has_continuation = Processors.regular(self) # when .channel? - items = Processors.channel(self) + items, has_continuation = Processors.channel(self) # when .subscriptions? if user items = Processors.subscriptions(self, user.as(Invidious::User)) + has_continuation = items.size >= 20 end end - return items + return items, has_continuation end # Return the HTTP::Params corresponding to this Query (invidious format) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 56325cf7b..f8819fbf9 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -720,8 +720,11 @@ private module Extractors raw_items = [] of Array(JSON::Any) target.dig("primaryContents", "sectionListRenderer", "contents").as_a.each do |node| - if node = node["itemSectionRenderer"]? - raw_items << node["contents"].as_a + if new_node = node["itemSectionRenderer"]? + raw_items << new_node["contents"].as_a + end + if node["continuationItemRenderer"]? + raw_items.push([node]) end end From 4119cecf75881c60e72935c99833896ee9adaa9d Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sun, 22 Oct 2023 19:09:02 -0400 Subject: [PATCH 2/2] simplify check for continuationItemRenderer Co-Authored-By: syeopite <70992037+syeopite@users.noreply.github.com> Co-Authored-By: Samantaz Fox --- src/invidious/yt_backend/extractors.cr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index f8819fbf9..ae8679e03 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -717,14 +717,14 @@ private module Extractors end private def self.extract(target) - raw_items = [] of Array(JSON::Any) + raw_items = [] of JSON::Any target.dig("primaryContents", "sectionListRenderer", "contents").as_a.each do |node| if new_node = node["itemSectionRenderer"]? - raw_items << new_node["contents"].as_a - end - if node["continuationItemRenderer"]? - raw_items.push([node]) + raw_items += new_node["contents"].as_a + elsif node["continuationItemRenderer"]? + # we put the node in an array so the ContinuationItemRendererParser is able to parse it + raw_items << node end end