Skip to content

Commit

Permalink
Move structs and parsing logic to hashtag.cr
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChunkyProgrammer
ChunkyProgrammer committed Jul 15, 2023
1 parent 6c8f9a6 commit d6a93fb
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 129 deletions.
141 changes: 105 additions & 36 deletions src/invidious/hashtag.cr
View file
Original file line number Diff line number Diff line change
@@ -1,69 +1,138 @@
module Invidious::Hashtag
extend self

struct HashtagPage
include DB::Serializable

property videos : Array(SearchItem) | Array(Video)
property header : HashtagHeader?
property has_next_continuation : Bool

def to_json(locale : String?, json : JSON::Builder)
json.object do
json.field "type", "hashtag"
if self.header != nil
json.field "header" do
self.header.to_json(json)
end
end
json.field "results" do
json.array do
self.videos.each do |item|
item.to_json(locale, json)
end
end
end
json.field "hasNextPage", self.has_next_continuation
end
end

# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
JSON.build do |json|
to_json(locale, json)
end
end

def to_json(json : JSON::Builder)
to_json(nil, json)
end
end

struct HashtagHeader
include DB::Serializable

property tag : String
property channel_count : Int64
property video_count : Int64

def to_json(json : JSON::Builder)
json.object do
json.field "hashtag", self.tag
json.field "channelCount", self.channel_count
json.field "videoCount", self.video_count
end
end

def to_json(_json : Nil)
JSON.build do |json|
to_json(json)
end
end
end

def fetch(hashtag : String, page : Int, region : String? = nil) : HashtagPage
cursor = (page - 1) * 60
header = nil
client_config = YoutubeAPI::ClientConfig.new(region: region)
# for any page besides the first page, get the list of videos
item = generate_continuation(hashtag, cursor)
# item is a ctoken
if cursor > 0
ctoken = generate_continuation(hashtag, cursor)
response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
response = YoutubeAPI.browse(continuation: item, client_config: client_config)
else
# get first page + header info
response = YoutubeAPI.browse("FEhashtag", params: get_hashtag_first_page(hashtag), client_config: client_config)
if header = response.dig?("header")
header = parse_item(header).as(HashtagHeader)
# item browses the first page (including metadata)
response = YoutubeAPI.browse("FEhashtag", params: item, client_config: client_config)
if item_contents = response.dig?("header", "hashtagHeaderRenderer")
header = parse_hashtag_renderer(item_contents)
end
end

items, _ = extract_items(response)
items, next_continuation = extract_items(response)
return HashtagPage.new({
videos: items,
header: header,
videos: items,
header: header,
has_next_continuation: next_continuation != nil
})
end

def get_hashtag_first_page(hashtag : String)
def generate_continuation(hashtag : String, cursor : Int)
object = {
"93:embedded" => {
"93:2:embedded" => {
"1:string" => hashtag,
"2:varint" => 0_i64,
"3:varint" => 1_i64,
},
}
if cursor > 0
object = {
"80226972:embedded" => {
"2:string" => "FEhashtag",
"3:base64" => {
"1:varint" => 60_i64, # result count
"15:base64" => {
"1:varint" => cursor.to_i64,
"2:varint" => 0_i64,
},
"93:2:embedded" => {
"1:string" => hashtag,
"2:varint" => 0_i64,
"3:varint" => 1_i64,
},
},
"35:string" => "browse-feedFEhashtag",
},
}
end

return object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
end

def generate_continuation(hashtag : String, cursor : Int)
object = {
"80226972:embedded" => {
"2:string" => "FEhashtag",
"3:base64" => {
"1:varint" => 60_i64, # result count
"15:base64" => {
"1:varint" => cursor.to_i64,
"2:varint" => 0_i64,
},
"93:2:embedded" => {
"1:string" => hashtag,
"2:varint" => 0_i64,
"3:varint" => 1_i64,
},
},
"35:string" => "browse-feedFEhashtag",
},
}
def parse_hashtag_renderer(item_contents)
info = extract_text(item_contents.dig?("hashtagInfoText")) || ""

continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
regex_match = /(?<videos>\d+\S)\D+(?<channels>\d+\S)/.match(info)

return continuation
hashtag = extract_text(item_contents.dig?("hashtag")) || ""
videos = short_text_to_number(regex_match.try &.["videos"]?.try &.to_s || "0")
channels = short_text_to_number(regex_match.try &.["channels"]?.try &.to_s || "0")

return HashtagHeader.new({
tag: hashtag,
channel_count: channels,
video_count: videos,
})
end
end
59 changes: 0 additions & 59 deletions src/invidious/helpers/serialized_yt_data.cr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,63 +274,4 @@ struct Continuation
end
end

struct HashtagPage
include DB::Serializable

property videos : Array(SearchItem) | Array(Video)
property header : HashtagHeader?

def to_json(locale : String?, json : JSON::Builder)
json.object do
json.field "type", "hashtag"
if self.header != nil
json.field "header" do
self.header.to_json(json)
end
end
json.field "results" do
json.array do
self.videos.each do |item|
item.to_json(locale, json)
end
end
end
end
end

# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
JSON.build do |json|
to_json(locale, json)
end
end

def to_json(json : JSON::Builder)
to_json(nil, json)
end
end

struct HashtagHeader
include DB::Serializable

property tag : String
property channel_count : Int64
property video_count : Int64

def to_json(json : JSON::Builder)
json.object do
json.field "type", "hashtagHeader"
json.field "hashtag", self.tag
json.field "channelCount", self.channel_count
json.field "videoCount", self.video_count
end
end

def to_json(_json : Nil)
JSON.build do |json|
to_json(json)
end
end
end

alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category
34 changes: 0 additions & 34 deletions src/invidious/yt_backend/extractors.cr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ private ITEM_PARSERS = {
Parsers::ReelItemRendererParser,
Parsers::ItemSectionRendererParser,
Parsers::ContinuationItemRendererParser,
Parsers::HashtagHeaderRenderer,
}

private alias InitialData = Hash(String, JSON::Any)
Expand Down Expand Up @@ -551,39 +550,6 @@ private module Parsers
return {{@type.name}}
end
end

# Parses an InnerTube hashtagHeaderRender into a HashtagHeaderRender.
# Returns nil when the given object isn't a hashtagHeaderRender.
#
# hashtagHeaderRender contains metadate of the hashtag page such as video count and channel count
#
module HashtagHeaderRenderer
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["hashtagHeaderRenderer"]?
return self.parse(item_contents)
end
end

private def self.parse(item_contents)
info = extract_text(item_contents.dig?("hashtagInfoText")) || ""

regex_match = /(?<videos>\d+\S)\D+(?<channels>\d+\S)/.match(info)

hashtag = extract_text(item_contents.dig?("hashtag")) || ""
videos = short_text_to_number(regex_match.try &.["videos"]?.try &.to_s || "0")
channels = short_text_to_number(regex_match.try &.["channels"]?.try &.to_s || "0")

return HashtagHeader.new({
tag: hashtag,
channel_count: channels,
video_count: videos,
})
end

def self.parser_name
return {{@type.name}}
end
end
end

# The following are the extractors for extracting an array of items from
Expand Down

0 comments on commit d6a93fb

Please sign in to comment.