diff --git a/Cargo.lock b/Cargo.lock index c072999..3373ca7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -435,6 +435,7 @@ dependencies = [ "futures", "handlebars", "hex", + "md5", "memchr", "once_cell", "prost", @@ -446,6 +447,7 @@ dependencies = [ "sea-orm", "serde", "serde_json", + "serde_urlencoded", "strum 0.26.3", "thiserror", "tokio", @@ -1615,6 +1617,12 @@ dependencies = [ "digest", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.7.4" diff --git a/Cargo.toml b/Cargo.toml index 9c3ec1e..8dcfd73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ float-ord = "0.3.2" futures = "0.3.30" handlebars = "5.1.2" hex = "0.4.3" +md5 = "0.7.0" memchr = "2.7.4" once_cell = "1.19.0" prost = "0.12.6" @@ -53,6 +54,7 @@ sea-orm = { version = "0.12.15", features = [ sea-orm-migration = { version = "0.12.15", features = [] } serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.120" +serde_urlencoded = "0.7.1" strum = { version = "0.26.3", features = ["derive"] } thiserror = "1.0.61" tokio = { version = "1.38.0", features = ["full"] } diff --git a/crates/bili_sync/Cargo.toml b/crates/bili_sync/Cargo.toml index 9fd3e24..e2e6835 100644 --- a/crates/bili_sync/Cargo.toml +++ b/crates/bili_sync/Cargo.toml @@ -24,6 +24,7 @@ float-ord = { workspace = true } futures = { workspace = true } handlebars = { workspace = true } hex = { workspace = true } +md5 = { workspace = true } memchr = { workspace = true } once_cell = { workspace = true } prost = { workspace = true } @@ -35,6 +36,7 @@ rsa = { workspace = true } sea-orm = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +serde_urlencoded = { workspace = true } strum = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/crates/bili_sync/src/adapter/collection.rs b/crates/bili_sync/src/adapter/collection.rs index 93e294e..5c3d6e7 100644 --- a/crates/bili_sync/src/adapter/collection.rs +++ b/crates/bili_sync/src/adapter/collection.rs @@ -20,11 +20,12 @@ use crate::utils::status::Status; pub async fn collection_from<'a>( collection_item: &'a CollectionItem, + mixin_key: &'a str, path: &Path, bili_client: &'a BiliClient, connection: &DatabaseConnection, ) -> Result<(Box, Pin + 'a>>)> { - let collection = Collection::new(bili_client, collection_item); + let collection = Collection::new(bili_client, collection_item, mixin_key); let collection_info = collection.get_info().await?; collection::Entity::insert(collection::ActiveModel { s_id: Set(collection_info.sid), diff --git a/crates/bili_sync/src/adapter/mod.rs b/crates/bili_sync/src/adapter/mod.rs index 92e796f..6faf5a4 100644 --- a/crates/bili_sync/src/adapter/mod.rs +++ b/crates/bili_sync/src/adapter/mod.rs @@ -17,8 +17,13 @@ use watch_later::watch_later_from; use crate::bilibili::{BiliClient, CollectionItem, VideoInfo}; pub enum Args<'a> { - Favorite { fid: &'a str }, - Collection { collection_item: &'a CollectionItem }, + Favorite { + fid: &'a str, + }, + Collection { + collection_item: &'a CollectionItem, + mixin_key: &'a str, + }, WatchLater, } @@ -30,7 +35,10 @@ pub async fn video_list_from<'a>( ) -> Result<(Box, Pin + 'a>>)> { match args { Args::Favorite { fid } => favorite_from(fid, path, bili_client, connection).await, - Args::Collection { collection_item } => collection_from(collection_item, path, bili_client, connection).await, + Args::Collection { + collection_item, + mixin_key, + } => collection_from(collection_item, mixin_key, path, bili_client, connection).await, Args::WatchLater => watch_later_from(path, bili_client, connection).await, } } diff --git a/crates/bili_sync/src/bilibili/client.rs b/crates/bili_sync/src/bilibili/client.rs index 2874d09..37ab29c 100644 --- a/crates/bili_sync/src/bilibili/client.rs +++ b/crates/bili_sync/src/bilibili/client.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use anyhow::{bail, Result}; use reqwest::{header, Method}; +use crate::bilibili::credential::WbiImg; use crate::bilibili::Credential; use crate::config::CONFIG; @@ -94,4 +95,13 @@ impl BiliClient { }; credential.is_login(&self.client).await } + + /// 获取 wbi img,用于生成请求签名 + pub async fn wbi_img(&self) -> Result { + let credential = CONFIG.credential.load(); + let Some(credential) = credential.as_deref() else { + bail!("no credential found"); + }; + credential.wbi_img(&self.client).await + } } diff --git a/crates/bili_sync/src/bilibili/collection.rs b/crates/bili_sync/src/bilibili/collection.rs index 97767d6..9f5dadf 100644 --- a/crates/bili_sync/src/bilibili/collection.rs +++ b/crates/bili_sync/src/bilibili/collection.rs @@ -1,14 +1,16 @@ #![allow(dead_code)] +use std::borrow::Cow; use std::fmt::{Display, Formatter}; -use anyhow::Result; +use anyhow::{bail, Result}; use async_stream::stream; use futures::Stream; use reqwest::Method; use serde::Deserialize; use serde_json::Value; +use crate::bilibili::credential::encoded_query; use crate::bilibili::{BiliClient, Validate, VideoInfo}; #[derive(PartialEq, Eq, Hash, Clone, Debug)] @@ -56,6 +58,7 @@ pub struct CollectionItem { pub struct Collection<'a> { client: &'a BiliClient, collection: &'a CollectionItem, + mixin_key: Cow<'a, str>, } #[derive(Debug, PartialEq)] @@ -94,8 +97,24 @@ impl<'de> Deserialize<'de> for CollectionInfo { } impl<'a> Collection<'a> { - pub fn new(client: &'a BiliClient, collection: &'a CollectionItem) -> Self { - Self { client, collection } + pub async fn build(client: &'a BiliClient, collection: &'a CollectionItem) -> Result { + let wbi_img = client.wbi_img().await?; + let Some(mixin_key) = wbi_img.into_mixin_key() else { + bail!("failed to get mixin key"); + }; + Ok(Self { + client, + collection, + mixin_key: Cow::Owned(mixin_key), + }) + } + + pub fn new(client: &'a BiliClient, collection: &'a CollectionItem, mixin_key: &'a str) -> Self { + Self { + client, + collection, + mixin_key: Cow::Borrowed(mixin_key), + } } pub async fn get_info(&self) -> Result { @@ -108,10 +127,6 @@ impl<'a> Collection<'a> { } async fn get_series_info(&self) -> Result { - assert!( - self.collection.collection_type == CollectionType::Series, - "collection type is not series" - ); self.client .request(Method::GET, "https://api.bilibili.com/x/series/series") .query(&[("series_id", self.collection.sid.as_str())]) @@ -125,27 +140,34 @@ impl<'a> Collection<'a> { async fn get_videos(&self, page: i32) -> Result { let page = page.to_string(); + let mixin_key = self.mixin_key.as_ref(); let (url, query) = match self.collection.collection_type { CollectionType::Series => ( "https://api.bilibili.com/x/series/archives", - vec![ - ("mid", self.collection.mid.as_str()), - ("series_id", self.collection.sid.as_str()), - ("only_normal", "true"), - ("sort", "desc"), - ("pn", page.as_str()), - ("ps", "30"), - ], + encoded_query( + vec![ + ("mid", self.collection.mid.as_str()), + ("series_id", self.collection.sid.as_str()), + ("only_normal", "true"), + ("sort", "desc"), + ("pn", page.as_str()), + ("ps", "30"), + ], + mixin_key, + ), ), CollectionType::Season => ( "https://api.bilibili.com/x/polymer/web-space/seasons_archives_list", - vec![ - ("mid", self.collection.mid.as_str()), - ("season_id", self.collection.sid.as_str()), - ("sort_reverse", "true"), - ("page_num", page.as_str()), - ("page_size", "30"), - ], + encoded_query( + vec![ + ("mid", self.collection.mid.as_str()), + ("season_id", self.collection.sid.as_str()), + ("sort_reverse", "true"), + ("page_num", page.as_str()), + ("page_size", "30"), + ], + mixin_key, + ), ), }; self.client diff --git a/crates/bili_sync/src/bilibili/credential.rs b/crates/bili_sync/src/bilibili/credential.rs index 1fae905..326b1e7 100644 --- a/crates/bili_sync/src/bilibili/credential.rs +++ b/crates/bili_sync/src/bilibili/credential.rs @@ -11,6 +11,12 @@ use serde::{Deserialize, Serialize}; use crate::bilibili::{Client, Validate}; +const MIXIN_KEY_ENC_TAB: [usize; 64] = [ + 46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39, 12, 38, + 41, 13, 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, 22, 25, 54, 21, 56, 59, 6, 63, 57, 62, 11, 36, + 20, 34, 44, 52, +]; + #[derive(Default, Debug, Clone, Serialize, Deserialize)] pub struct Credential { pub sessdata: String, @@ -20,7 +26,30 @@ pub struct Credential { pub ac_time_value: String, } +#[derive(Debug, Deserialize)] +pub struct WbiImg { + img_url: String, + sub_url: String, +} + +impl WbiImg { + pub fn into_mixin_key(self) -> Option { + get_mixin_key(self) + } +} + impl Credential { + pub async fn wbi_img(&self, client: &Client) -> Result { + let mut res = client + .request(Method::GET, "https://api.bilibili.com/x/web-interface/nav", Some(self)) + .send() + .await? + .json::() + .await? + .validate()?; + Ok(serde_json::from_value(res["data"]["wbi_img"].take())?) + } + /// 检查凭据是否有效 pub async fn need_refresh(&self, client: &Client) -> Result { let res = client @@ -181,6 +210,41 @@ fn regex_find(pattern: &str, doc: &str) -> Result { .to_string()) } +fn get_filename(url: &str) -> Option<&str> { + url.rsplit_once('/') + .and_then(|(_, s)| s.rsplit_once('.')) + .map(|(s, _)| s) +} + +fn get_mixin_key(wbi_img: WbiImg) -> Option { + let key = match ( + get_filename(wbi_img.img_url.as_str()), + get_filename(wbi_img.sub_url.as_str()), + ) { + (Some(img_key), Some(sub_key)) => img_key.to_string() + sub_key, + _ => return None, + }; + let key = key.as_bytes(); + Some(MIXIN_KEY_ENC_TAB.iter().take(32).map(|&x| key[x] as char).collect()) +} + +pub fn encoded_query<'a>(params: Vec<(&'a str, impl Into)>, mixin_key: &str) -> Vec<(&'a str, String)> { + let params = params.into_iter().map(|(k, v)| (k, v.into())).collect(); + _encoded_query(params, mixin_key, chrono::Local::now().timestamp().to_string()) +} + +fn _encoded_query<'a>(params: Vec<(&'a str, String)>, mixin_key: &str, timestamp: String) -> Vec<(&'a str, String)> { + let mut params: Vec<(&'a str, String)> = params + .into_iter() + .map(|(k, v)| (k, v.chars().filter(|&x| !"!'()*".contains(x)).collect::())) + .collect(); + params.push(("wts", timestamp)); + params.sort_by(|a, b| a.0.cmp(b.0)); + let query = serde_urlencoded::to_string(¶ms).unwrap().replace('+', "%20"); + params.push(("w_rid", format!("{:x}", md5::compute(query.clone() + mixin_key)))); + params +} + #[cfg(test)] mod tests { use super::*; @@ -199,4 +263,45 @@ mod tests { "b0cc8411ded2f9db2cff2edb3123acac", ); } + + #[test] + fn test_encode_query() { + let query = vec![ + ("bar", "五一四".to_string()), + ("baz", "1919810".to_string()), + ("foo", "one one four".to_string()), + ]; + assert_eq!( + serde_urlencoded::to_string(query).unwrap().replace('+', "%20"), + "bar=%E4%BA%94%E4%B8%80%E5%9B%9B&baz=1919810&foo=one%20one%20four" + ); + } + + #[test] + fn test_wbi_key() { + let key = WbiImg { + img_url: "https://i0.hdslb.com/bfs/wbi/7cd084941338484aae1ad9425b84077c.png".to_string(), + sub_url: "https://i0.hdslb.com/bfs/wbi/4932caff0ff746eab6f01bf08b70ac45.png".to_string(), + }; + let mixin_key = get_mixin_key(key); + assert_eq!(mixin_key, Some("ea1db124af3c7062474693fa704f4ff8".to_string())); + assert_eq!( + _encoded_query( + vec![ + ("foo", "114".to_string()), + ("bar", "514".to_string()), + ("zab", "1919810".to_string()) + ], + &mixin_key.unwrap(), + "1702204169".to_string(), + ), + vec![ + ("bar", "514".to_string()), + ("foo", "114".to_string()), + ("wts", "1702204169".to_string()), + ("zab", "1919810".to_string()), + ("w_rid", "8f6f2b5b3d485fe1886cec6a0be8c5d4".to_string()), + ] + ) + } } diff --git a/crates/bili_sync/src/bilibili/mod.rs b/crates/bili_sync/src/bilibili/mod.rs index 03a9a88..63bafb0 100644 --- a/crates/bili_sync/src/bilibili/mod.rs +++ b/crates/bili_sync/src/bilibili/mod.rs @@ -121,7 +121,7 @@ mod tests { #[ignore = "only for manual test"] #[tokio::test] - async fn assert_video_info_type() { + async fn test_video_info_type() { let bili_client = BiliClient::new(); let video = Video::new(&bili_client, "BV1Z54y1C7ZB".to_string()); assert!(matches!(video.get_view_info().await, Ok(VideoInfo::View { .. }))); @@ -130,7 +130,7 @@ mod tests { sid: "387214".to_string(), collection_type: CollectionType::Series, }; - let collection = Collection::new(&bili_client, &collection_item); + let collection = Collection::build(&bili_client, &collection_item).await.unwrap(); let stream = collection.into_simple_video_stream(); pin_mut!(stream); assert!(matches!(stream.next().await, Some(VideoInfo::Simple { .. }))); diff --git a/crates/bili_sync/src/main.rs b/crates/bili_sync/src/main.rs index bacaa19..0c3205a 100644 --- a/crates/bili_sync/src/main.rs +++ b/crates/bili_sync/src/main.rs @@ -50,11 +50,26 @@ async fn main() { } } info!("所有收藏夹处理完毕"); - for (collection_item, path) in &CONFIG.collection_list { - if let Err(e) = - process_video_list(Args::Collection { collection_item }, &bili_client, path, &connection).await - { - error!("处理合集 {collection_item:?} 时遇到非预期的错误:{e}"); + match bili_client.wbi_img().await.map(|wbi_img| wbi_img.into_mixin_key()) { + Ok(Some(mixin_key)) => { + for (collection_item, path) in &CONFIG.collection_list { + if let Err(e) = process_video_list( + Args::Collection { + collection_item, + mixin_key: &mixin_key, + }, + &bili_client, + path, + &connection, + ) + .await + { + error!("处理合集 {collection_item:?} 时遇到非预期的错误:{e}"); + } + } + } + _ => { + error!("获取 mixin key 失败,无法进行 wbi 签名,跳过本轮合集处理"); } } info!("所有合集处理完毕");