Add new youtube fetch backend - rusty_ytdl

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "lofigirl_sys", 
     "lofigirl_web_client",
 ]
+resolver = "2"
 
 [profile.release]
 lto = true

lofigirl_client/Cargo.toml

@@ -10,16 +10,16 @@ lofigirl_shared_common = { path = "../lofigirl_shared_common"}
 lofigirl_shared_listen = { path = "../lofigirl_shared_listen"}
 reqwest = { version = "0.11", features = ["json"] }
 serde_json = "1.0"
-toml = "0.7"
+toml = "0.8"
 serde = { version = "1.0", features = ["derive"]}
-tokio = { version = "1.28", features = ["rt", "fs", "time"] }
+tokio = { version = "1.33", features = ["rt", "fs", "time"] }
 clap = { version = "4.4", features = ["derive"] }
 anyhow = "1.0"
 thiserror = "1.0"
 tracing = "0.1"
 tracing-subscriber = "0.3"
-url = { version = "2.3", optional = true}
-notify-rust = { version = "4.8", optional = true }
+url = { version = "2.4", optional = true}
+notify-rust = { version = "4.9", optional = true }
 
 
 [features]

lofigirl_server/Cargo.toml

@@ -11,16 +11,16 @@ lofigirl_sys = { path = "../lofigirl_sys"}
 lofigirl_shared_common = { path = "../lofigirl_shared_common"}
 lofigirl_shared_listen = { path = "../lofigirl_shared_listen"}
 serde = { version = "1.0", features = ["derive"] }
-actix-web = "4.3"
+actix-web = "4.4"
 actix-cors = "0.6"
-actix-rt = "2.8"
+actix-rt = "2.9"
 sqlx = { version = "0.7", features = ["sqlite", "runtime-tokio-native-tls"] }
-uuid = { version = "1.4", features = ["v4"] }
+uuid = { version = "1.5", features = ["v4"] }
 thiserror = "1.0"
 clap = { version = "4.4", features = ["derive"] }
-toml = "0.7"
+toml = "0.8"
 anyhow = "1.0"
-url = "2.3"
+url = "2.4"
 parking_lot = "0.12"
 tracing = "0.1"
 tracing-subscriber = "0.3"

lofigirl_shared_common/Cargo.toml

@@ -12,5 +12,5 @@ serde = { version = "1.0", features = ["derive"] }
 anyhow = "1.0"
 thiserror = "1.0"
 strsim = "0.10"
-once_cell = "1.17"
+once_cell = "1.18"
 tracing = "0.1"

lofigirl_shared_listen/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 
 [dependencies]
 lofigirl_shared_common = {path = "../lofigirl_shared_common"}
-notify-rust = { version = "4.8", optional = true }
+notify-rust = { version = "4.9", optional = true }
 rustfm-scrobble-proxy = "2.0"
 listenbrainz = "0.7"
 anyhow = "1.0"

lofigirl_sys/Cargo.toml

@@ -8,14 +8,15 @@ edition = "2021"
 
 [dependencies]
 lofigirl_shared_common = { path = "../lofigirl_shared_common"}
-opencv = { version = "0.84", features = ["clang-runtime"] }
-ytextract = { version = "0.11", optional = true}
-rustube = "0.6"
+opencv = { version = "0.86", features = ["clang-runtime"] }
+rustube = { version = "0.6", optional = true }
+rusty_ytdl = "0.6"
 leptess = "0.14"
 anyhow = "1.0"
 thiserror = "1.0"
-url = "2.3"
+url = "2.4"
 tracing = "0.1"
+tempfile = "3.8"
 
 [features]
-use_ytextract = ["ytextract"]
+alt_yt_backend = ["rustube"]

lofigirl_sys/src/capture.rs

@@ -1,14 +1,17 @@
 use anyhow::Result;
+use rusty_ytdl::VideoQuality;
+use std::io::Write;
+use tempfile::{NamedTempFile, TempDir};
 use thiserror::Error;
 use tracing::info;
 use url::Url;
 
-#[cfg(not(feature = "use_ytextract"))]
+#[cfg(feature = "alt_yt_backend")]
 pub struct YoutubeLinkCapturer;
-#[cfg(not(feature = "use_ytextract"))]
+#[cfg(feature = "alt_yt_backend")]
 impl YoutubeLinkCapturer {
-    pub fn new() -> Self {
-        YoutubeLinkCapturer
+    pub fn new() -> Result<Self> {
+        Ok(YoutubeLinkCapturer)
     }
     pub async fn get_raw_link(&self, url: &Url) -> Result<String> {
         let descrambler = rustube::VideoFetcher::from_url(url)?
@@ -27,33 +30,46 @@ impl YoutubeLinkCapturer {
     }
 }
 
-#[cfg(feature = "use_ytextract")]
+#[cfg(not(feature = "alt_yt_backend"))]
 pub struct YoutubeLinkCapturer {
-    client: ytextract::Client,
+    temp_dir: TempDir,
+    last_persistent_fetch_path: std::path::PathBuf,
+    last_persistent_fetch_path_str: String,
 }
-#[cfg(feature = "use_ytextract")]
+#[cfg(not(feature = "alt_yt_backend"))]
 impl YoutubeLinkCapturer {
-    pub fn new() -> YoutubeLinkCapturer {
-        YoutubeLinkCapturer {
-            client: ytextract::Client::new(),
-        }
+    pub fn new() -> Result<Self> {
+        let temp_dir = tempfile::tempdir()?;
+        let last_persistent_fetch_path = temp_dir.path().join("current_chunk");
+        let last_persistent_fetch_path_str = last_persistent_fetch_path
+            .as_os_str()
+            .to_str()
+            .ok_or(CaptureError::YoutubeLinkCaptureError)?
+            .to_owned();
+        Ok(YoutubeLinkCapturer {
+            temp_dir,
+            last_persistent_fetch_path,
+            last_persistent_fetch_path_str,
+        })
     }
-
     pub async fn get_raw_link(&self, url: &Url) -> Result<String> {
-        let video = self.client.video(url.as_str().parse()?).await?;
-
-        let raw_stream = video
-            .streams()
-            .await?
-            .filter_map(|stream| match stream {
-                ytextract::Stream::Audio(_) => None,
-                ytextract::Stream::Video(v) => Some(v),
-            })
-            .max_by_key(|stream| stream.width())
-            .ok_or(CaptureError::YoutubeLinkCaptureError)?;
-        let raw_link = raw_stream.url().to_string();
-        info!("Raw link is captured using ytextract: {}", raw_link);
-        Ok(raw_link)
+        let video_options = rusty_ytdl::VideoOptions {
+            quality: VideoQuality::HighestVideo,
+            ..Default::default()
+        };
+        let video = rusty_ytdl::Video::new_with_options(url.as_str(), video_options)?;
+        let stream = video.stream().await?;
+        // get one chunk and save to temp
+        let mut raw_file = NamedTempFile::new_in(&self.temp_dir)?;
+        if let Some(chunk) = stream.chunk().await? {
+            raw_file.write_all(&chunk)?;
+        }
+        raw_file.persist(&self.last_persistent_fetch_path)?;
+        info!(
+            "Raw stream is captured using rusty_ytdl to file: {}",
+            &self.last_persistent_fetch_path_str
+        );
+        Ok(self.last_persistent_fetch_path_str.to_owned())
     }
 }
 

lofigirl_sys/src/image.rs

@@ -27,7 +27,7 @@ impl ImageProcessor {
         let low_bounds = Mat::from_slice(&[200, 200, 200])?;
         let high_bounds = Mat::from_slice(&[255, 255, 255])?;
         let ocr = LepTess::new(None, "eng")?;
-        let link_capturer = YoutubeLinkCapturer::new();
+        let link_capturer = YoutubeLinkCapturer::new()?;
         Ok(ImageProcessor {
             link_capturer,
             video_url,

lofigirl_web_client/Cargo.toml

@@ -14,9 +14,9 @@ wasm-bindgen-test = "0.3"
 
 [dependencies]
 anyhow = "1.0"
-gloo-console = "0.2"
+gloo-console = "0.3"
 gloo-net = "0.4"
-gloo-storage = "0.2"
+gloo-storage = "0.3"
 seed = "0.10"
 lofigirl_shared_common = { path = "../lofigirl_shared_common"}