Merge pull request #98 from madiele/main

Update deps

.github/dependabot.yml

@@ -7,9 +7,19 @@ version: 2
 updates:
   - package-ecosystem: "cargo" # See documentation for possible values
     directory: "/" # Location of package manifests
+    allow: 
+      - dependency-type: "direct"
     schedule:
-      interval: "daily"
+      interval: "weekly"
+    groups:
+      rust-dependencies:
+        patterns:
+          - "*"
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
       interval: "daily"
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly

CONTRIBUTING.md

@@ -11,6 +11,18 @@ When adding a feature try to also add some test if possible.
 
 All pull request should point to the main branch and should pass all tests (`make test`)
 
+## How does it work?
+
+vod2pod-rss under the hood is a service that takes an hosted RSS feed (public feed or generated by a service reggistered in it's docker-compose.yml)
+and adapts it by rewriting every media url to point to the trancode endpoint of vod2pod-rss webserver, you can actually give it any
+RSS podcast feed and if ffmpeg can transcode the contained media URLS then it will work (you would need to add the domain to the whitelist, check the README for more info). 
+
+If that is not enough and you want to add a new media provider that need some API calls to work just check the provider folder
+and implement the MediaProvider trait (more info are in the module), then feel free to open a PR.
+
+also check the [podtube](https://github.com/amckee/PodTube) repo if the provider you need is already offered by it, if it does you just need to
+write the provider to point to the right podtube feed urls
+
 ## Initial setup
 
 ### github spaces development setup

Cargo.toml

@@ -12,31 +12,31 @@ name = "app"
 path = "src/main.rs"
 
 [dependencies]
-actix-rt = "2.8.0"
-actix-web = "4.3.1"
-async-trait = "0.1.68"
+actix-rt = "=2.9.0"
+actix-web = "=4.3.1"
+async-trait = "=0.1.73"
 url = { version="*", features = ["serde"]}
-futures = "0.3.28"
-log = "0.4.17"
-regex = "1.7.3"
-reqwest = { version = "0.11.16", features = ["json"] }
-serde = "1.0.159"
-serde_json = "1.0.95"
-tokio = { version = "1.27.0" , features = ["macros", "process"]}
-uuid = { version= "1.3.0", features = ["v4", "serde"]}
-genawaiter = {version = "0.99", features = ["futures03"] }
-rss = { version = "2.0", features = ["serde"] }
-eyre = "0.6"
-simple_logger = "4.1"
-redis = { version = "0.23", features = ["tokio-comp"] }
-feed-rs = "1.3.0"
-md5 = "0.7.0"
-mime = "0.3.17"
-cached = { version = "0.44.0", features = ["redis_tokio"] }
-iso8601-duration = "0.2.0"
-chrono = "0.4.24"
+futures = "=0.3.28"
+log = "=0.4.20"
+regex = "=1.9.4"
+reqwest = { version = "=0.11.20", features = ["json"] }
+serde = "=1.0.188"
+serde_json = "=1.0.105"
+tokio = { version = "=1.32.0" , features = ["macros", "process"]}
+uuid = { version= "=1.4.1", features = ["v4", "serde"]}
+genawaiter = {version = "=0.99", features = ["futures03"] }
+rss = { version = "=2.0", features = ["serde"] }
+eyre = "=0.6"
+simple_logger = "=4.2"
+redis = { version = "=0.23", features = ["tokio-comp"] }
+feed-rs = "=1.3.0"
+md5 = "=0.7.0"
+mime = "=0.3.17"
+cached = { version = "=0.44.0", features = ["redis_tokio"] }
+iso8601-duration = "=0.2.0"
+chrono = "=0.4.26"
 
 [dev-dependencies]
-temp-env ={ version = "0.3.4", features = ["async_closure"] }
+temp-env ={ version = "=0.3.4", features = ["async_closure"] }
 env_logger = "*"
 test-log = "0.2.11"

Dockerfile

@@ -31,7 +31,7 @@ ARG TARGETPLATFORM
 COPY requirements.txt ./
 RUN apt-get update && \
     apt-get install -y --no-install-recommends python3 curl ca-certificates ffmpeg && \
-    export YT_DLP_VERSION=$(cat requirements.txt | grep yt-dlp | cut -d "=" -f3) && \
+    export YT_DLP_VERSION=$(cat requirements.txt | grep yt-dlp | cut -d "=" -f3 | awk -F. '{printf "%d.%02d.%02d\n", $1, $2, $3}') && \
     curl -L https://github.com/yt-dlp/yt-dlp/releases/download/$YT_DLP_VERSION/yt-dlp -o /usr/local/bin/yt-dlp && \
     chmod a+rx /usr/local/bin/yt-dlp && \
     apt-get -y purge curl && \

README.md

@@ -97,6 +97,11 @@ You can set the following environment variables:
 - `SUBFOLDER`: Set the the root path of the app, useful for reverse proxies (default: "/")
 - `VALID_URL_DOMAINS`: (optional) Set a comma separated list of domains urls that are allowed to be converted into RSS  (defaults to yotube and twitch urls)
 
+## Honorable mentions
+
+* Youtube support is possible thanks to the cool [podtube fork project by amckee](https://github.com/amckee/PodTube) consider dropping him a star.
+* Twitch support is possible thanks to [my fork](https://github.com/madiele/TwitchToPodcastRSS) of [lzeke0's TwitchRSS](https://github.com/lzeke0/TwitchRSS) drop a star to him too!
+
 ## Donations
 
 This is a passion project, and mostly made for personal use, but if you want to gift a pizza margherita, feel free!

requirements.txt

@@ -1,2 +1,2 @@
 # this is only used to track the version in the Dockerfile and with depend-a-bot
-yt-dlp==2023.03.04
+yt-dlp==2023.7.6

src/configs/mod.rs

@@ -23,6 +23,7 @@ pub enum ConfName {
     SubfolderPath,
     ValidUrlDomains,
     AudioCodec,
+    PeerTubeValidHosts,
 }
 
 struct EnvConf { }
@@ -60,7 +61,7 @@ impl Conf for EnvConf {
                 }
                 Ok(folder)
             },
-            ConfName::ValidUrlDomains => Ok(std::env::var("VALID_URL_DOMAINS").unwrap_or_else(|_| "https://*.youtube.com/,https://youtube.com/,https://youtu.be/,https://*.youtu.be/,https://*.twitch.tv/,https://twitch.tv/,https://*.googlevideo.com/,https://*.cloudfront.net/".to_string())),
+            ConfName::ValidUrlDomains => Ok(std::env::var("VALID_URL_DOMAINS").unwrap_or_else(|_| "".to_string())),
             ConfName::AudioCodec => Ok(std::env::var("AUDIO_CODEC").map(|c| {
                 match c.as_str() {
                     "MP3" => c,
@@ -74,6 +75,7 @@ impl Conf for EnvConf {
                     }
                 }
             }).unwrap_or_else(|_| "MP3".to_string())),
+            ConfName::PeerTubeValidHosts => Ok(std::env::var("PEERTUBE_VALID_DOMAINS").unwrap_or_else(|_| "".to_string())),
         }
     }
 }

src/lib.rs

@@ -1,7 +1,7 @@
 pub mod transcoder;
 pub mod rss_transcodizer;
-pub mod url_convert;
 pub mod configs;
+pub mod provider;
 
 use actix_web::dev::Server;
 use actix_web::HttpResponse;

src/main.rs

@@ -6,7 +6,7 @@ use serde::Deserialize;
 use simple_logger::SimpleLogger;
 use std::{collections::HashMap, time::Instant};
 use vod2pod_rss::{
-    transcoder::{ Transcoder, FfmpegParameters }, rss_transcodizer::RssTranscodizer, url_convert, configs::{Conf, conf, ConfName },
+    transcoder::{ Transcoder, FfmpegParameters }, rss_transcodizer::RssTranscodizer, configs::{Conf, conf, ConfName }, provider,
 };
 
 #[actix_web::main]
@@ -97,17 +97,19 @@ async fn transcodize_rss(
         Err(e) => return HttpResponse::BadRequest().body(e.to_string()),
     };
 
-    if !url_convert::check_if_in_whitelist(&parsed_url) {
+    let provider = provider::from(&parsed_url);
+
+    if !provider.domain_whitelist_regexes().iter().any(|r| r.is_match(&parsed_url.to_string())) {
         error!("supplied url ({parsed_url}) not in whitelist (whitelist is needed to prevent SSRF attack)");
         return HttpResponse::Forbidden().body("scheme and host not in whitelist");
     }
 
-    let converted_url = match url_convert::from(parsed_url).to_feed_url().await {
+    let converted_url = match provider.feed_url().await {
         Ok(x) => x,
         Err(e) => {error!("fail when trying to convert channel {e}"); return HttpResponse::BadRequest().body(e.to_string())},
     };
 
-    let rss_transcodizer = RssTranscodizer::new(converted_url, transcode_service_url, should_transcode).await;
+    let rss_transcodizer = RssTranscodizer::new(converted_url, transcode_service_url, should_transcode);
 
     let body = match rss_transcodizer.transcodize().await {
         Ok(body) => body,
@@ -173,7 +175,9 @@ async fn transcode_to_mp3(
         }
     }
 
-    if !url_convert::check_if_in_whitelist(&stream_url) {
+    let provider = provider::from(&stream_url);
+
+    if !provider.domain_whitelist_regexes().iter().any(|r| r.is_match(&stream_url.to_string())) {
         error!("supplied url ({stream_url}) not in whitelist (whitelist is needed to prevent SSRF attack)");
         return HttpResponse::Forbidden().body("scheme and host not in whitelist");
     }

src/provider/generic.rs

@@ -0,0 +1,54 @@
+use async_trait::async_trait;
+use feed_rs::model::Entry;
+use regex::Regex;
+use reqwest::Url;
+
+use crate::configs::{conf, Conf, ConfName};
+
+use super::MediaProvider;
+
+pub(super) struct GenericProvider {
+    url: Url
+}
+
+#[async_trait]
+impl MediaProvider for GenericProvider {
+    fn new(url: &Url) -> Self { GenericProvider { url: url.clone() } }
+
+    async fn get_item_duration(&self, _url: &Url) -> eyre::Result<Option<u64>> { Ok(None) }
+
+    async fn get_stream_url(&self, media_url: &Url) -> eyre::Result<Url> {
+        Ok(media_url.to_owned())
+    }
+
+    async fn filter_item(&self, _rss_item: &Entry) -> bool { false }
+
+    fn media_url_regexes(&self) -> Vec<Regex> {
+        let generic_whitelist = get_generic_whitelist();
+
+        #[cfg(not(test))]
+        return generic_whitelist;
+        #[cfg(test)] //this will allow test to use localhost ad still work
+        return [generic_whitelist, vec!(regex::Regex::new(r"^http://127\.0\.0\.1:9872").unwrap())].concat();
+    }
+
+    fn domain_whitelist_regexes(&self) -> Vec<Regex> { get_generic_whitelist() }
+
+    async fn feed_url(&self) -> eyre::Result<Url> { Ok(self.url.clone()) }
+}
+
+fn get_generic_whitelist() -> Vec<Regex> {
+    let binding = conf().get(ConfName::ValidUrlDomains).unwrap();
+    let patterns: Vec<&str> = binding.split(",").filter(|e| e.trim().len() > 0).collect();
+
+    let mut regexes: Vec<Regex> = Vec::with_capacity(patterns.len() + 1);
+    for pattern in patterns {
+        regexes.push(Regex::new(&pattern.to_string().replace(".", "\\.").replace("*", ".+")).unwrap())
+    }
+
+    let match_extensions = regex::Regex::new("^(https?://)?.+\\.(mp3|mp4|wav|avi|mov|flv|wmv|mkv|aac|ogg|webm|3gp|3g2|asf|m4a|mpg|mpeg|ts|m3u|m3u8|pls)$").unwrap();
+    regexes.push(match_extensions);
+
+    return regexes;
+}
+