Skip to content

Commit

Permalink
Allow ytdlp parser and app to work with future versions of ytdlp
Browse files Browse the repository at this point in the history
  • Loading branch information
@williamyang98
williamyang98 committed Jul 24, 2024
1 parent f9f05be commit e825722
Show file tree
Hide file tree
Showing 11 changed files with 322 additions and 453 deletions.
7 changes: 4 additions & 3 deletions scripts/download_ubuntu.sh
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
#!/bin/sh
# Install yt-dlp
mkdir ./bin/
curl -fLo ./bin/yt-dlp https://github.com/yt-dlp/yt-dlp/releases/download/2024.05.27/yt-dlp_linux
cd ./bin/
curl -fLo ./yt-dlp https://github.com/yt-dlp/yt-dlp/releases/download/2024.07.16/yt-dlp_linux
set -e
echo af0570b5e60196a1785a12e7f48fc7cb7b5745b0bb9870ca2fe6ed90ddd80b46 ./bin/yt-dlp | sha256sum --check
sudo chmod 777 ./bin/yt-dlp
echo a6b840e536014ce7b2c7c40b758080498ed5054aa96979e64fcc369752cdc8d3 yt-dlp | sha256sum --check
sudo chmod 777 ./yt-dlp
# Install ffmpeg
sudo apt-get --yes install ffmpeg
4 changes: 2 additions & 2 deletions scripts/download_win32.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
# Download files
mkdir ./bin/
cd ./bin
curl -fLo ./yt-dlp.exe https://github.com/yt-dlp/yt-dlp/releases/download/2024.05.27/yt-dlp.exe
curl -fLo ./yt-dlp.exe https://github.com/yt-dlp/yt-dlp/releases/download/2024.07.16/yt-dlp.exe &
curl -fLo ./ffmpeg.7z https://github.com/GyanD/codexffmpeg/releases/download/7.0.1/ffmpeg-7.0.1-essentials_build.7z &
wait
# Verify hash
set -e
echo e19115321897a27c2fcf73d3b23d5139847a8c4fd4792eecce1712bfd9accd05 ffmpeg.7z | sha256sum --check
echo e96f6348244306ac999501b1e8e2b096b8a57f098c3b2b9ffe64b2107039e0ae yt-dlp.exe | sha256sum --check
echo f01b37ca4f3e934208a5439d1ec8ae49a18f2be9f68fec5e3cfed08cc38b3275 yt-dlp.exe | sha256sum --check
# Unzip
7z x ./ffmpeg.7z -offmpeg -y
cp ./ffmpeg/ffmpeg-7.0.1-essentials_build/bin/ffmpeg.exe ./ffmpeg.exe
Expand Down
212 changes: 101 additions & 111 deletions src/database.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use rusqlite::OptionalExtension;
use rusqlite::{params, OptionalExtension};
use serde::Serialize;
use num_derive::{FromPrimitive, ToPrimitive};
use num_traits::cast::{FromPrimitive, ToPrimitive};
Expand Down Expand Up @@ -84,10 +84,8 @@ impl WorkerStatus {
#[derive(Debug, Clone, Serialize)]
pub struct YtdlpRow {
pub video_id: VideoId,
pub audio_ext: AudioExtension,
pub status: WorkerStatus,
pub unix_time: u64,
pub infojson_path: Option<String>,
pub stdout_log_path: Option<String>,
pub stderr_log_path: Option<String>,
pub system_log_path: Option<String>,
Expand All @@ -113,15 +111,13 @@ pub fn setup_database(conn: DatabaseConnection) -> Result<(), Box<dyn std::error
conn.execute(
"CREATE TABLE IF NOT EXISTS ytdlp (
video_id TEXT,
audio_ext TEXT,
status INTEGER DEFAULT 0,
unix_time INTEGER,
infojson_path TEXT,
stdout_log_path TEXT,
stderr_log_path TEXT,
system_log_path TEXT,
audio_path TEXT,
PRIMARY KEY (video_id, audio_ext)
PRIMARY KEY (video_id)
)",
(),
)?;
Expand All @@ -143,151 +139,120 @@ pub fn setup_database(conn: DatabaseConnection) -> Result<(), Box<dyn std::error
}

#[derive(Debug,Clone,Copy)]
pub enum WorkerTable {
YTDLP,
FFMPEG,
enum WorkerTable {
Ytdlp,
Ffmpeg,
}

generate_bidirectional_binding!(
WorkerTable, &'static str, &str,
(YTDLP, "ytdlp"),
(FFMPEG, "ffmpeg"),
(Ytdlp, "ytdlp"),
(Ffmpeg, "ffmpeg"),
);

pub fn insert_worker_entry(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension, table: WorkerTable,
// insert
pub fn insert_ytdlp_entry(
db_conn: &DatabaseConnection, video_id: &VideoId,
) -> Result<usize, rusqlite::Error> {
let table: &'static str = table.into();
let table: &'static str = WorkerTable::Ytdlp.into();
db_conn.execute(
format!("INSERT OR REPLACE INTO {table} (video_id, audio_ext, status, unix_time) VALUES (?1,?2,?3,?4)").as_str(),
(video_id.as_str(), audio_ext.as_str(), WorkerStatus::Queued as u8, get_unix_time()),
format!("INSERT OR REPLACE INTO {table} (video_id, status, unix_time) VALUES (?1,?2,?3)").as_str(),
(video_id.as_str(), WorkerStatus::Queued as u8, get_unix_time()),
)
}

pub fn update_worker_status(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension, status: WorkerStatus, table: WorkerTable,
pub fn insert_ffmpeg_entry(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension,
) -> Result<usize, rusqlite::Error> {
let table: &'static str = table.into();
let table: &'static str = WorkerTable::Ffmpeg.into();
db_conn.execute(
format!("UPDATE {table} SET status=?3, unix_time=?4 WHERE video_id=?1 AND audio_ext=?2").as_str(),
(video_id.as_str(), audio_ext.as_str(), status.to_u8(), get_unix_time()),
format!("INSERT OR REPLACE INTO {table} (video_id, audio_ext, status, unix_time) VALUES (?1,?2,?3,?4)").as_str(),
(video_id.as_str(), audio_ext.as_str(), WorkerStatus::Queued as u8, get_unix_time()),
)
}

pub fn update_worker_fields(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension,
table: WorkerTable, fields: &[&str], values: &[&(dyn rusqlite::ToSql)],
// update
pub fn update_ytdlp_entry(
db_conn: &DatabaseConnection, entry: &YtdlpRow,
) -> Result<usize, rusqlite::Error> {
use std::fmt::Write;
let mut query_string = String::with_capacity(128);
let table: &'static str = table.into();
write!(&mut query_string, "UPDATE {table} SET ").unwrap();
for (i, field) in fields.iter().enumerate() {
write!(&mut query_string, "{0}=?{1}", field, i+3).unwrap();
if i < (fields.len()-1) {
query_string.push(',');
}
}
write!(&mut query_string, " WHERE video_id=?1 AND audio_ext=?2").unwrap();

let mut params = Vec::<Box::<dyn rusqlite::ToSql>>::with_capacity(2 + values.len());
params.push(Box::new(video_id.as_str()));
params.push(Box::new(audio_ext.as_str()));
for v in values {
params.push(Box::new(*v));
}
db_conn.execute(query_string.as_str(), rusqlite::params_from_iter(params.iter()))
let table: &'static str = WorkerTable::Ytdlp.into();
db_conn.execute(
format!(
"UPDATE {table} SET \
unix_time=?2, status=?3, \
stdout_log_path=?4, stderr_log_path=?5, system_log_path=?6, audio_path=?7 \
WHERE video_id=?1"
).as_str(),
params![
entry.video_id.as_str(),
entry.unix_time, entry.status.to_u8(),
entry.stdout_log_path, entry.stderr_log_path, entry.system_log_path, entry.audio_path,
],
)
}

#[derive(Debug)]
pub enum StatusFetchError {
DatabasePrepare(rusqlite::Error),
DatabaseQuery(rusqlite::Error),
InvalidEnumValue(u8),
MissingValue,
pub fn update_ffmpeg_entry(
db_conn: &DatabaseConnection, entry: &FfmpegRow,
) -> Result<usize, rusqlite::Error> {
let table: &'static str = WorkerTable::Ffmpeg.into();
db_conn.execute(
format!(
"UPDATE {table} SET \
unix_time=?3, status=?4, stdout_log_path=?5, stderr_log_path=?6, system_log_path=?7, audio_path=?8 \
WHERE video_id=?1 AND audio_ext=?2"
).as_str(),
params![
entry.video_id.as_str(), entry.audio_ext.as_str(),
entry.unix_time, entry.status.to_u8(),
entry.stdout_log_path, entry.stderr_log_path, entry.system_log_path, entry.audio_path,
],
)
}

pub fn select_worker_status(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension, table: WorkerTable,
) -> Result<WorkerStatus, StatusFetchError> {
let table: &'static str = table.into();
let mut select_query = db_conn.prepare(format!("SELECT status FROM {table} WHERE video_id=?1 AND audio_ext=?2").as_str())
.map_err(StatusFetchError::DatabasePrepare)?;
let status: Option<u8> = select_query.query_row([video_id.as_str(), audio_ext.as_str()], |row| row.get(0))
.map_err(StatusFetchError::DatabaseQuery)?;
let Some(status) = status else {
return Err(StatusFetchError::MissingValue);
};
let Some(status) = WorkerStatus::from_u8(status) else {
return Err(StatusFetchError::InvalidEnumValue(status));
};
Ok(status)
// delete
pub fn delete_ytdlp_entry(db_conn: &DatabaseConnection, video_id: &VideoId) -> Result<usize, rusqlite::Error> {
let table: &'static str = WorkerTable::Ytdlp.into();
db_conn.execute(format!("DELETE FROM {table} WHERE video_id=?1").as_str(), (video_id.as_str(),))
}

pub fn select_worker_fields<T, F>(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension, table: WorkerTable,
fields: &[&str], transform: F,
) -> Result<Option<T>, rusqlite::Error>
where F: FnOnce(&rusqlite::Row<'_>) -> Result<T, rusqlite::Error>,
{
use std::fmt::Write;
let table: &'static str = table.into();
let mut query = String::with_capacity(128);
query.push_str("SELECT ");
for (i, field) in fields.iter().enumerate() {
query.push_str(field);
if i < (fields.len()-1) {
query.push(',');
}
}
write!(&mut query, " FROM {table} WHERE video_id=?1 AND audio_ext=?2").expect("Query builder shouldn't fail");
let mut select_query = db_conn.prepare(query.as_str())?;
select_query.query_row([video_id.as_str(), audio_ext.as_str()], transform).optional()
}

pub fn delete_worker_entry(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension, table: WorkerTable,
pub fn delete_ffmpeg_entry(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension,
) -> Result<usize, rusqlite::Error> {
let table: &'static str = table.into();
let table: &'static str = WorkerTable::Ffmpeg.into();
db_conn.execute(
format!("DELETE FROM {table} WHERE video_id=?1 AND audio_ext=?2").as_str(),
(video_id.as_str(), audio_ext.as_str()),
)
}

// select
fn map_ytdlp_row_to_entry(row: &rusqlite::Row) -> Result<YtdlpRow, rusqlite::Error> {
let video_id: Option<String> = row.get(0)?;
let video_id = video_id.expect("video_id is a primary key");
let video_id = VideoId::try_new(video_id.as_str()).expect("video_id should be valid");

let audio_ext: Option<String> = row.get(1)?;
let audio_ext = audio_ext.expect("audio_ext is a primary key");
let audio_ext = AudioExtension::try_from(audio_ext.as_str()).expect("audio_ext should be valid");

let status: Option<u8> = row.get(2)?;
let status: Option<u8> = row.get(1)?;
let status = status.expect("status should be present");
let status = WorkerStatus::from_u8(status).expect("status should be valid");

let unix_time: Option<u64> = row.get(3)?;
let unix_time: Option<u64> = row.get(2)?;
let unix_time = unix_time.unwrap_or(0);

Ok(YtdlpRow {
video_id,
audio_ext,
status,
unix_time,
infojson_path: row.get(4)?,
stdout_log_path: row.get(5)?,
stderr_log_path: row.get(6)?,
system_log_path: row.get(7)?,
audio_path: row.get(8)?,
stdout_log_path: row.get(3)?,
stderr_log_path: row.get(4)?,
system_log_path: row.get(5)?,
audio_path: row.get(6)?,
})
}

pub fn select_ytdlp_entries(db_conn: &DatabaseConnection) -> Result<Vec<YtdlpRow>, rusqlite::Error> {
let table: &'static str = WorkerTable::YTDLP.into();
let table: &'static str = WorkerTable::Ytdlp.into();
let mut stmt = db_conn.prepare(format!(
"SELECT video_id, audio_ext, status, unix_time, infojson_path,\
"SELECT video_id, status, unix_time,\
stdout_log_path, stderr_log_path, system_log_path, audio_path FROM {table}").as_str())?;
let row_iter = stmt.query_map([], map_ytdlp_row_to_entry)?;
let mut entries = Vec::<YtdlpRow>::new();
Expand All @@ -297,15 +262,13 @@ pub fn select_ytdlp_entries(db_conn: &DatabaseConnection) -> Result<Vec<YtdlpRow
Ok(entries)
}

pub fn select_ytdlp_entry(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension,
) -> Result<Option<YtdlpRow>, rusqlite::Error> {
let table: &'static str = WorkerTable::YTDLP.into();
pub fn select_ytdlp_entry(db_conn: &DatabaseConnection, video_id: &VideoId) -> Result<Option<YtdlpRow>, rusqlite::Error> {
let table: &'static str = WorkerTable::Ytdlp.into();
let mut stmt = db_conn.prepare(format!(
"SELECT video_id, audio_ext, status, unix_time, infojson_path, \
"SELECT video_id, status, unix_time, \
stdout_log_path, stderr_log_path, system_log_path, audio_path \
FROM {table} WHERE video_id=?1 AND audio_ext=?2").as_str())?;
stmt.query_row([video_id.as_str(), audio_ext.as_str()], map_ytdlp_row_to_entry).optional()
FROM {table} WHERE video_id=?1").as_str())?;
stmt.query_row([video_id.as_str()], map_ytdlp_row_to_entry).optional()
}

fn map_ffmpeg_row_to_entry(row: &rusqlite::Row) -> Result<FfmpegRow, rusqlite::Error> {
Expand Down Expand Up @@ -337,7 +300,7 @@ fn map_ffmpeg_row_to_entry(row: &rusqlite::Row) -> Result<FfmpegRow, rusqlite::E
}

pub fn select_ffmpeg_entries(db_conn: &DatabaseConnection) -> Result<Vec<FfmpegRow>, rusqlite::Error> {
let table: &'static str = WorkerTable::FFMPEG.into();
let table: &'static str = WorkerTable::Ffmpeg.into();
let mut stmt = db_conn.prepare(format!(
"SELECT video_id, audio_ext, status, unix_time,\
stdout_log_path, stderr_log_path, system_log_path, audio_path FROM {table}").as_str())?;
Expand All @@ -353,10 +316,37 @@ pub fn select_ffmpeg_entries(db_conn: &DatabaseConnection) -> Result<Vec<FfmpegR
pub fn select_ffmpeg_entry(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension,
) -> Result<Option<FfmpegRow>, rusqlite::Error> {
let table: &'static str = WorkerTable::FFMPEG.into();
let table: &'static str = WorkerTable::Ffmpeg.into();
let mut stmt = db_conn.prepare(format!(
"SELECT video_id, audio_ext, status, unix_time,\
stdout_log_path, stderr_log_path, system_log_path, audio_path \
FROM {table} WHERE video_id=?1 AND audio_ext=?2").as_str())?;
stmt.query_row([video_id.as_str(), audio_ext.as_str()], map_ffmpeg_row_to_entry).optional()
}

// select and update
pub fn select_and_update_ytdlp_entry<F>(
db_conn: &DatabaseConnection, video_id: &VideoId, callback: F,
) -> Result<usize, rusqlite::Error>
where F: FnOnce(&mut YtdlpRow)
{
let entry = select_ytdlp_entry(db_conn, video_id)?;
let Some(mut entry) = entry else {
return Ok(0);
};
callback(&mut entry);
update_ytdlp_entry(db_conn, &entry)
}

pub fn select_and_update_ffmpeg_entry<F>(
db_conn: &DatabaseConnection, video_id: &VideoId, audio_ext: AudioExtension, callback: F,
) -> Result<usize, rusqlite::Error>
where F: FnOnce(&mut FfmpegRow)
{
let entry = select_ffmpeg_entry(db_conn, video_id, audio_ext)?;
let Some(mut entry) = entry else {
return Err(rusqlite::Error::QueryReturnedNoRows);
};
callback(&mut entry);
update_ffmpeg_entry(db_conn, &entry)
}
Loading

0 comments on commit e825722

Please sign in to comment.