From 39cfa998b93eb8973491abebd79da19012395270 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 27 Jan 2024 16:14:47 +0100 Subject: [PATCH] GH-85: Add support for new mumble proto - Add timestamp and uid to text messages - Add multi profile option - Add compression for audio --- .gitignore | 1 + public/locales/en/audio.json | 6 +- src-tauri/Cargo.toml | 1 + src-tauri/build.rs | 18 ++++- src-tauri/src/commands/mod.rs | 3 +- src-tauri/src/commands/settings_cmd.rs | 20 ++--- src-tauri/src/commands/utils/settings.rs | 9 +++ src-tauri/src/connection/mod.rs | 7 +- .../src/connection/threads/input_thread.rs | 1 - .../src/connection/threads/output_thread.rs | 2 + .../src/connection/threads/ping_thread.rs | 7 +- src-tauri/src/main.rs | 2 +- src-tauri/src/manager/channel.rs | 8 +- src-tauri/src/manager/connection_state.rs | 8 +- src-tauri/src/manager/text_message.rs | 23 ++++-- src-tauri/src/manager/user.rs | 31 ++++---- src-tauri/src/manager/voice.rs | 12 ++- src-tauri/src/proto/Mumble.proto.patch | 11 +++ src-tauri/src/protocol/message_router.rs | 13 +++- src-tauri/src/protocol/mod.rs | 12 +-- src-tauri/src/protocol/serialize/mod.rs | 4 +- src-tauri/src/utils/audio/decoder.rs | 35 ++++++++- src-tauri/src/utils/audio/encoder.rs | 29 +++++--- .../src/utils/audio/processing/compress.rs | 68 +++++++++++++++++ src-tauri/src/utils/audio/processing/mod.rs | 1 + src-tauri/src/utils/audio/recorder.rs | 56 +++++++------- src-tauri/src/utils/certificate_store.rs | 24 +++--- src-tauri/src/utils/frontend/mod.rs | 8 +- src-tauri/src/utils/mod.rs | 2 +- src-tauri/src/utils/server.rs | 2 +- src/components/ChatMessage.tsx | 7 +- src/components/settings/Audio.tsx | 73 ++++++++++++++++++- src/store/features/users/audioSettings.ts | 42 ++++++++++- src/store/features/users/chatMessageSlice.ts | 4 +- 34 files changed, 420 insertions(+), 130 deletions(-) create mode 100644 src-tauri/src/utils/audio/processing/compress.rs diff --git a/.gitignore b/.gitignore index a22c351..b7b4010 100644 --- a/.gitignore +++ b/.gitignore @@ -107,4 +107,5 @@ dist-ssr src-tauri/data src-tauri/src/proto/Mumble.proto +src-tauri/src/proto/MumbleUDP.proto src-tauri/gen \ No newline at end of file diff --git a/public/locales/en/audio.json b/public/locales/en/audio.json index 829b69a..2033c8a 100644 --- a/public/locales/en/audio.json +++ b/public/locales/en/audio.json @@ -11,5 +11,9 @@ "Audio deactivation at": "Audio deactivation at {{threshold}}", "Amplification dB": "Amplification +{{amplification}}dB", "Echo Cancelation": "Echo Cancelation", - "Noise Suppression": "Noise Suppression" + "Noise Suppression": "Noise Suppression", + "Compressor Threshold": "Compressor Threshold {{threshold}}dB", + "Compressor Ratio": "Compressor Ratio {{ratio}}:1", + "Attack Time": "Attack Time {{duration}}", + "Release Time": "Release Time {{duration}}" } \ No newline at end of file diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index db05001..21f8ea7 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -46,6 +46,7 @@ scraper = "0.18.1" tauri-plugin-window-state = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v1" } symphonia = "0.5.3" mime_guess = "2.0.4" +uuid = "1.7.0" [dev-dependencies] tempfile = "3.5.0" diff --git a/src-tauri/build.rs b/src-tauri/build.rs index 734cea3..0e8e933 100644 --- a/src-tauri/build.rs +++ b/src-tauri/build.rs @@ -13,6 +13,10 @@ const MUMBLE_PROTO_SHA256: &str = "0f86d85938ff2268e3eb05ce0120805fb049ad0d062f4d01c6657b048dcc9245"; const PATCHED_MUMBLE_PROTO_HASH: &str = "ebadea7bcb720da05149076b1b0ec7a9ff1107a5107a4137b75e8e45fb52f68d"; +const DOWNLOAD_MUMBLE_UDP_PROTO_DIR: &str = + "https://raw.githubusercontent.com/mumble-voip/mumble/6a48c0478477054b4e7356b0bd7dc9da24cf0880/src/MumbleUDP.proto"; +const MUMBLE_UDP_PROTO_SHA256: &str = + "8087983b0d9a12e11380cad99870a0ef3cee7550b13a114a733aa835acd3d040"; fn apply(diff: Patch, old: &str) -> String { let old_lines = old.lines().collect::>(); @@ -81,6 +85,7 @@ async fn download_file( fn main() -> io::Result<()> { let mumble_proto = Path::new("src/proto/Mumble.proto"); + let mumble_udp_proto = Path::new("src/proto/MumbleUDP.proto"); let patch_file = Path::new("src/proto/Mumble.proto.patch"); let mumble_proto_bytes = read_file_as_bytes(mumble_proto).unwrap_or_default(); @@ -90,20 +95,29 @@ fn main() -> io::Result<()> { if hash != PATCHED_MUMBLE_PROTO_HASH { let rt = Runtime::new()?; rt.block_on(async { - let resonse_file = + let response_file = download_file(DOWNLOAD_MUMBLE_PROTO_DIR, MUMBLE_PROTO_SHA256, mumble_proto) .await .expect("Failed to download Mumble.proto"); - let response_str = str::from_utf8(&resonse_file).expect("Failed to parse response"); + let response_str = str::from_utf8(&response_file).expect("Failed to parse response"); let patch_output = read_file_as_bytes(patch_file).expect("Failed to read file"); let patch = Patch::from_single(patch_output.as_str()).expect("Failed to parse patch"); let new_content = apply(patch, response_str); write_to_file(new_content.as_bytes(), mumble_proto); + + download_file( + DOWNLOAD_MUMBLE_UDP_PROTO_DIR, + MUMBLE_UDP_PROTO_SHA256, + mumble_udp_proto, + ) + .await + .expect("Failed to download MumbleUDP.proto"); }); } prost_build::compile_protos(&["src/proto/Mumble.proto"], &["src/"])?; + prost_build::compile_protos(&["src/proto/MumbleUDP.proto"], &["src/"])?; tauri_build::build(); Ok(()) diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 918b602..e784fe8 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -161,11 +161,12 @@ pub async fn logout(state: State<'_, ConnectionState>) -> Result<(), String> { #[tauri::command] pub async fn like_message( message_id: String, + reciever: Vec, state: State<'_, ConnectionState>, ) -> Result<(), String> { let guard = state.connection.lock().await; if let Some(guard) = guard.as_ref() { - if let Err(e) = guard.like_message(&message_id) { + if let Err(e) = guard.like_message(&message_id, reciever) { return Err(format!("{e:?}")); } } diff --git a/src-tauri/src/commands/settings_cmd.rs b/src-tauri/src/commands/settings_cmd.rs index d442297..cffb459 100644 --- a/src-tauri/src/commands/settings_cmd.rs +++ b/src-tauri/src/commands/settings_cmd.rs @@ -4,11 +4,13 @@ use std::{ sync::RwLock, }; -use reqwest::Identity; use tauri::State; use tracing::{info, trace}; -use crate::{utils::{constants::get_project_dirs, server::{Server, UserIdentity}}, errors::certificate_error::CertificateError}; +use crate::{ + errors::certificate_error::CertificateError, + utils::{constants::get_project_dirs, server::Server}, +}; use super::utils::settings::FrontendSettings; @@ -168,24 +170,24 @@ pub fn get_frontend_settings( Ok(settings_data) } - #[tauri::command] pub fn get_identity_certs() -> Result, String> { let project_dirs = get_project_dirs() - .ok_or_else(|| CertificateError::new("Unable to load project dir")).map_err(|e| format!("{:?}", e))?; + .ok_or_else(|| CertificateError::new("Unable to load project dir")) + .map_err(|e| format!("{e:?}"))?; let data_dir = project_dirs.data_dir(); if !data_dir.exists() { - std::fs::create_dir_all(&data_dir).map_err(|e| format!("{:?}", e))?; + std::fs::create_dir_all(data_dir).map_err(|e| format!("{e:?}"))?; } let mut certs = Vec::new(); - let dir_entries = fs::read_dir(&data_dir) - .map_err(|e| format!("Error reading directory: {}", e))?; + let dir_entries = + fs::read_dir(data_dir).map_err(|e| format!("Error reading directory: {e}"))?; for entry in dir_entries { - let entry = entry.map_err(|e| format!("Error reading directory entry: {}", e))?; + let entry = entry.map_err(|e| format!("Error reading directory entry: {e}"))?; let file_name = entry.file_name(); let file_name_str = file_name.to_string_lossy(); @@ -199,4 +201,4 @@ pub fn get_identity_certs() -> Result, String> { } Ok(certs) -} \ No newline at end of file +} diff --git a/src-tauri/src/commands/utils/settings.rs b/src-tauri/src/commands/utils/settings.rs index af8728b..8b7a519 100644 --- a/src-tauri/src/commands/utils/settings.rs +++ b/src-tauri/src/commands/utils/settings.rs @@ -31,6 +31,14 @@ pub struct VoiceActivationOptions { pub voice_hysteresis_upper_threshold: f32, } +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CompressorOptions { + pub attack_time: usize, + pub release_time: usize, + pub threshold: f32, + pub ratio: f32, +} + #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] pub enum InputMode { VoiceActivation = 0, @@ -42,6 +50,7 @@ pub struct AudioOptions { pub amplification: f32, pub input_mode: InputMode, pub voice_activation_options: Option, + pub compressor_options: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/src-tauri/src/connection/mod.rs b/src-tauri/src/connection/mod.rs index 0367802..d1797cc 100644 --- a/src-tauri/src/connection/mod.rs +++ b/src-tauri/src/connection/mod.rs @@ -29,6 +29,7 @@ use self::threads::ConnectionThread; const QUEUE_SIZE: usize = 256; const BUFFER_SIZE: usize = 8192; +const FANCY_MUMBLE_DATA_ID: &str = "fancy_mumble"; struct ServerData { username: String, @@ -162,12 +163,12 @@ impl Connection { } //TODO: Move to output Thread - pub fn like_message(&self, message_id: &str) -> AnyError<()> { + pub fn like_message(&self, message_id: &str, reciever: Vec) -> AnyError<()> { let like_message = mumble::proto::PluginDataTransmission { sender_session: None, - receiver_sessions: Vec::new(), + receiver_sessions: reciever, data: Some(message_id.as_bytes().to_vec()), - data_id: None, + data_id: Some(FANCY_MUMBLE_DATA_ID.to_owned()), }; self.tx_out.send(message_builder(&like_message)?)?; diff --git a/src-tauri/src/connection/threads/input_thread.rs b/src-tauri/src/connection/threads/input_thread.rs index 40cbe26..4c5e944 100644 --- a/src-tauri/src/connection/threads/input_thread.rs +++ b/src-tauri/src/connection/threads/input_thread.rs @@ -1,7 +1,6 @@ use std::sync::atomic::Ordering; use crate::connection::Connection; -use crate::connection::traits::Shutdown; use crate::protocol::message_router::MessageRouter; use crate::protocol::stream_reader::StreamReader; use tokio::select; diff --git a/src-tauri/src/connection/threads/output_thread.rs b/src-tauri/src/connection/threads/output_thread.rs index d2871b7..111216d 100644 --- a/src-tauri/src/connection/threads/output_thread.rs +++ b/src-tauri/src/connection/threads/output_thread.rs @@ -34,6 +34,8 @@ impl OutputThread for Connection { channel_id: result.channel_id.iter().copied().collect(), tree_id: Vec::new(), message: result.message, + message_id: None, + timestamp: None, }; trace!("Sending message: {:?}", message); let buffer = message_builder(&message).unwrap_or_default(); diff --git a/src-tauri/src/connection/threads/ping_thread.rs b/src-tauri/src/connection/threads/ping_thread.rs index 2f7c7c9..8dc1301 100644 --- a/src-tauri/src/connection/threads/ping_thread.rs +++ b/src-tauri/src/connection/threads/ping_thread.rs @@ -1,7 +1,8 @@ use crate::{ - connection::{Connection, PingThread, threads::MAX_PING_FAILURES}, + connection::{threads::MAX_PING_FAILURES, Connection, PingThread}, mumble, - utils::{messages::message_builder, frontend::send_to_frontend}, protocol::serialize::message_container::FrontendMessage, + protocol::serialize::message_container::FrontendMessage, + utils::{frontend::send_to_frontend, messages::message_builder}, }; use std::{ sync::atomic::Ordering, @@ -67,7 +68,7 @@ impl PingThread for Connection { deadman_counter += 1; if deadman_counter > MAX_PING_FAILURES { let message = FrontendMessage::new("ping_timeout", "Timeout while sending Ping"); - send_to_frontend(&frontend_channel, &message) + send_to_frontend(&frontend_channel, &message); } }, } diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index 3e86b09..274505f 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -35,7 +35,7 @@ use crate::commands::{ enable_audio_info, get_audio_devices, like_message, logout, send_message, set_audio_input_setting, set_audio_output_setting, set_user_image, settings_cmd::{ - get_identity_certs, get_frontend_settings, get_server_list, save_frontend_settings, + get_frontend_settings, get_identity_certs, get_server_list, save_frontend_settings, save_server, }, web_cmd::{get_open_graph_data_from_website, open_browser}, diff --git a/src-tauri/src/manager/channel.rs b/src-tauri/src/manager/channel.rs index b31a176..206bd42 100644 --- a/src-tauri/src/manager/channel.rs +++ b/src-tauri/src/manager/channel.rs @@ -4,11 +4,13 @@ use std::{ }; use serde::Serialize; -use tracing::{debug, error, info}; +use tracing::{debug, info}; use crate::{ - errors::AnyError, mumble, protocol::serialize::message_container::FrontendMessage, - utils::{messages::message_builder, frontend::send_to_frontend}, + errors::AnyError, + mumble, + protocol::serialize::message_container::FrontendMessage, + utils::{frontend::send_to_frontend, messages::message_builder}, }; use super::Update; diff --git a/src-tauri/src/manager/connection_state.rs b/src-tauri/src/manager/connection_state.rs index 8c01887..a0af233 100644 --- a/src-tauri/src/manager/connection_state.rs +++ b/src-tauri/src/manager/connection_state.rs @@ -1,7 +1,6 @@ -use serde::Serialize; -use tracing::error; - -use crate::{protocol::serialize::message_container::FrontendMessage, utils::frontend::send_to_frontend}; +use crate::{ + protocol::serialize::message_container::FrontendMessage, utils::frontend::send_to_frontend, +}; use tokio::sync::broadcast::Sender; @@ -18,7 +17,6 @@ impl Manager { } } - pub fn notify_disconnected(&self, message: &Option) { let msg = FrontendMessage::new("disconnected", message); diff --git a/src-tauri/src/manager/text_message.rs b/src-tauri/src/manager/text_message.rs index 521bb51..9dd92d8 100644 --- a/src-tauri/src/manager/text_message.rs +++ b/src-tauri/src/manager/text_message.rs @@ -4,7 +4,7 @@ use serde::Serialize; use tokio::sync::broadcast::Sender; use tracing::error; -use crate::{errors::AnyError, mumble, protocol::serialize::message_container::FrontendMessage}; +use crate::{mumble, protocol::serialize::message_container::FrontendMessage}; use super::user::User; @@ -19,6 +19,7 @@ struct TextMessage { sender: SenderInfo, message: String, timestamp: u128, + id: Option, } pub struct Manager { @@ -64,21 +65,27 @@ impl Manager { self.notify(Some(last)); } - pub fn add_text_message( - &mut self, - text_message: mumble::proto::TextMessage, - user: &User, - ) -> AnyError<()> { + pub fn add_text_message(&mut self, text_message: mumble::proto::TextMessage, user: &User) { + let timestamp = text_message.timestamp.map_or_else( + || { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + }, + u128::from, + ); + let message = TextMessage { sender: SenderInfo { user_id: user.id, user_name: user.name.clone(), }, message: text_message.message, - timestamp: SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(), + timestamp, + id: text_message.message_id, }; self.message_log.push(message); self.notify_last(); - Ok(()) } } diff --git a/src-tauri/src/manager/user.rs b/src-tauri/src/manager/user.rs index bfe5028..2084eb7 100644 --- a/src-tauri/src/manager/user.rs +++ b/src-tauri/src/manager/user.rs @@ -2,7 +2,7 @@ use base64::{engine::general_purpose, Engine as _}; use std::collections::{hash_map::Entry, HashMap}; use serde::{Deserialize, Serialize}; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, info, trace}; use crate::{ errors::AnyError, @@ -10,7 +10,8 @@ use crate::{ protocol::serialize::message_container::FrontendMessage, utils::{ file::{read_data_from_cache, store_data_in_cache}, - messages::message_builder, frontend::send_to_frontend, + frontend::send_to_frontend, + messages::message_builder, }, }; @@ -66,10 +67,10 @@ pub struct UpdateableUserState { #[derive(Debug, Default, Serialize, Deserialize, Clone)] pub struct SyncInfo { - pub session: Option, - pub max_bandwidth: Option, - pub welcome_text: Option, - pub permissions: Option, + pub session: Option, + pub max_bandwidth: Option, + pub welcome_text: Option, + pub permissions: Option, } #[derive(Debug, Default, Serialize)] @@ -100,7 +101,6 @@ impl Update for User { pub struct Manager { users: HashMap, - current_user_id: Option, frontend_channel: Sender, server_channel: Sender>, } @@ -109,7 +109,6 @@ impl Manager { pub fn new(send_to: Sender, server_channel: Sender>) -> Self { Self { users: HashMap::new(), - current_user_id: None, frontend_channel: send_to, server_channel, } @@ -320,14 +319,14 @@ impl Manager { } pub fn notify_current_user(&mut self, sync_info: &mumble::proto::ServerSync) { - let sync_info = SyncInfo { - session: sync_info.session, - max_bandwidth: sync_info.max_bandwidth, - welcome_text: sync_info.welcome_text.clone(), - permissions: sync_info.permissions, - }; - let message = FrontendMessage::new("sync_info", sync_info); - send_to_frontend(&self.frontend_channel, &message); + let sync_info = SyncInfo { + session: sync_info.session, + max_bandwidth: sync_info.max_bandwidth, + welcome_text: sync_info.welcome_text.clone(), + permissions: sync_info.permissions, + }; + let message = FrontendMessage::new("sync_info", sync_info); + send_to_frontend(&self.frontend_channel, &message); } } diff --git a/src-tauri/src/manager/voice.rs b/src-tauri/src/manager/voice.rs index bad58d9..238d7e1 100644 --- a/src-tauri/src/manager/voice.rs +++ b/src-tauri/src/manager/voice.rs @@ -13,7 +13,6 @@ use async_trait::async_trait; use serde::Serialize; use std::collections::{hash_map::Entry, HashMap}; use tokio::sync::broadcast::{Receiver, Sender}; -use tracing::error; const SAMPLE_RATE: u32 = 48000; const CHANNELS: opus::Channels = opus::Channels::Mono; @@ -30,7 +29,7 @@ pub struct Manager { user_audio_info: HashMap, audio_player: Player, recoder: Recorder, - decoder: audio::decoder::Decoder, + decoder: Box, } impl Manager { @@ -63,7 +62,7 @@ impl Manager { user_audio_info: HashMap::new(), audio_player: player, recoder, - decoder: audio::decoder::Decoder::new(SAMPLE_RATE, CHANNELS), + decoder: Box::new(audio::decoder::UDPDecoder::new(SAMPLE_RATE, CHANNELS)), }) } @@ -118,6 +117,13 @@ impl Manager { Ok(()) } + + pub(crate) fn set_codec(&self, codec_version: &mumble::proto::CodecVersion) { + send_to_frontend( + &self.frontend_channel, + &FrontendMessage::new("set_codec", format!("{codec_version:?}")), + ); + } } #[async_trait] diff --git a/src-tauri/src/proto/Mumble.proto.patch b/src-tauri/src/proto/Mumble.proto.patch index bf0120f..f6a91a6 100644 --- a/src-tauri/src/proto/Mumble.proto.patch +++ b/src-tauri/src/proto/Mumble.proto.patch @@ -7,3 +7,14 @@ index c3a23f2..d7ce213 100644 optional uint64 version_v2 = 5; + // Fancy Mumble version string. + optional uint64 fancy_version = 6; +@@ -282,6 +282,10 @@ message TextMessage { + repeated uint32 tree_id = 4; + // The UTF-8 encoded message. May be HTML if the server allows. + required string message = 5; ++ // unique identifier for this message ++ optional string message_id = 6; ++ // message timestamp ++ optional uint64 timestamp = 7; + } + + message PermissionDenied { diff --git a/src-tauri/src/protocol/message_router.rs b/src-tauri/src/protocol/message_router.rs index af42390..a4e07a5 100644 --- a/src-tauri/src/protocol/message_router.rs +++ b/src-tauri/src/protocol/message_router.rs @@ -3,7 +3,7 @@ use std::error::Error; use tokio::sync::broadcast::{Receiver, Sender}; -use tracing::{error, trace, warn}; +use tracing::{error, info, trace, warn}; use crate::{ commands::utils::settings::GlobalSettings, @@ -73,7 +73,7 @@ impl MessageRouter { .user_manager .get_user_by_id(actor) .ok_or_else(|| Box::new(ApplicationError::new("msg")) as Box)?; - self.text_manager.add_text_message(text_message, actor)?; + self.text_manager.add_text_message(text_message, actor); } None => { error!("Received text message without actor: {:?}", text_message); @@ -152,12 +152,17 @@ impl MessageRouter { //self.shutdown().await?; } } - crate::utils::messages::MessageTypes::CodecVersion => {} + crate::utils::messages::MessageTypes::CodecVersion => { + let codec_version = Self::handle_downcast::(message)?; + self.voice_manager.set_codec(&codec_version); + } crate::utils::messages::MessageTypes::UserStats => {} crate::utils::messages::MessageTypes::RequestBlob => {} crate::utils::messages::MessageTypes::ServerConfig => {} crate::utils::messages::MessageTypes::SuggestConfig => {} - crate::utils::messages::MessageTypes::PluginDataTransmission => {} + crate::utils::messages::MessageTypes::PluginDataTransmission => { + info!("Received plugin data transmission"); + } }; Ok(()) diff --git a/src-tauri/src/protocol/mod.rs b/src-tauri/src/protocol/mod.rs index e65482f..5ab1ce9 100644 --- a/src-tauri/src/protocol/mod.rs +++ b/src-tauri/src/protocol/mod.rs @@ -46,13 +46,13 @@ fn to_legacy_version(version: u64) -> u32 { } pub fn init_connection(username: &str, channel: &Sender>, package_info: &PackageInfo) { - let version = from_components( - package_info.version.major + 2, + let fancy_version = from_components( + package_info.version.major, package_info.version.minor, package_info.version.patch, ); - let mumble_version = from_components(1, 5, 0); + let mumble_version = from_components(1, 4, 0); let info = os_info::get(); @@ -67,8 +67,8 @@ pub fn init_connection(username: &str, channel: &Sender>, package_info: )), os_version: Some(info.version().to_string()), release: Some(package_info.package_name()), - version_v2: Some(version), - fancy_version: Some(version), + version_v2: Some(mumble_version), + fancy_version: Some(fancy_version), }; let buffer = message_builder(&version).unwrap_or_default(); @@ -76,7 +76,7 @@ pub fn init_connection(username: &str, channel: &Sender>, package_info: let auth = mumble::proto::Authenticate { opus: Some(true), - celt_versions: vec![-2_147_483_632, -2_147_483_637], + celt_versions: vec![0, -2_147_483_632, -2_147_483_637], password: None, tokens: vec![], username: Some(username.to_string()), diff --git a/src-tauri/src/protocol/serialize/mod.rs b/src-tauri/src/protocol/serialize/mod.rs index 83f496b..5216f5f 100644 --- a/src-tauri/src/protocol/serialize/mod.rs +++ b/src-tauri/src/protocol/serialize/mod.rs @@ -14,13 +14,15 @@ impl Serialize for TextMessage { where S: serde::Serializer, { - let mut s = serializer.serialize_struct("TextMessage", 5)?; + let mut s = serializer.serialize_struct("TextMessage", 7)?; s.serialize_field("actor", &self.actor)?; s.serialize_field("channel_id", &self.channel_id)?; s.serialize_field("message", &self.message)?; s.serialize_field("session", &self.session)?; s.serialize_field("tree_id", &self.tree_id)?; + s.serialize_field("timestamp", &self.timestamp)?; + s.serialize_field("message_id", &self.message_id)?; s.end() } } diff --git a/src-tauri/src/utils/audio/decoder.rs b/src-tauri/src/utils/audio/decoder.rs index 3382977..32803b8 100644 --- a/src-tauri/src/utils/audio/decoder.rs +++ b/src-tauri/src/utils/audio/decoder.rs @@ -11,14 +11,19 @@ pub struct DecodedMessage { pub data: Vec, } +pub trait Decoder: Send { + fn decode_audio(&mut self, audio_data: &[u8]) -> AnyError; +} + #[allow(clippy::struct_field_names)] -pub struct Decoder { +#[allow(clippy::module_name_repetitions)] +pub struct UDPDecoder { decoder_map: DecoderMap, sample_rate: u32, channels: opus::Channels, } -impl Decoder { +impl UDPDecoder { pub fn new(sample_rate: u32, channels: opus::Channels) -> Self { Self { decoder_map: DecoderMap::new(sample_rate, channels), @@ -26,12 +31,14 @@ impl Decoder { channels, } } +} +impl Decoder for UDPDecoder { // we want a downcast, because we are reading from a stream #[allow(clippy::cast_possible_truncation)] // We are aware of the possible truncation, but we are not using the full range of u32 #[allow(clippy::cast_sign_loss)] - pub fn decode_audio(&mut self, audio_data: &[u8]) -> AnyError { + fn decode_audio(&mut self, audio_data: &[u8]) -> AnyError { let audio_header = audio_data[0]; let audio_type = (audio_header & 0xE0) >> 5; @@ -114,3 +121,25 @@ impl DecoderMap { Ok(opus::Decoder::new(sample_rate, channels)?) } } + +// pub struct ProtobufDecoder { +// decoder_map: DecoderMap, +// sample_rate: u32, +// channels: opus::Channels, +// } + +// impl ProtobufDecoder { +// pub fn new(sample_rate: u32, channels: opus::Channels) -> Self { +// Self { +// decoder_map: DecoderMap::new(sample_rate, channels), +// sample_rate, +// channels, +// } +// } +// } + +// impl Decoder for ProtobufDecoder { +// fn decode_audio(&mut self, audio_data: &[u8]) -> AnyError { +// Err("Not implemented".into()) +// } +// } diff --git a/src-tauri/src/utils/audio/encoder.rs b/src-tauri/src/utils/audio/encoder.rs index 48bb52b..fa94b6a 100644 --- a/src-tauri/src/utils/audio/encoder.rs +++ b/src-tauri/src/utils/audio/encoder.rs @@ -7,14 +7,19 @@ use super::microphone::DeviceConfig; const MAXIMUM_SAMPLES_PER_TALK: u64 = 600; const QUALITY: opus::Application = opus::Application::Audio; +pub trait Encoder { + fn encode_audio(&mut self, data: &[f32], sequence_number: &mut u64) -> Option>; +} + #[allow(clippy::struct_field_names)] // yes -pub struct Encoder { +#[allow(clippy::module_name_repetitions)] // yes +pub struct UDPEncoder { encoder: opus::Encoder, audio_buffer_size: usize, talking: bool, } -impl Encoder { +impl UDPEncoder { pub fn new(config: DeviceConfig) -> Self { let opus_channels = match config.channels { 1 => Channels::Mono, @@ -32,7 +37,17 @@ impl Encoder { } } - pub fn encode_audio(&mut self, data: &[f32], sequence_number: &mut u64) -> Option> { + fn is_zero(buf: &[f32]) -> bool { + let (prefix, aligned, suffix) = unsafe { buf.align_to::() }; + + prefix.iter().all(|&x| x == 0.0) + && suffix.iter().all(|&x| x == 0.0) + && aligned.iter().all(|&x| x == 0) + } +} + +impl Encoder for UDPEncoder { + fn encode_audio(&mut self, data: &[f32], sequence_number: &mut u64) -> Option> { let is_only_zero = Self::is_zero(data); if !self.talking && is_only_zero { return None; @@ -79,12 +94,4 @@ impl Encoder { Some(audio_buffer) } - - fn is_zero(buf: &[f32]) -> bool { - let (prefix, aligned, suffix) = unsafe { buf.align_to::() }; - - prefix.iter().all(|&x| x == 0.0) - && suffix.iter().all(|&x| x == 0.0) - && aligned.iter().all(|&x| x == 0) - } } diff --git a/src-tauri/src/utils/audio/processing/compress.rs b/src-tauri/src/utils/audio/processing/compress.rs new file mode 100644 index 0000000..dae9563 --- /dev/null +++ b/src-tauri/src/utils/audio/processing/compress.rs @@ -0,0 +1,68 @@ +use std::time::Duration; +pub struct Compressor { + sample_rate: usize, + threshold: f32, + ratio: f32, + attack: Duration, + release: Duration, +} + +impl Compressor { + pub const fn new( + sample_rate: usize, + threshold: f32, + ratio: f32, + attack: Duration, + release: Duration, + ) -> Self { + Self { + sample_rate, + threshold, + ratio, + attack, + release, + } + } + + #[allow(clippy::cast_precision_loss)] // loss is expected due to resampling + pub fn process(&self, input: &mut [f32]) { + let attack_samples = self.attack.as_secs_f32() * self.sample_rate as f32; + let release_samples = self.release.as_secs_f32() * self.sample_rate as f32; + + for sample in input.iter_mut() { + let abs_sample = sample.abs(); + let sign = sample.signum(); + + if abs_sample > self.threshold { + let db_above_threshold = 20.0 * (abs_sample / self.threshold).log10(); + let gain_reduction_db = (db_above_threshold - self.threshold) / self.ratio; + let gain_reduction_linear = 10.0f32.powf(gain_reduction_db / 20.0); + + // Apply attack and release time + if gain_reduction_linear < abs_sample { + *sample = + sign * (abs_sample - (abs_sample - gain_reduction_linear) / attack_samples); + } else { + *sample = sign + * (abs_sample - (abs_sample - gain_reduction_linear) / release_samples); + } + }; + } + } + + pub fn set_threshold(&mut self, threshold: f32) { + self.threshold = threshold; + } + + pub fn set_ratio(&mut self, ratio: f32) { + self.ratio = ratio; + } + + pub fn set_attack(&mut self, attack: Duration) { + self.attack = attack; + } + + pub fn set_release(&mut self, release: Duration) { + self.release = release; + } +} diff --git a/src-tauri/src/utils/audio/processing/mod.rs b/src-tauri/src/utils/audio/processing/mod.rs index 5399ba3..14aafb6 100644 --- a/src-tauri/src/utils/audio/processing/mod.rs +++ b/src-tauri/src/utils/audio/processing/mod.rs @@ -1,2 +1,3 @@ +pub mod compress; pub mod hysteresis; pub mod voice_activation; diff --git a/src-tauri/src/utils/audio/recorder.rs b/src-tauri/src/utils/audio/recorder.rs index 6a77b23..87e5ef5 100644 --- a/src-tauri/src/utils/audio/recorder.rs +++ b/src-tauri/src/utils/audio/recorder.rs @@ -8,7 +8,6 @@ use std::{ time::Duration, }; -use serde::de::IntoDeserializer; use tokio::sync::broadcast::{self, Receiver}; use tracing::{error, info, trace, warn}; @@ -16,7 +15,10 @@ use crate::{ commands::utils::settings::{AudioOptions, AudioPreviewContainer, GlobalSettings, InputMode}, errors::AnyError, mumble::proto::UdpTunnel, - utils::{audio::microphone::Microphone, messages::raw_message_builder}, + utils::{ + audio::{encoder::UDPEncoder, microphone::Microphone, processing::compress::Compressor}, + messages::raw_message_builder, + }, }; use super::{ @@ -24,28 +26,6 @@ use super::{ processing::voice_activation::{VoiceActivation, VoiceActivationType}, }; -struct GlobalMaxAvg { - max_avg: f32, - max_avg_count: u64, -} - -impl Default for GlobalMaxAvg { - fn default() -> Self { - Self { - max_avg: 0.0, - max_avg_count: 0, - } - } -} - -impl GlobalMaxAvg { - fn update(&mut self, value: f32) { - self.max_avg_count += 1; - self.max_avg = - (self.max_avg * (self.max_avg_count - 1) as f32 + value) / self.max_avg_count as f32; - } -} - pub struct Recorder { audio_thread: Option>, playing: Arc, @@ -89,7 +69,7 @@ impl Recorder { let (tx, rx) = mpsc::channel(); let mut microphone = Microphone::new(tx).expect("Failed to create microphone"); - let mut encoder = Encoder::new(microphone.config()); + let mut encoder = UDPEncoder::new(microphone.config()); match microphone.start() { Ok(()) => {} Err(e) => { @@ -113,6 +93,13 @@ impl Recorder { 0.6, 0.3, )); + let mut compressor: Option = Some(Compressor::new( + sample_rate, + 0.0, + 0.0, + Duration::from_millis(0), + Duration::from_millis(0), + )); let mut audio_preview: Option = None; @@ -120,6 +107,7 @@ impl Recorder { update_settings( &mut settings_channel, &mut va, + &mut compressor, µphone, &mut audio_preview, ); @@ -129,6 +117,9 @@ impl Recorder { if let Some(va) = va.as_mut() { max_amplitude = va.process(&mut value); } + if let Some(compress) = compressor.as_mut() { + compress.process(&mut value); + } if let Some(audio_preview) = audio_preview.as_mut() { let _ = audio_preview.window.try_lock().map(|window| { @@ -167,6 +158,7 @@ impl Recorder { fn update_settings( settings_channel: &mut Receiver, va: &mut Option>, + compressor: &mut Option, microphone: &Microphone, audio_settings: &mut Option, ) { @@ -174,6 +166,7 @@ fn update_settings( Ok(GlobalSettings::AudioInputSettings(audio_settings)) => { info!("Received settings: {:?}", audio_settings); update_voice_activation_options(&audio_settings, va); + update_compressor_options(&audio_settings, compressor); let _ = microphone.volume_adjustment(audio_settings.amplification); } @@ -215,6 +208,19 @@ fn update_voice_activation_options( }; } +fn update_compressor_options(audio_settings: &AudioOptions, compressor: &mut Option) { + if let Some(compressor) = compressor.as_mut() { + if let Some(compressor_options) = &audio_settings.compressor_options { + compressor.set_attack(Duration::from_millis(compressor_options.attack_time as u64)); + compressor.set_release(Duration::from_millis( + compressor_options.release_time as u64, + )); + compressor.set_threshold(compressor_options.threshold); + compressor.set_ratio(compressor_options.ratio); + } + }; +} + impl Drop for Recorder { fn drop(&mut self) { self.stop(); diff --git a/src-tauri/src/utils/certificate_store.rs b/src-tauri/src/utils/certificate_store.rs index 3809375..5d2fdf4 100644 --- a/src-tauri/src/utils/certificate_store.rs +++ b/src-tauri/src/utils/certificate_store.rs @@ -24,14 +24,6 @@ pub struct CertificateBuilder { } impl CertificateBuilder { - pub const fn new() -> Self { - Self { - load_or_generate_new: false, - store_to_project_dir: false, - identity: None, - } - } - pub fn try_from(identity: &Option) -> Self { Self { load_or_generate_new: false, @@ -56,11 +48,21 @@ impl CertificateBuilder { let data_dir = project_dirs.data_dir(); if !data_dir.exists() { - std::fs::create_dir_all(&data_dir)?; + std::fs::create_dir_all(data_dir)?; } - let certificate_path = data_dir.join(self.identity.as_ref().map_or("certificate.pem".to_string(), |v| format!("cert_{}.pem", v))); - let private_key_path = data_dir.join(self.identity.as_ref().map_or("private_key.pem".to_string(), |v| format!("priv_key_{}.pem", v))); + let certificate_path = data_dir.join( + self.identity + .as_ref() + .map_or("certificate.pem".to_string(), |v| format!("cert_{v}.pem")), + ); + let private_key_path = data_dir.join( + self.identity + .as_ref() + .map_or("private_key.pem".to_string(), |v| { + format!("priv_key_{v}.pem") + }), + ); if self.load_or_generate_new { trace!("Trying to load certificate from project dir"); diff --git a/src-tauri/src/utils/frontend/mod.rs b/src-tauri/src/utils/frontend/mod.rs index a19b19d..1cace19 100644 --- a/src-tauri/src/utils/frontend/mod.rs +++ b/src-tauri/src/utils/frontend/mod.rs @@ -4,7 +4,11 @@ use tracing::error; use crate::protocol::serialize::message_container::FrontendMessage; -pub fn send_to_frontend(frontend_channel: &Sender, msg: &FrontendMessage) { +#[allow(clippy::module_name_repetitions)] // We want to be explicit here +pub fn send_to_frontend( + frontend_channel: &Sender, + msg: &FrontendMessage, +) { match serde_json::to_string(&msg) { Ok(json) => { if let Err(e) = frontend_channel.send(json) { @@ -15,4 +19,4 @@ pub fn send_to_frontend(frontend_channel: &Sender, error!("Failed to serialize user list: {}", e); } } -} \ No newline at end of file +} diff --git a/src-tauri/src/utils/mod.rs b/src-tauri/src/utils/mod.rs index 1a1b4cc..1f3d273 100644 --- a/src-tauri/src/utils/mod.rs +++ b/src-tauri/src/utils/mod.rs @@ -2,10 +2,10 @@ pub mod audio; pub mod certificate_store; pub mod constants; pub mod file; +pub mod frontend; pub mod messages; pub mod server; pub mod varint; -pub mod frontend; #[cfg(test)] mod tests; diff --git a/src-tauri/src/utils/server.rs b/src-tauri/src/utils/server.rs index 105a091..f166199 100644 --- a/src-tauri/src/utils/server.rs +++ b/src-tauri/src/utils/server.rs @@ -10,5 +10,5 @@ pub struct Server { pub host: String, pub port: u16, pub username: String, - pub identity: Option + pub identity: Option, } diff --git a/src/components/ChatMessage.tsx b/src/components/ChatMessage.tsx index 75f4a23..c0b00e3 100644 --- a/src/components/ChatMessage.tsx +++ b/src/components/ChatMessage.tsx @@ -65,8 +65,7 @@ const generateDate = (timestamp: number) => { const ChatMessage: React.FC = React.memo(({ message, messageId, onLoaded }) => { const userList = useSelector((state: RootState) => state.reducer.userInfo); const dispatch = useDispatch(); - const { t, i18n } = useTranslation(); - const [loaded, setLoaded] = React.useState(false); + const { t } = useTranslation(); const user = React.useMemo(() => userList.users.find(e => e.id === message.sender.user_id) @@ -80,7 +79,7 @@ const ChatMessage: React.FC = React.memo(({ message, messageId }, [dispatch, messageId]); const likeMessage = React.useCallback((messageId: string) => { - invoke('like_message', { messageId: messageId }); + invoke('like_message', { messageId: messageId, reciever: userList.users.map(e => e.id) }); }, []); return ( @@ -95,7 +94,7 @@ const ChatMessage: React.FC = React.memo(({ message, messageId {message.sender.user_name} - {date} - likeMessage("abc")}> + likeMessage(message.id)}> diff --git a/src/components/settings/Audio.tsx b/src/components/settings/Audio.tsx index 7d385b9..e74e517 100644 --- a/src/components/settings/Audio.tsx +++ b/src/components/settings/Audio.tsx @@ -5,7 +5,7 @@ import { useEffect, useState } from "react"; import KeyboardIcon from '@mui/icons-material/Keyboard'; import FloatingApply from "./components/FloatingApply"; import { listen } from "@tauri-apps/api/event"; -import { InputMode, setAmplification, setFadeOutDuration, setInputMode, setVoiceHold, setVoiceHysteresis } from "../../store/features/users/audioSettings"; +import { InputMode, setAmplification, setAttackTime, setCompressorRatio, setCompressorThreshold, setFadeOutDuration, setInputMode, setReleaseTime, setVoiceHold, setVoiceHysteresis } from "../../store/features/users/audioSettings"; import { RootState } from "../../store/store"; import { useDispatch, useSelector } from "react-redux"; import { useTranslation } from "react-i18next"; @@ -47,6 +47,12 @@ function AudioSettings() { fade_out_duration: Math.floor(audioSettings.voice_activation_options.fade_out_duration), voice_hysteresis_lower_threshold: audioSettings.voice_activation_options.voice_hysteresis_lower_threshold, voice_hysteresis_upper_threshold: audioSettings.voice_activation_options.voice_hysteresis_upper_threshold, + }, + compressor_options: { + threshold: audioSettings.compressor_options.threshold, + ratio: audioSettings.compressor_options.ratio, + attack_time: Math.floor(audioSettings.compressor_options.attack_time), + release_time: Math.floor(audioSettings.compressor_options.release_time), } }; console.log(settings); @@ -248,6 +254,71 @@ function AudioSettings() { + + + + {t("Compressor Threshold", { ns: "audio", threshold: audioSettings.compressor_options.threshold })} + + dispatch(setCompressorThreshold(v as number))} + valueLabelDisplay="auto" + aria-labelledby="non-linear-slider" + /> + + + + {t("Compressor Ratio", { ns: "audio", ratio: audioSettings.compressor_options.ratio })} + + dispatch(setCompressorRatio(v as number))} + valueLabelDisplay="auto" + aria-labelledby="non-linear-slider" + /> + + + + {t("Attack Time", { ns: "audio", duration: valueLabelFormat(audioSettings.compressor_options.attack_time) })} + + dispatch(setAttackTime(calculateVoiceHold(value as number)))} + valueLabelDisplay="auto" + aria-labelledby="non-linear-slider" + /> + + + + {t("Release Time", { ns: "audio", duration: valueLabelFormat(audioSettings.compressor_options.release_time) })} + + dispatch(setReleaseTime(calculateVoiceHold(value as number)))} + valueLabelDisplay="auto" + aria-labelledby="non-linear-slider" + /> + + + {t("Echo Cancelation", { ns: "audio" })} diff --git a/src/store/features/users/audioSettings.ts b/src/store/features/users/audioSettings.ts index 0418fa1..7e59de7 100644 --- a/src/store/features/users/audioSettings.ts +++ b/src/store/features/users/audioSettings.ts @@ -12,10 +12,18 @@ interface VoiceActivationOptions { voice_hysteresis_upper_threshold: number; } +interface CompressorOptions { + attack_time: number, + release_time: number, + threshold: number, + ratio: number, +} + interface AudioInputSettings { amplification: number; input_mode: InputMode; voice_activation_options: VoiceActivationOptions; + compressor_options: CompressorOptions; } const initialState: AudioInputSettings = { @@ -26,7 +34,12 @@ const initialState: AudioInputSettings = { fade_out_duration: 850, voice_hysteresis_lower_threshold: 0.03, voice_hysteresis_upper_threshold: 0.07 - + }, + compressor_options: { + attack_time: 0.1, + release_time: 0.1, + threshold: -30.0, + ratio: 10.0 } }; @@ -52,10 +65,33 @@ export const frontendSettings = createSlice({ setVoiceHysteresis(state, action) { state.voice_activation_options.voice_hysteresis_lower_threshold = action.payload[0]; state.voice_activation_options.voice_hysteresis_upper_threshold = action.payload[1]; - } + }, + setAttackTime(state, action) { + state.compressor_options.attack_time = action.payload + }, + setReleaseTime(state, action) { + state.compressor_options.release_time = action.payload + }, + setCompressorThreshold(state, action) { + state.compressor_options.threshold = action.payload + }, + setCompressorRatio(state, action) { + state.compressor_options.ratio = action.payload + }, }, }) -export const { updateAudioSettings, setAmplification, setInputMode, setVoiceHold, setFadeOutDuration, setVoiceHysteresis } = frontendSettings.actions +export const { + updateAudioSettings, + setAmplification, + setInputMode, + setVoiceHold, + setFadeOutDuration, + setVoiceHysteresis, + setAttackTime, + setReleaseTime, + setCompressorThreshold, + setCompressorRatio +} = frontendSettings.actions export default frontendSettings.reducer \ No newline at end of file diff --git a/src/store/features/users/chatMessageSlice.ts b/src/store/features/users/chatMessageSlice.ts index 242e5f7..7d8d5b5 100644 --- a/src/store/features/users/chatMessageSlice.ts +++ b/src/store/features/users/chatMessageSlice.ts @@ -17,7 +17,9 @@ export interface TextMessage { // The UTF-8 encoded message. May be HTML if the server allows. message: string, // custom property to keep track of time - timestamp: number + timestamp: number, + // unique id of the message + id: string }