Merge branch 'main' into lukas/chat-api

.gitignore

@@ -1,3 +1,5 @@
 target
 /.idea
+soxr-sys/test-input.wav
+soxr-sys/test-output.wav
 .DS_Store

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "livekit-protocol",
     "livekit-ffi",
     "libwebrtc",
+    "soxr-sys",
     "webrtc-sys",
     "webrtc-sys/build",
 ]

livekit-api/src/services/sip.rs

@@ -200,8 +200,9 @@ impl SIPClient {
                         allowed_addresses: options.allowed_addresses.to_owned(),
                         auth_username: options.auth_username.to_owned(),
                         auth_password: options.auth_password.to_owned(),
-                        headers: HashMap::new(),               // FIXME Lukas
-                        headers_to_attributes: HashMap::new(), // FIXME Lukas
+
+                        headers: Default::default(),
+                        headers_to_attributes: Default::default(),
                     }),
                 },
                 self.base.auth_header(
@@ -236,8 +237,8 @@ impl SIPClient {
                         auth_username: options.auth_username.to_owned(),
                         auth_password: options.auth_password.to_owned(),
 
-                        headers: HashMap::new(),               // FIXME Lukas
-                        headers_to_attributes: HashMap::new(), // FIXME Lukas
+                        headers: Default::default(),
+                        headers_to_attributes: Default::default(),
                     }),
                 },
                 self.base.auth_header(

livekit-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "livekit-ffi"
-version = "0.10.3"
+version = "0.11.3"
 edition = "2021"
 license = "Apache-2.0"
 description = "FFI interface for bindings in other languages"
@@ -19,6 +19,7 @@ tracing = ["tokio/tracing", "console-subscriber"]
 
 [dependencies]
 livekit = { path = "../livekit", version = "0.6.0" }
+soxr-sys = { path = "../soxr-sys" }
 livekit-protocol = { path = "../livekit-protocol", version = "0.3.5" }
 tokio = { version = "1", features = ["full", "parking_lot"] }
 futures-util = { version = "0.3", default-features = false, features = ["sink"] }

livekit-ffi/protocol/audio_frame.proto

@@ -87,6 +87,70 @@ message RemixAndResampleResponse {
   OwnedAudioFrameBuffer buffer = 1;
 }
 
+
+// New resampler using SoX (much better quality)
+
+message NewSoxResamplerRequest {
+  double input_rate = 1;
+  double output_rate  = 2;
+  uint32 num_channels = 3;
+  SoxResamplerDataType input_data_type = 4;
+  SoxResamplerDataType output_data_type = 5;
+  SoxQualityRecipe quality_recipe = 6;
+  uint32 flags = 7;
+}
+message NewSoxResamplerResponse {
+  OwnedSoxResampler resampler = 1;
+  optional string error = 2;
+}
+
+message PushSoxResamplerRequest {
+  uint64 resampler_handle = 1;
+  uint64 data_ptr = 2; // *const i16
+  uint32 size = 3; // in bytes
+}
+
+message PushSoxResamplerResponse {
+  uint64 output_ptr = 1; // *const i16 (could be null)
+  uint32 size = 2; // in bytes
+  optional string error = 3;
+}
+
+message FlushSoxResamplerRequest {
+  uint64 resampler_handle = 1;
+}
+
+message FlushSoxResamplerResponse {
+  uint64 output_ptr = 1; // *const i16 (could be null)
+  uint32 size = 2; // in bytes
+  optional string error = 3;
+}
+
+enum SoxResamplerDataType {
+  // TODO(theomonnom): support other datatypes (shouldn't really be needed)
+  SOXR_DATATYPE_INT16I = 0;
+  SOXR_DATATYPE_INT16S = 1;
+}
+
+enum SoxQualityRecipe {
+  SOXR_QUALITY_QUICK = 0;
+  SOXR_QUALITY_LOW = 1;
+  SOXR_QUALITY_MEDIUM = 2;
+  SOXR_QUALITY_HIGH = 3;
+  SOXR_QUALITY_VERYHIGH = 4;
+}
+
+enum SoxFlagBits {
+  SOXR_ROLLOFF_SMALL = 0;  // 1 << 0
+  SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1
+  SOXR_ROLLOFF_NONE = 2;   // 1 << 2
+  SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3
+  SOXR_DOUBLE_PRECISION = 4; // 1 << 4
+  SOXR_VR = 5; // 1 << 5
+}
+
+
+
 //
 // AudioFrame buffer
 //
@@ -168,3 +232,17 @@ message OwnedAudioResampler {
   FfiOwnedHandle handle = 1;
   AudioResamplerInfo info = 2;
 }
+
+
+
+//
+// Sox AudioResampler
+//
+
+
+message SoxResamplerInfo {}
+
+message OwnedSoxResampler {
+  FfiOwnedHandle handle = 1;
+  SoxResamplerInfo info = 2;
+}

livekit-ffi/protocol/ffi.proto

@@ -69,8 +69,6 @@ message FfiRequest {
     GetSessionStatsRequest get_session_stats = 12;
     PublishTranscriptionRequest publish_transcription = 13;
     PublishSipDtmfRequest publish_sip_dtmf = 14;
-    SendChatMessageRequest send_chat_message = 33;
-    EditChatMessageRequest edit_chat_message = 34;
 
     // Track
     CreateVideoTrackRequest create_video_track = 15;
@@ -79,7 +77,7 @@ message FfiRequest {
     EnableRemoteTrackRequest enable_remote_track = 18;
     GetStatsRequest get_stats = 19;
 
-    // Video
+// Video
     NewVideoStreamRequest new_video_stream = 20;
     NewVideoSourceRequest new_video_source = 21;
     CaptureVideoFrameRequest capture_video_frame = 22;
@@ -95,8 +93,12 @@ message FfiRequest {
     RemixAndResampleRequest remix_and_resample = 30;
     E2eeRequest e2ee = 31;
     AudioStreamFromParticipantRequest audio_stream_from_participant = 32;
+    NewSoxResamplerRequest new_sox_resampler = 33;
+    PushSoxResamplerRequest push_sox_resampler = 34;
+    FlushSoxResamplerRequest flush_sox_resampler = 35;
 
-    // next-id 34
+    SendChatMessageRequest send_chat_message = 36;
+    EditChatMessageRequest edit_chat_message = 37;
   }
 }
 
@@ -118,7 +120,6 @@ message FfiResponse {
     GetSessionStatsResponse get_session_stats = 12;
     PublishTranscriptionResponse publish_transcription = 13;
     PublishSipDtmfResponse publish_sip_dtmf = 14;
-    SendChatMessageResponse send_chat_message = 33;
 
     // Track
     CreateVideoTrackResponse create_video_track = 15;
@@ -143,8 +144,11 @@ message FfiResponse {
     RemixAndResampleResponse remix_and_resample = 30;
     AudioStreamFromParticipantResponse audio_stream_from_participant = 31;
     E2eeResponse e2ee = 32;
+    NewSoxResamplerResponse new_sox_resampler = 33;
+    PushSoxResamplerResponse push_sox_resampler = 34;
+    FlushSoxResamplerResponse flush_sox_resampler = 35;
 
-    // next-id 34
+    SendChatMessageResponse send_chat_message = 36;
   }
 }
 

livekit-ffi/src/cabi.rs

@@ -85,6 +85,11 @@ pub extern "C" fn livekit_ffi_drop_handle(handle_id: FfiHandleId) -> bool {
     FFI_SERVER.drop_handle(handle_id)
 }
 
+#[no_mangle]
+pub extern "C" fn livekit_ffi_dispose() {
+    FFI_SERVER.async_runtime.block_on(FFI_SERVER.dispose());
+}
+
 #[cfg(target_os = "android")]
 pub mod android {
     use jni::{

livekit-ffi/src/conversion/mod.rs

@@ -14,6 +14,7 @@
 
 pub mod audio_frame;
 pub mod participant;
+pub mod resampler;
 pub mod room;
 pub mod stats;
 pub mod track;

livekit-ffi/src/conversion/resampler.rs

@@ -0,0 +1 @@
+