From be3dde54f91ea30319d0e8401a34a600e38cfbc6 Mon Sep 17 00:00:00 2001 From: b-ma Date: Fri, 29 Nov 2024 16:48:27 +0100 Subject: [PATCH 1/7] wip: test fft_convolver crate --- Cargo.toml | 1 + src/node/convolver.rs | 374 +++++++++++++++++++++--------------------- 2 files changed, 190 insertions(+), 185 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e66e5b40..d5ef90ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ rust-version = "1.76" almost = "0.2.0" arc-swap = "1.6" arrayvec = "0.7" +fft-convolver = "0.2.0" cpal = { version = "0.15", optional = true } creek = "1.2" crossbeam-channel = "0.5" diff --git a/src/node/convolver.rs b/src/node/convolver.rs index 53489316..219fa88e 100644 --- a/src/node/convolver.rs +++ b/src/node/convolver.rs @@ -2,6 +2,7 @@ use std::any::Any; use std::sync::Arc; use realfft::{num_complex::Complex, ComplexToReal, RealFftPlanner, RealToComplex}; +use fft_convolver::FFTConvolver; use crate::buffer::AudioBuffer; use crate::context::{AudioContextRegistration, BaseAudioContext}; @@ -215,23 +216,26 @@ impl ConvolverNode { }; // Pad the response buffer with zeroes so its size is a power of 2, with 2 * 128 as min size - let length = buffer.length(); - let padded_length = length.next_power_of_two().max(2 * RENDER_QUANTUM_SIZE); - let samples: Vec<_> = (0..number_of_channels) - .map(|_| { - let mut samples = vec![0.; padded_length]; - samples[..length] - .iter_mut() - .zip(buffer.get_channel_data(0)) - .for_each(|(o, i)| *o = *i * scale); - samples - }) - .collect(); - - let padded_buffer = AudioBuffer::from(samples, sample_rate); - let convolve = ConvolverRendererInner::new(padded_buffer); - - self.registration.post_message(Some(convolve)); + // let length = buffer.length(); + // let padded_length = length.next_power_of_two().max(2 * RENDER_QUANTUM_SIZE); + // let samples: Vec<_> = (0..number_of_channels) + // .map(|_| { + // let mut samples = vec![0.; padded_length]; + // samples[..length] + // .iter_mut() + // .zip(buffer.get_channel_data(0)) + // .for_each(|(o, i)| *o = *i * scale); + // samples + // }) + // .collect(); + + let mut convolver = FFTConvolver::::default(); + let _ = convolver.init(RENDER_QUANTUM_SIZE, buffer.get_channel_data(0)); + + // let padded_buffer = AudioBuffer::from(samples, sample_rate); + // let convolve = ConvolverRendererInner::new(padded_buffer); + + self.registration.post_message(Some(convolver)); self.buffer = Some(buffer); } @@ -246,171 +250,171 @@ impl ConvolverNode { } } -fn roll_zero(signal: &mut [T], n: usize) { - // roll array by n elements - // zero out the last n elements - let len = signal.len(); - signal.copy_within(n.., 0); - signal[len - n..].fill(T::default()); -} - -struct Fft { - fft_forward: Arc>, - fft_inverse: Arc>, - fft_input: Vec, - fft_scratch: Vec>, - fft_output: Vec>, -} - -impl Fft { - fn new(length: usize) -> Self { - let mut fft_planner = RealFftPlanner::::new(); - - let fft_forward = fft_planner.plan_fft_forward(length); - let fft_inverse = fft_planner.plan_fft_inverse(length); - - let fft_input = fft_forward.make_input_vec(); - let fft_scratch = fft_forward.make_scratch_vec(); - let fft_output = fft_forward.make_output_vec(); - - Self { - fft_forward, - fft_inverse, - fft_input, - fft_scratch, - fft_output, - } - } - - fn real(&mut self) -> &mut [f32] { - &mut self.fft_input[..] - } - - fn complex(&mut self) -> &mut [Complex] { - &mut self.fft_output[..] - } - - fn process(&mut self) -> &[Complex] { - self.fft_forward - .process_with_scratch( - &mut self.fft_input, - &mut self.fft_output, - &mut self.fft_scratch, - ) - .unwrap(); - &self.fft_output[..] - } - - fn inverse(&mut self) -> &[f32] { - self.fft_inverse - .process_with_scratch( - &mut self.fft_output, - &mut self.fft_input, - &mut self.fft_scratch, - ) - .unwrap(); - &self.fft_input[..] - } -} - -struct ConvolverRendererInner { - num_ir_blocks: usize, - h: Vec>, - fdl: Vec>, - out: Vec, - fft2: Fft, -} - -impl ConvolverRendererInner { - fn new(response: AudioBuffer) -> Self { - // mono processing only for now - let response = response.channel_data(0).as_slice(); - - let mut fft2 = Fft::new(2 * RENDER_QUANTUM_SIZE); - let p = response.len(); - - let num_ir_blocks = p / RENDER_QUANTUM_SIZE; - - let mut h = vec![Complex::default(); num_ir_blocks * 2 * RENDER_QUANTUM_SIZE]; - for (resp_fft, resp) in h - .chunks_mut(2 * RENDER_QUANTUM_SIZE) - .zip(response.chunks(RENDER_QUANTUM_SIZE)) - { - // fill resp_fft with FFT of resp.zero_pad(RENDER_QUANTUM_SIZE) - fft2.real()[..RENDER_QUANTUM_SIZE].copy_from_slice(resp); - fft2.real()[RENDER_QUANTUM_SIZE..].fill(0.); - resp_fft[..fft2.complex().len()].copy_from_slice(fft2.process()); - } - - let fdl = vec![Complex::default(); 2 * RENDER_QUANTUM_SIZE * num_ir_blocks]; - let out = vec![0.; 2 * RENDER_QUANTUM_SIZE - 1]; - - Self { - num_ir_blocks, - h, - fdl, - out, - fft2, - } - } - - fn process(&mut self, input: &[f32], output: &mut [f32]) { - self.fft2.real()[..RENDER_QUANTUM_SIZE].copy_from_slice(input); - self.fft2.real()[RENDER_QUANTUM_SIZE..].fill(0.); - let spectrum = self.fft2.process(); - - self.fdl - .chunks_mut(2 * RENDER_QUANTUM_SIZE) - .zip(self.h.chunks(2 * RENDER_QUANTUM_SIZE)) - .for_each(|(fdl_c, h_c)| { - fdl_c - .iter_mut() - .zip(h_c) - .zip(spectrum) - .for_each(|((f, h), s)| *f += h * s) - }); - - let c_len = self.fft2.complex().len(); - self.fft2.complex().copy_from_slice(&self.fdl[..c_len]); - let inverse = self.fft2.inverse(); - self.out.iter_mut().zip(inverse).for_each(|(o, i)| { - *o += i / (2 * RENDER_QUANTUM_SIZE) as f32; - }); - - output.copy_from_slice(&self.out[..RENDER_QUANTUM_SIZE]); - - roll_zero(&mut self.fdl[..], 2 * RENDER_QUANTUM_SIZE); - roll_zero(&mut self.out[..], RENDER_QUANTUM_SIZE); - } - - fn tail(&mut self, output: &mut AudioRenderQuantum) -> bool { - if self.num_ir_blocks == 0 { - output.make_silent(); - return false; - } - - self.num_ir_blocks -= 1; - - let c_len = self.fft2.complex().len(); - self.fft2.complex().copy_from_slice(&self.fdl[..c_len]); - let inverse = self.fft2.inverse(); - self.out.iter_mut().zip(inverse).for_each(|(o, i)| { - *o += i / (2 * RENDER_QUANTUM_SIZE) as f32; - }); - - output - .channel_data_mut(0) - .copy_from_slice(&self.out[..RENDER_QUANTUM_SIZE]); - - roll_zero(&mut self.fdl[..], 2 * RENDER_QUANTUM_SIZE); - roll_zero(&mut self.out[..], RENDER_QUANTUM_SIZE); - - self.num_ir_blocks > 0 - } -} +// fn roll_zero(signal: &mut [T], n: usize) { +// // roll array by n elements +// // zero out the last n elements +// let len = signal.len(); +// signal.copy_within(n.., 0); +// signal[len - n..].fill(T::default()); +// } + +// struct Fft { +// fft_forward: Arc>, +// fft_inverse: Arc>, +// fft_input: Vec, +// fft_scratch: Vec>, +// fft_output: Vec>, +// } + +// impl Fft { +// fn new(length: usize) -> Self { +// let mut fft_planner = RealFftPlanner::::new(); + +// let fft_forward = fft_planner.plan_fft_forward(length); +// let fft_inverse = fft_planner.plan_fft_inverse(length); + +// let fft_input = fft_forward.make_input_vec(); +// let fft_scratch = fft_forward.make_scratch_vec(); +// let fft_output = fft_forward.make_output_vec(); + +// Self { +// fft_forward, +// fft_inverse, +// fft_input, +// fft_scratch, +// fft_output, +// } +// } + +// fn real(&mut self) -> &mut [f32] { +// &mut self.fft_input[..] +// } + +// fn complex(&mut self) -> &mut [Complex] { +// &mut self.fft_output[..] +// } + +// fn process(&mut self) -> &[Complex] { +// self.fft_forward +// .process_with_scratch( +// &mut self.fft_input, +// &mut self.fft_output, +// &mut self.fft_scratch, +// ) +// .unwrap(); +// &self.fft_output[..] +// } + +// fn inverse(&mut self) -> &[f32] { +// self.fft_inverse +// .process_with_scratch( +// &mut self.fft_output, +// &mut self.fft_input, +// &mut self.fft_scratch, +// ) +// .unwrap(); +// &self.fft_input[..] +// } +// } + +// struct ConvolverRendererInner { +// num_ir_blocks: usize, +// h: Vec>, +// fdl: Vec>, +// out: Vec, +// fft2: Fft, +// } + +// impl ConvolverRendererInner { +// fn new(response: AudioBuffer) -> Self { +// // mono processing only for now +// let response = response.channel_data(0).as_slice(); + +// let mut fft2 = Fft::new(2 * RENDER_QUANTUM_SIZE); +// let p = response.len(); + +// let num_ir_blocks = p / RENDER_QUANTUM_SIZE; + +// let mut h = vec![Complex::default(); num_ir_blocks * 2 * RENDER_QUANTUM_SIZE]; +// for (resp_fft, resp) in h +// .chunks_mut(2 * RENDER_QUANTUM_SIZE) +// .zip(response.chunks(RENDER_QUANTUM_SIZE)) +// { +// // fill resp_fft with FFT of resp.zero_pad(RENDER_QUANTUM_SIZE) +// fft2.real()[..RENDER_QUANTUM_SIZE].copy_from_slice(resp); +// fft2.real()[RENDER_QUANTUM_SIZE..].fill(0.); +// resp_fft[..fft2.complex().len()].copy_from_slice(fft2.process()); +// } + +// let fdl = vec![Complex::default(); 2 * RENDER_QUANTUM_SIZE * num_ir_blocks]; +// let out = vec![0.; 2 * RENDER_QUANTUM_SIZE - 1]; + +// Self { +// num_ir_blocks, +// h, +// fdl, +// out, +// fft2, +// } +// } + +// fn process(&mut self, input: &[f32], output: &mut [f32]) { +// self.fft2.real()[..RENDER_QUANTUM_SIZE].copy_from_slice(input); +// self.fft2.real()[RENDER_QUANTUM_SIZE..].fill(0.); +// let spectrum = self.fft2.process(); + +// self.fdl +// .chunks_mut(2 * RENDER_QUANTUM_SIZE) +// .zip(self.h.chunks(2 * RENDER_QUANTUM_SIZE)) +// .for_each(|(fdl_c, h_c)| { +// fdl_c +// .iter_mut() +// .zip(h_c) +// .zip(spectrum) +// .for_each(|((f, h), s)| *f += h * s) +// }); + +// let c_len = self.fft2.complex().len(); +// self.fft2.complex().copy_from_slice(&self.fdl[..c_len]); +// let inverse = self.fft2.inverse(); +// self.out.iter_mut().zip(inverse).for_each(|(o, i)| { +// *o += i / (2 * RENDER_QUANTUM_SIZE) as f32; +// }); + +// output.copy_from_slice(&self.out[..RENDER_QUANTUM_SIZE]); + +// roll_zero(&mut self.fdl[..], 2 * RENDER_QUANTUM_SIZE); +// roll_zero(&mut self.out[..], RENDER_QUANTUM_SIZE); +// } + +// fn tail(&mut self, output: &mut AudioRenderQuantum) -> bool { +// if self.num_ir_blocks == 0 { +// output.make_silent(); +// return false; +// } + +// self.num_ir_blocks -= 1; + +// let c_len = self.fft2.complex().len(); +// self.fft2.complex().copy_from_slice(&self.fdl[..c_len]); +// let inverse = self.fft2.inverse(); +// self.out.iter_mut().zip(inverse).for_each(|(o, i)| { +// *o += i / (2 * RENDER_QUANTUM_SIZE) as f32; +// }); + +// output +// .channel_data_mut(0) +// .copy_from_slice(&self.out[..RENDER_QUANTUM_SIZE]); + +// roll_zero(&mut self.fdl[..], 2 * RENDER_QUANTUM_SIZE); +// roll_zero(&mut self.out[..], RENDER_QUANTUM_SIZE); + +// self.num_ir_blocks > 0 +// } +// } struct ConvolverRenderer { - inner: Option, + inner: Option>, } impl AudioProcessor for ConvolverRenderer { @@ -436,22 +440,22 @@ impl AudioProcessor for ConvolverRenderer { }; // handle tail time - if input.is_silent() { - return convolver.tail(output); - } + // if input.is_silent() { + // return convolver.tail(output); + // } let mut mono = input.clone(); mono.mix(1, ChannelInterpretation::Speakers); let input = &mono.channel_data(0)[..]; let output = &mut output.channel_data_mut(0)[..]; - convolver.process(input, output); + let _ = convolver.process(input, output); true } fn onmessage(&mut self, msg: &mut dyn Any) { - if let Some(convolver) = msg.downcast_mut::>() { + if let Some(convolver) = msg.downcast_mut::>>() { // Avoid deallocation in the render thread by swapping the convolver. std::mem::swap(&mut self.inner, convolver); return; From 88c499f53d464362b23df22ee6a893548e7d07ff Mon Sep 17 00:00:00 2001 From: b-ma Date: Fri, 29 Nov 2024 17:08:55 +0100 Subject: [PATCH 2/7] cleaning --- src/node/convolver.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/node/convolver.rs b/src/node/convolver.rs index 219fa88e..1d44d81f 100644 --- a/src/node/convolver.rs +++ b/src/node/convolver.rs @@ -1,7 +1,7 @@ use std::any::Any; -use std::sync::Arc; +// use std::sync::Arc; -use realfft::{num_complex::Complex, ComplexToReal, RealFftPlanner, RealToComplex}; +// use realfft::{num_complex::Complex, ComplexToReal, RealFftPlanner, RealToComplex}; use fft_convolver::FFTConvolver; use crate::buffer::AudioBuffer; @@ -229,8 +229,16 @@ impl ConvolverNode { // }) // .collect(); + let mut samples = vec![0.; buffer.length()]; + samples.iter_mut() + .zip(buffer.get_channel_data(0)) + .for_each(|(o, i)| *o = *i * scale); + let mut convolver = FFTConvolver::::default(); - let _ = convolver.init(RENDER_QUANTUM_SIZE, buffer.get_channel_data(0)); + // Size of the partition changes a lot the perf... + // - RENDER_QUANTUM_SIZE -> 20x (compared to real-time) + // - RENDER_QUANTUM_SIZE * 8 -> 134x + let _ = convolver.init(RENDER_QUANTUM_SIZE * 8, &samples); // let padded_buffer = AudioBuffer::from(samples, sample_rate); // let convolve = ConvolverRendererInner::new(padded_buffer); @@ -474,12 +482,12 @@ mod tests { use super::*; - #[test] - fn test_roll_zero() { - let mut input = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - roll_zero(&mut input, 3); - assert_eq!(&input, &[4, 5, 6, 7, 8, 9, 10, 0, 0, 0]); - } + // #[test] + // fn test_roll_zero() { + // let mut input = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + // roll_zero(&mut input, 3); + // assert_eq!(&input, &[4, 5, 6, 7, 8, 9, 10, 0, 0, 0]); + // } #[test] #[should_panic] From 037c2eba1a178b3aad321b8644b50b8e84753421 Mon Sep 17 00:00:00 2001 From: Otto Date: Sun, 1 Dec 2024 14:01:24 +0100 Subject: [PATCH 3/7] ConvolverNode: cleanup commented out code --- src/node/convolver.rs | 187 +----------------------------------------- 1 file changed, 2 insertions(+), 185 deletions(-) diff --git a/src/node/convolver.rs b/src/node/convolver.rs index 1d44d81f..2814a628 100644 --- a/src/node/convolver.rs +++ b/src/node/convolver.rs @@ -215,22 +215,9 @@ impl ConvolverNode { 1. }; - // Pad the response buffer with zeroes so its size is a power of 2, with 2 * 128 as min size - // let length = buffer.length(); - // let padded_length = length.next_power_of_two().max(2 * RENDER_QUANTUM_SIZE); - // let samples: Vec<_> = (0..number_of_channels) - // .map(|_| { - // let mut samples = vec![0.; padded_length]; - // samples[..length] - // .iter_mut() - // .zip(buffer.get_channel_data(0)) - // .for_each(|(o, i)| *o = *i * scale); - // samples - // }) - // .collect(); - let mut samples = vec![0.; buffer.length()]; - samples.iter_mut() + samples + .iter_mut() .zip(buffer.get_channel_data(0)) .for_each(|(o, i)| *o = *i * scale); @@ -258,169 +245,6 @@ impl ConvolverNode { } } -// fn roll_zero(signal: &mut [T], n: usize) { -// // roll array by n elements -// // zero out the last n elements -// let len = signal.len(); -// signal.copy_within(n.., 0); -// signal[len - n..].fill(T::default()); -// } - -// struct Fft { -// fft_forward: Arc>, -// fft_inverse: Arc>, -// fft_input: Vec, -// fft_scratch: Vec>, -// fft_output: Vec>, -// } - -// impl Fft { -// fn new(length: usize) -> Self { -// let mut fft_planner = RealFftPlanner::::new(); - -// let fft_forward = fft_planner.plan_fft_forward(length); -// let fft_inverse = fft_planner.plan_fft_inverse(length); - -// let fft_input = fft_forward.make_input_vec(); -// let fft_scratch = fft_forward.make_scratch_vec(); -// let fft_output = fft_forward.make_output_vec(); - -// Self { -// fft_forward, -// fft_inverse, -// fft_input, -// fft_scratch, -// fft_output, -// } -// } - -// fn real(&mut self) -> &mut [f32] { -// &mut self.fft_input[..] -// } - -// fn complex(&mut self) -> &mut [Complex] { -// &mut self.fft_output[..] -// } - -// fn process(&mut self) -> &[Complex] { -// self.fft_forward -// .process_with_scratch( -// &mut self.fft_input, -// &mut self.fft_output, -// &mut self.fft_scratch, -// ) -// .unwrap(); -// &self.fft_output[..] -// } - -// fn inverse(&mut self) -> &[f32] { -// self.fft_inverse -// .process_with_scratch( -// &mut self.fft_output, -// &mut self.fft_input, -// &mut self.fft_scratch, -// ) -// .unwrap(); -// &self.fft_input[..] -// } -// } - -// struct ConvolverRendererInner { -// num_ir_blocks: usize, -// h: Vec>, -// fdl: Vec>, -// out: Vec, -// fft2: Fft, -// } - -// impl ConvolverRendererInner { -// fn new(response: AudioBuffer) -> Self { -// // mono processing only for now -// let response = response.channel_data(0).as_slice(); - -// let mut fft2 = Fft::new(2 * RENDER_QUANTUM_SIZE); -// let p = response.len(); - -// let num_ir_blocks = p / RENDER_QUANTUM_SIZE; - -// let mut h = vec![Complex::default(); num_ir_blocks * 2 * RENDER_QUANTUM_SIZE]; -// for (resp_fft, resp) in h -// .chunks_mut(2 * RENDER_QUANTUM_SIZE) -// .zip(response.chunks(RENDER_QUANTUM_SIZE)) -// { -// // fill resp_fft with FFT of resp.zero_pad(RENDER_QUANTUM_SIZE) -// fft2.real()[..RENDER_QUANTUM_SIZE].copy_from_slice(resp); -// fft2.real()[RENDER_QUANTUM_SIZE..].fill(0.); -// resp_fft[..fft2.complex().len()].copy_from_slice(fft2.process()); -// } - -// let fdl = vec![Complex::default(); 2 * RENDER_QUANTUM_SIZE * num_ir_blocks]; -// let out = vec![0.; 2 * RENDER_QUANTUM_SIZE - 1]; - -// Self { -// num_ir_blocks, -// h, -// fdl, -// out, -// fft2, -// } -// } - -// fn process(&mut self, input: &[f32], output: &mut [f32]) { -// self.fft2.real()[..RENDER_QUANTUM_SIZE].copy_from_slice(input); -// self.fft2.real()[RENDER_QUANTUM_SIZE..].fill(0.); -// let spectrum = self.fft2.process(); - -// self.fdl -// .chunks_mut(2 * RENDER_QUANTUM_SIZE) -// .zip(self.h.chunks(2 * RENDER_QUANTUM_SIZE)) -// .for_each(|(fdl_c, h_c)| { -// fdl_c -// .iter_mut() -// .zip(h_c) -// .zip(spectrum) -// .for_each(|((f, h), s)| *f += h * s) -// }); - -// let c_len = self.fft2.complex().len(); -// self.fft2.complex().copy_from_slice(&self.fdl[..c_len]); -// let inverse = self.fft2.inverse(); -// self.out.iter_mut().zip(inverse).for_each(|(o, i)| { -// *o += i / (2 * RENDER_QUANTUM_SIZE) as f32; -// }); - -// output.copy_from_slice(&self.out[..RENDER_QUANTUM_SIZE]); - -// roll_zero(&mut self.fdl[..], 2 * RENDER_QUANTUM_SIZE); -// roll_zero(&mut self.out[..], RENDER_QUANTUM_SIZE); -// } - -// fn tail(&mut self, output: &mut AudioRenderQuantum) -> bool { -// if self.num_ir_blocks == 0 { -// output.make_silent(); -// return false; -// } - -// self.num_ir_blocks -= 1; - -// let c_len = self.fft2.complex().len(); -// self.fft2.complex().copy_from_slice(&self.fdl[..c_len]); -// let inverse = self.fft2.inverse(); -// self.out.iter_mut().zip(inverse).for_each(|(o, i)| { -// *o += i / (2 * RENDER_QUANTUM_SIZE) as f32; -// }); - -// output -// .channel_data_mut(0) -// .copy_from_slice(&self.out[..RENDER_QUANTUM_SIZE]); - -// roll_zero(&mut self.fdl[..], 2 * RENDER_QUANTUM_SIZE); -// roll_zero(&mut self.out[..], RENDER_QUANTUM_SIZE); - -// self.num_ir_blocks > 0 -// } -// } - struct ConvolverRenderer { inner: Option>, } @@ -482,13 +306,6 @@ mod tests { use super::*; - // #[test] - // fn test_roll_zero() { - // let mut input = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - // roll_zero(&mut input, 3); - // assert_eq!(&input, &[4, 5, 6, 7, 8, 9, 10, 0, 0, 0]); - // } - #[test] #[should_panic] fn test_buffer_sample_rate_matches() { From 115f04a13f30ed85a7ae6ff07bf5e6605cb32f2e Mon Sep 17 00:00:00 2001 From: Otto Date: Sun, 1 Dec 2024 14:02:11 +0100 Subject: [PATCH 4/7] ConvolverNode: crash explicitly when not able to construct engine --- src/node/convolver.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/node/convolver.rs b/src/node/convolver.rs index 2814a628..19e0ce11 100644 --- a/src/node/convolver.rs +++ b/src/node/convolver.rs @@ -225,7 +225,9 @@ impl ConvolverNode { // Size of the partition changes a lot the perf... // - RENDER_QUANTUM_SIZE -> 20x (compared to real-time) // - RENDER_QUANTUM_SIZE * 8 -> 134x - let _ = convolver.init(RENDER_QUANTUM_SIZE * 8, &samples); + convolver + .init(RENDER_QUANTUM_SIZE * 8, &samples) + .expect("Unable to initialize convolution engine"); // let padded_buffer = AudioBuffer::from(samples, sample_rate); // let convolve = ConvolverRendererInner::new(padded_buffer); From e48ac74cf44e90226a3c2ab5ad3e2f3d2f7d42ad Mon Sep 17 00:00:00 2001 From: b-ma Date: Sun, 1 Dec 2024 16:21:54 +0100 Subject: [PATCH 5/7] feat: apply channelCount and channelCountMode constraints --- src/node/convolver.rs | 81 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 9 deletions(-) diff --git a/src/node/convolver.rs b/src/node/convolver.rs index 19e0ce11..8031de28 100644 --- a/src/node/convolver.rs +++ b/src/node/convolver.rs @@ -11,7 +11,7 @@ use crate::render::{ }; use crate::RENDER_QUANTUM_SIZE; -use super::{AudioNode, AudioNodeOptions, ChannelConfig, ChannelInterpretation}; +use super::{AudioNode, AudioNodeOptions, ChannelConfig, ChannelCountMode, ChannelInterpretation}; /// Scale buffer by an equal-power normalization // see - @@ -59,7 +59,7 @@ fn normalize_buffer(buffer: &AudioBuffer) -> f32 { // AudioBuffer? buffer; // boolean disableNormalization = false; //}; -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] pub struct ConvolverOptions { /// The desired buffer for the ConvolverNode pub buffer: Option, @@ -69,6 +69,53 @@ pub struct ConvolverOptions { pub audio_node_options: AudioNodeOptions, } +impl Default for ConvolverOptions { + fn default() -> Self { + Self { + buffer: Default::default(), + disable_normalization: Default::default(), + audio_node_options: AudioNodeOptions { + channel_count: 2, + channel_count_mode: ChannelCountMode::ClampedMax, + channel_interpretation: ChannelInterpretation::Speakers, + }, + } + } +} + +/// Assert that the channel count is valid for the ConvolverNode +/// see +/// +/// # Panics +/// +/// This function panics if given count is greater than 2 +/// +#[track_caller] +#[inline(always)] +fn assert_valid_channel_count(count: usize) { + assert!( + count <= 2, + "NotSupportedError - ConvolverNode channel count cannot be greater than two" + ); +} + +/// Assert that the channel count mode is valid for the ConvolverNode +/// see +/// +/// # Panics +/// +/// This function panics if given count mode is [`ChannelCountMode::Max`] +/// +#[track_caller] +#[inline(always)] +fn assert_valid_channel_count_mode(mode: ChannelCountMode) { + assert_ne!( + mode, + ChannelCountMode::Max, + "NotSupportedError - ConvolverNode channel count mode cannot be set to max" + ); +} + /// Processing node which applies a linear convolution effect given an impulse response. /// /// - MDN documentation: @@ -137,6 +184,19 @@ impl AudioNode for ConvolverNode { fn number_of_outputs(&self) -> usize { 1 } + + // see + fn set_channel_count(&self, count: usize) { + assert_valid_channel_count(count); + self.channel_config.set_count(count, self.registration()); + } + + // see + fn set_channel_count_mode(&self, mode: ChannelCountMode) { + assert_valid_channel_count_mode(mode); + self.channel_config + .set_count_mode(mode, self.registration()); + } } impl ConvolverNode { @@ -155,15 +215,18 @@ impl ConvolverNode { let ConvolverOptions { buffer, disable_normalization, - audio_node_options: channel_config, + audio_node_options, } = options; + assert_valid_channel_count(audio_node_options.channel_count); + assert_valid_channel_count_mode(audio_node_options.channel_count_mode); + let mut node = context.base().register(move |registration| { let renderer = ConvolverRenderer { inner: None }; let node = Self { registration, - channel_config: channel_config.into(), + channel_config: audio_node_options.into(), normalize: !disable_normalization, buffer: None, }; @@ -273,11 +336,6 @@ impl AudioProcessor for ConvolverRenderer { Some(convolver) => convolver, }; - // handle tail time - // if input.is_silent() { - // return convolver.tail(output); - // } - let mut mono = input.clone(); mono.mix(1, ChannelInterpretation::Speakers); let input = &mono.channel_data(0)[..]; @@ -285,6 +343,11 @@ impl AudioProcessor for ConvolverRenderer { let _ = convolver.process(input, output); + // handle tail time + // if input.is_silent() { + // return convolver.tail(output); + // } + true } From fefec2603b4864dcc7dcb4eb1c01066bde892010 Mon Sep 17 00:00:00 2001 From: b-ma Date: Sun, 1 Dec 2024 18:05:47 +0100 Subject: [PATCH 6/7] fix: reimplement tail time + prepare multichannel --- src/node/convolver.rs | 89 ++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/src/node/convolver.rs b/src/node/convolver.rs index 8031de28..b29fd576 100644 --- a/src/node/convolver.rs +++ b/src/node/convolver.rs @@ -72,8 +72,8 @@ pub struct ConvolverOptions { impl Default for ConvolverOptions { fn default() -> Self { Self { - buffer: Default::default(), - disable_normalization: Default::default(), + buffer: None, + disable_normalization: false, audio_node_options: AudioNodeOptions { channel_count: 2, channel_count_mode: ChannelCountMode::ClampedMax, @@ -222,7 +222,11 @@ impl ConvolverNode { assert_valid_channel_count_mode(audio_node_options.channel_count_mode); let mut node = context.base().register(move |registration| { - let renderer = ConvolverRenderer { inner: None }; + let renderer = ConvolverRenderer { + convolvers: None, + impulse_length: 0, + tail_count: 0, + }; let node = Self { registration, @@ -278,24 +282,33 @@ impl ConvolverNode { 1. }; - let mut samples = vec![0.; buffer.length()]; - samples - .iter_mut() - .zip(buffer.get_channel_data(0)) - .for_each(|(o, i)| *o = *i * scale); - - let mut convolver = FFTConvolver::::default(); + let mut convolvers = Vec::>::new(); // Size of the partition changes a lot the perf... // - RENDER_QUANTUM_SIZE -> 20x (compared to real-time) // - RENDER_QUANTUM_SIZE * 8 -> 134x - convolver - .init(RENDER_QUANTUM_SIZE * 8, &samples) - .expect("Unable to initialize convolution engine"); + let partition_size = RENDER_QUANTUM_SIZE * 8; + + [0..buffer.number_of_channels()].iter().for_each(|_| { + let mut scaled_channel = vec![0.; buffer.length()]; + scaled_channel + .iter_mut() + .zip(buffer.get_channel_data(0)) + .for_each(|(o, i)| *o = *i * scale); + + let mut convolver = FFTConvolver::::default(); + convolver + .init(RENDER_QUANTUM_SIZE * 8, &samples) + .expect("Unable to initialize convolution engine"); - // let padded_buffer = AudioBuffer::from(samples, sample_rate); - // let convolve = ConvolverRendererInner::new(padded_buffer); + convolvers.push(convolver); + }); + + let msg = ConvolverInfosMessage { + convolvers: Some(convolvers), + impulse_length: buffer.length(), + }; - self.registration.post_message(Some(convolver)); + self.registration.post_message(msg); self.buffer = Some(buffer); } @@ -310,8 +323,15 @@ impl ConvolverNode { } } +struct ConvolverInfosMessage { + convolvers: Option>>, + impulse_length: usize, +} + struct ConvolverRenderer { - inner: Option>, + convolvers: Option>>, + impulse_length: usize, + tail_count: usize, } impl AudioProcessor for ConvolverRenderer { @@ -327,34 +347,49 @@ impl AudioProcessor for ConvolverRenderer { let output = &mut outputs[0]; output.force_mono(); - let convolver = match &mut self.inner { + let convolvers = match &mut self.convolvers { None => { // no convolution buffer set, passthrough *output = input.clone(); return !input.is_silent(); } - Some(convolver) => convolver, + Some(convolvers) => convolvers, }; + // @todo - https://webaudio.github.io/web-audio-api/#Convolution-channel-configurations let mut mono = input.clone(); mono.mix(1, ChannelInterpretation::Speakers); - let input = &mono.channel_data(0)[..]; - let output = &mut output.channel_data_mut(0)[..]; - let _ = convolver.process(input, output); + // let input = &mono.channel_data(0)[..]; + // let output = &mut output.channel_data_mut(0)[..]; + let _ = convolvers[0].process(&mono.channel_data(0), &mut output.channel_data_mut(0)); // handle tail time - // if input.is_silent() { - // return convolver.tail(output); - // } + if input.is_silent() { + self.tail_count += RENDER_QUANTUM_SIZE; + + if self.tail_count >= self.impulse_length { + return false; + } else { + return true + } + } + + self.tail_count = 0; true } fn onmessage(&mut self, msg: &mut dyn Any) { - if let Some(convolver) = msg.downcast_mut::>>() { + if let Some(msg) = msg.downcast_mut::() { + let ConvolverInfosMessage { + convolvers, + impulse_length, + } = msg; // Avoid deallocation in the render thread by swapping the convolver. - std::mem::swap(&mut self.inner, convolver); + std::mem::swap(&mut self.convolvers, convolvers); + self.impulse_length = *impulse_length; + return; } From a144d9bee686a7106cb1bb7f29db6256cefc9a39 Mon Sep 17 00:00:00 2001 From: b-ma Date: Mon, 2 Dec 2024 08:37:22 +0100 Subject: [PATCH 7/7] fix: explicitly declare partition size --- src/node/convolver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/convolver.rs b/src/node/convolver.rs index b29fd576..e970ea89 100644 --- a/src/node/convolver.rs +++ b/src/node/convolver.rs @@ -297,7 +297,7 @@ impl ConvolverNode { let mut convolver = FFTConvolver::::default(); convolver - .init(RENDER_QUANTUM_SIZE * 8, &samples) + .init(partition_size, &scaled_channel) .expect("Unable to initialize convolution engine"); convolvers.push(convolver);