#715 Make realearn_timestamp sample-accurate for MIDI events

- calculate timestamp based on audio callbacks
- respect the frame offset for MIDI events

Cargo.toml

@@ -120,6 +120,7 @@ pathdiff = "0.2.1"
 open = "5.0.1"
 url = "2.5.2"
 atomic = "0.6.0"
+static_assertions = "1.1.0"
 
 [profile.release]
 # This is important for having line numbers in bug reports.

main/Cargo.toml

@@ -207,6 +207,8 @@ palette.workspace = true
 serde_plain.workspace = true
 # For sharing variables between main thread and real-time threads
 atomic.workspace = true
+# For making sure that sharing global audio state uses atomics
+static_assertions.workspace = true
 
 [target.'cfg(any(target_os = "windows", target_os = "macos"))'.dependencies]
 # For speech source

main/src/domain/accelerator.rs

@@ -44,7 +44,7 @@ where
     ///
     /// If yes, we should completely "eat" the message and don't do anything else with it.
     fn process_control(&mut self, msg: KeyMessage) -> bool {
-        let evt = ControlEvent::new(msg, ControlEventTimestamp::now());
+        let evt = ControlEvent::new(msg, ControlEventTimestamp::from_main_thread());
         let mut filter_out_event = false;
         let mut notified_backbone = false;
         for proc in &mut *self.main_processors.borrow_mut() {

main/src/domain/audio_hook.rs

@@ -2,21 +2,23 @@ use crate::domain::{
     classify_midi_message, AudioBlockProps, ControlEvent, ControlEventTimestamp,
     DisplayAsPrettyHex, IncomingMidiMessage, InstanceId, MidiControlInput, MidiEvent,
     MidiMessageClassification, MidiScanResult, MidiScanner, MidiTransformationContainer,
-    RealTimeProcessor, SharedRealTimeInstance, UnitId,
+    RealTimeProcessor, SharedRealTimeInstance, UnitId, GLOBAL_AUDIO_STATE,
 };
 use base::byte_pattern::{BytePattern, PatternByte};
 use base::metrics_util::{measure_time, record_duration};
 use base::non_blocking_lock;
 use helgoboss_learn::{AbstractTimestamp, MidiSourceValue, RawMidiEvent, RawMidiEvents};
 use helgoboss_midi::{DataEntryByteOrder, RawShortMessage, ShortMessage, ShortMessageType};
 use helgobox_allocator::*;
+use reaper_common_types::DurationInSeconds;
 use reaper_high::{MidiInputDevice, MidiOutputDevice, Reaper};
 use reaper_medium::{
     MidiInputDeviceId, MidiOutputDeviceId, OnAudioBuffer, OnAudioBufferArgs, SendMidiTime,
+    MIDI_INPUT_FRAME_RATE,
 };
 use smallvec::SmallVec;
 use std::fmt::{Display, Formatter};
-use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex, MutexGuard, OnceLock};
 use std::time::{Duration, Instant};
 use tinyvec::ArrayVec;
@@ -134,7 +136,6 @@ pub struct RealearnAudioHook {
     feedback_task_receiver: crossbeam_channel::Receiver<FeedbackAudioHookTask>,
     time_of_last_run: Option<Instant>,
     initialized: bool,
-    counter: Arc<AtomicU32>,
     midi_transformation_container: MidiTransformationContainer,
     #[cfg(feature = "playtime")]
     clip_engine_audio_hook: playtime_clip_engine::rt::audio_hook::PlaytimeAudioHook,
@@ -155,7 +156,6 @@ impl RealearnAudioHook {
     pub fn new(
         normal_task_receiver: crossbeam_channel::Receiver<NormalAudioHookTask>,
         feedback_task_receiver: crossbeam_channel::Receiver<FeedbackAudioHookTask>,
-        counter: Arc<AtomicU32>,
     ) -> RealearnAudioHook {
         Self {
             state: AudioHookState::Normal,
@@ -166,7 +166,6 @@ impl RealearnAudioHook {
             feedback_task_receiver,
             time_of_last_run: None,
             initialized: false,
-            counter,
             midi_transformation_container: MidiTransformationContainer::new(),
             #[cfg(feature = "playtime")]
             clip_engine_audio_hook: playtime_clip_engine::rt::audio_hook::PlaytimeAudioHook::new(),
@@ -204,8 +203,9 @@ impl RealearnAudioHook {
         let current_time = Instant::now();
         let time_of_last_run = self.time_of_last_run.replace(current_time);
         // Increment counter
-        self.counter.fetch_add(1, Ordering::Relaxed);
         let block_props = AudioBlockProps::from_on_audio_buffer_args(&args);
+        let block_count = GLOBAL_AUDIO_STATE.advance(block_props);
+        let sample_count = block_count * args.len as u64;
         // Call ReaLearn real-time processors (= process MIDI messages coming in from hardware devices).
         // We do this here already, *before* pre-polling recording and advancing Playtime's tempo buffer (done in `on_pre_poll_1`)!
         // Reason: When recording a new clip with tempo detection (= recording in silence mode), it's ideal
@@ -238,7 +238,7 @@ impl RealearnAudioHook {
                 false
             }
         };
-        self.call_real_time_processors(block_props, might_be_rebirth);
+        self.call_real_time_processors(block_props, sample_count, might_be_rebirth);
         // Process ReaLearn feedback commands
         self.process_feedback_commands();
         // Process incoming commands, including Playtime commands
@@ -317,14 +317,18 @@ impl RealearnAudioHook {
         }
     }
 
-    fn call_real_time_processors(&mut self, block_props: AudioBlockProps, might_be_rebirth: bool) {
+    fn call_real_time_processors(
+        &mut self,
+        block_props: AudioBlockProps,
+        sample_count: u64,
+        might_be_rebirth: bool,
+    ) {
         match &mut self.state {
             AudioHookState::Normal => {
-                let timestamp = ControlEventTimestamp::now();
                 self.call_real_time_processors_in_normal_state(
                     block_props,
                     might_be_rebirth,
-                    timestamp,
+                    sample_count,
                 );
             }
             AudioHookState::LearningSource {
@@ -358,7 +362,7 @@ impl RealearnAudioHook {
         &mut self,
         block_props: AudioBlockProps,
         might_be_rebirth: bool,
-        timestamp: ControlEventTimestamp,
+        sample_count: u64,
     ) {
         // 1a. Drive real-time processors and determine used MIDI devices "on the go".
         //
@@ -371,6 +375,11 @@ impl RealearnAudioHook {
         //
         let mut midi_dev_id_is_used = [false; MidiInputDeviceId::MAX_DEVICE_COUNT as usize];
         let mut midi_devs_used_at_all = false;
+        let start_of_block_timestamp = ControlEventTimestamp::from_rt(
+            sample_count,
+            block_props.frame_rate,
+            DurationInSeconds::ZERO,
+        );
         for (_, p) in self.real_time_processors.iter() {
             // Since 1.12.0, we "drive" each plug-in instance's real-time processor
             // primarily by the global audio hook. See https://github.com/helgoboss/helgobox/issues/84 why this is
@@ -379,7 +388,7 @@ impl RealearnAudioHook {
             // stop doing so synchronously if the plug-in is
             // gone.
             let mut guard = p.lock_recover();
-            guard.run_from_audio_hook_all(block_props, might_be_rebirth, timestamp);
+            guard.run_from_audio_hook_all(block_props, might_be_rebirth, start_of_block_timestamp);
             if guard.control_is_globally_enabled() {
                 if let MidiControlInput::Device(dev_id) = guard.midi_control_input() {
                     midi_dev_id_is_used[dev_id.get() as usize] = true;
@@ -390,15 +399,19 @@ impl RealearnAudioHook {
         // 1b. Forward MIDI events from MIDI devices to ReaLearn instances and filter
         //     them globally if desired by the instance.
         if midi_devs_used_at_all {
-            self.distribute_midi_events_to_processors(block_props, &midi_dev_id_is_used, timestamp);
+            self.distribute_midi_events_to_processors(
+                block_props,
+                &midi_dev_id_is_used,
+                sample_count,
+            );
         }
     }
 
     fn distribute_midi_events_to_processors(
         &mut self,
         block_props: AudioBlockProps,
         midi_dev_id_is_used: &[bool; MidiInputDeviceId::MAX_DEVICE_COUNT as usize],
-        timestamp: ControlEventTimestamp,
+        sample_count: u64,
     ) {
         self.midi_transformation_container
             .prepare(block_props.frame_rate);
@@ -420,6 +433,13 @@ impl RealearnAudioHook {
                                 Err(_) => continue,
                                 Ok(e) => e,
                             };
+                        let frame_offset_in_secs =
+                            res.midi_event.frame_offset() as f64 / MIDI_INPUT_FRAME_RATE.get();
+                        let timestamp = ControlEventTimestamp::from_rt(
+                            sample_count,
+                            block_props.frame_rate,
+                            DurationInSeconds::new_panic(frame_offset_in_secs),
+                        );
                         let our_event = ControlEvent::new(our_event, timestamp);
                         let mut filter_out_event = false;
                         for (_, p) in self.real_time_processors.iter() {

main/src/domain/control_event.rs

@@ -1,4 +1,6 @@
+use crate::domain::GLOBAL_AUDIO_STATE;
 use helgoboss_learn::AbstractTimestamp;
+use reaper_common_types::{DurationInSeconds, Hz};
 use std::fmt::{Display, Formatter};
 use std::ops::Sub;
 use std::sync::LazyLock;
@@ -11,24 +13,44 @@ pub type ControlEvent<P> = helgoboss_learn::ControlEvent<P, ControlEventTimestam
 // Don't expose the inner field, it should stay private. We might swap the time unit in future to
 // improve performance and accuracy.
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
-pub struct ControlEventTimestamp(Instant);
+pub struct ControlEventTimestamp(Duration);
 
-impl AbstractTimestamp for ControlEventTimestamp {
-    fn now() -> Self {
-        Self(Instant::now())
+impl ControlEventTimestamp {
+    pub fn from_main_thread() -> Self {
+        let block_count = GLOBAL_AUDIO_STATE.load_block_count();
+        let block_size = GLOBAL_AUDIO_STATE.load_block_size();
+        let sample_count = block_count * block_size as u64;
+        Self::from_rt(
+            sample_count,
+            GLOBAL_AUDIO_STATE.load_sample_rate(),
+            DurationInSeconds::ZERO,
+        )
     }
+}
 
+impl AbstractTimestamp for ControlEventTimestamp {
     fn duration(&self) -> Duration {
-        static INSTANT: LazyLock<Instant> = LazyLock::new(|| Instant::now());
-        self.0.saturating_duration_since(*INSTANT)
+        self.0
+    }
+}
+
+impl ControlEventTimestamp {
+    pub fn from_rt(
+        sample_count: u64,
+        sample_rate: Hz,
+        intra_block_offset: DurationInSeconds,
+    ) -> Self {
+        let start_secs = sample_count as f64 / sample_rate.get();
+        let final_secs = start_secs + intra_block_offset.get();
+        Self(Duration::from_secs_f64(final_secs))
     }
 }
 
 impl Sub for ControlEventTimestamp {
     type Output = Duration;
 
     fn sub(self, rhs: Self) -> Self::Output {
-        self.0 - rhs.0
+        self.0.saturating_sub(rhs.0)
     }
 }
 

main/src/domain/control_surface.rs

@@ -253,7 +253,7 @@ impl<EH: DomainEventHandler> RealearnControlSurfaceMiddleware<EH> {
     }
 
     fn run_internal(&mut self) {
-        let timestamp = ControlEventTimestamp::now();
+        let timestamp = ControlEventTimestamp::from_main_thread();
         #[cfg(debug_assertions)]
         {
             // TODO-high-playtime-refactoring This is propagated using main processors but it's a global event. We

main/src/domain/global_audio_state.rs

@@ -0,0 +1,55 @@
+use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
+
+use atomic::Atomic;
+use reaper_medium::Hz;
+use static_assertions::const_assert;
+
+use crate::domain::AudioBlockProps;
+
+pub static GLOBAL_AUDIO_STATE: GlobalAudioState = GlobalAudioState::new();
+
+#[derive(Debug)]
+pub struct GlobalAudioState {
+    block_count: AtomicU64,
+    block_size: AtomicU32,
+    sample_rate: Atomic<f64>,
+}
+
+impl Default for GlobalAudioState {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl GlobalAudioState {
+    pub const fn new() -> Self {
+        const_assert!(Atomic::<f64>::is_lock_free());
+        Self {
+            block_count: AtomicU64::new(0),
+            block_size: AtomicU32::new(0),
+            sample_rate: Atomic::new(1.0),
+        }
+    }
+
+    /// Returns previous block count
+    pub fn advance(&self, block_props: AudioBlockProps) -> u64 {
+        let prev_block_count = self.block_count.fetch_add(1, Ordering::Relaxed);
+        self.block_size
+            .store(block_props.block_length as u32, Ordering::Relaxed);
+        self.sample_rate
+            .store(block_props.frame_rate.get(), Ordering::Relaxed);
+        prev_block_count
+    }
+
+    pub fn load_block_count(&self) -> u64 {
+        self.block_count.load(Ordering::Relaxed)
+    }
+
+    pub fn load_block_size(&self) -> u32 {
+        self.block_size.load(Ordering::Relaxed)
+    }
+
+    pub fn load_sample_rate(&self) -> Hz {
+        Hz::new_panic(self.sample_rate.load(Ordering::Relaxed))
+    }
+}

main/src/domain/main_processor.rs

@@ -979,9 +979,11 @@ impl<EH: DomainEventHandler> MainProcessor<EH> {
                 UpdateAllParams(params) => {
                     self.update_all_params(params);
                 }
-                UpdateSingleParamValue { index, value } => {
-                    self.update_single_param_value(index, value)
-                }
+                UpdateSingleParamValue {
+                    index,
+                    value,
+                    timestamp,
+                } => self.update_single_param_value(index, value, timestamp),
             }
             count += 1;
             if count == PARAMETER_TASK_BULK_SIZE {
@@ -992,7 +994,12 @@ impl<EH: DomainEventHandler> MainProcessor<EH> {
 
     // https://github.com/rust-lang/rust-clippy/issues/6066
     #[allow(clippy::needless_collect)]
-    fn update_single_param_value(&mut self, index: PluginParamIndex, value: RawParamValue) {
+    fn update_single_param_value(
+        &mut self,
+        index: PluginParamIndex,
+        value: RawParamValue,
+        timestamp: ControlEventTimestamp,
+    ) {
         debug!("Updating parameter {} to {}...", index, value);
         // Work around REAPER's inability to notify about parameter changes in
         // monitoring FX by simulating the notification ourselves.
@@ -1052,10 +1059,7 @@ impl<EH: DomainEventHandler> MainProcessor<EH> {
         if self.basics.settings.real_input_logging_enabled {
             self.log_incoming_message(&control_msg);
         }
-        let control_event = ControlEvent::new(
-            MainSourceMessage::Reaper(&control_msg),
-            ControlEventTimestamp::now(),
-        );
+        let control_event = ControlEvent::new(MainSourceMessage::Reaper(&control_msg), timestamp);
         self.process_incoming_message_internal(control_event);
     }
 
@@ -2982,6 +2986,7 @@ pub enum ParameterMainTask {
     UpdateSingleParamValue {
         index: PluginParamIndex,
         value: RawParamValue,
+        timestamp: ControlEventTimestamp,
     },
     UpdateAllParams(PluginParams),
 }

main/src/domain/midi_types.rs

@@ -1,3 +1,4 @@
+use reaper_common_types::DurationInSeconds;
 use reaper_medium::{Hz, MIDI_INPUT_FRAME_RATE};
 
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
@@ -48,4 +49,8 @@ impl SampleOffset {
     pub fn get(self) -> u64 {
         self.0
     }
+
+    pub fn to_seconds(&self, sample_rate: Hz) -> DurationInSeconds {
+        DurationInSeconds::new_panic(self.0 as f64 / sample_rate.get())
+    }
 }

main/src/domain/mod.rs

@@ -170,3 +170,6 @@ mod playtime_util;
 
 mod hex;
 pub use hex::*;
+
+mod global_audio_state;
+pub use global_audio_state::*;