refactor(fe/audio): Change to use audio element pool

frontend/apps/Cargo.toml

@@ -123,6 +123,7 @@ web-sys = { version = "0.3.55", features = [
     'ScrollBehavior',
     'ScrollIntoViewOptions',
     'Performance',
+    'console',
 ] }
 
 [profile.release]

frontend/apps/crates/components/Cargo.toml

@@ -49,6 +49,7 @@ quiet = ["utils/quiet"]
 local = ["quiet"]
 release = []
 sandbox = []
+iframe_audio = []
 
 animation = []
 audio_input = []

frontend/apps/crates/components/src/audio/mixer/mixer.rs

@@ -15,9 +15,6 @@ use web_sys::{HtmlIFrameElement, MessageEvent};
 
 use super::{mixer_iframe::AudioMixerIframe, mixer_top::AudioMixerTop};
 
-// TODO: Currently initializing a AUDIO_MIXER in all window levels but only really used in player,
-// might be a good idea to only initialize in player
-
 thread_local! {
     pub static AUDIO_MIXER:AudioMixer = AudioMixer::new()
 }
@@ -44,6 +41,8 @@ fn setup_iframe_to_parent_listener() {
     );
 }
 
+// Might make sense to only have one kind on each entry and use conditional compilation to set correct kind.
+// But the compiler might be doing that already.
 pub struct AudioMixer {
     kind: AudioMixerKind,
     settings: Rc<RefCell<AudioSettings>>,
@@ -85,14 +84,14 @@ impl AudioMixer {
     /// Oneshots are AudioClips because they drop themselves
     /// They're intended solely to be kicked off and not being held anywhere
     pub fn play_oneshot<A: Into<AudioSource>>(&self, audio: A) {
-        let path = match audio.into() {
+        let url = match audio.into() {
             AudioSource::Url(audio_path) => audio_path,
             AudioSource::Buffer(_) => todo!(),
         };
         let handle_id = AudioHandleId::new();
         let audio_message = PlayAudioMessage {
             handle_id: handle_id.clone(),
-            path,
+            url,
             auto_play: true,
             is_loop: false,
         };
@@ -109,14 +108,14 @@ impl AudioMixer {
         F: FnMut() + 'static,
         A: Into<AudioSource>,
     {
-        let path = match audio.into() {
+        let url = match audio.into() {
             AudioSource::Url(audio_path) => audio_path,
             AudioSource::Buffer(_) => todo!(),
         };
         let handle_id = AudioHandleId::new();
         let audio_message = PlayAudioMessage {
             handle_id: handle_id.clone(),
-            path,
+            url,
             auto_play: true,
             is_loop: false,
         };
@@ -133,14 +132,14 @@ impl AudioMixer {
 
     /// Play a clip and get a Handle to hold (simple API around add_source)
     pub fn play<A: Into<AudioSource>>(&self, audio: A, is_loop: bool) -> AudioHandle {
-        let path = match audio.into() {
+        let url = match audio.into() {
             AudioSource::Url(audio_path) => audio_path,
             AudioSource::Buffer(_) => todo!(),
         };
         let handle = AudioHandle::new();
         let audio_message = PlayAudioMessage {
             handle_id: handle.id().clone(),
-            path,
+            url,
             auto_play: true,
             is_loop,
         };
@@ -153,14 +152,14 @@ impl AudioMixer {
         F: FnMut() + 'static,
         A: Into<AudioSource>,
     {
-        let path = match audio.into() {
+        let url = match audio.into() {
             AudioSource::Url(audio_path) => audio_path,
             AudioSource::Buffer(_) => todo!(),
         };
         let handle = AudioHandle::new();
         let audio_message = PlayAudioMessage {
             handle_id: handle.id().clone(),
-            path,
+            url,
             auto_play: true,
             is_loop,
         };
@@ -177,14 +176,14 @@ impl AudioMixer {
     where
         F: FnMut() + 'static,
     {
-        let path = match audio.into() {
+        let url = match audio.into() {
             AudioSource::Url(audio_path) => audio_path,
             AudioSource::Buffer(_) => todo!(),
         };
         let handle = AudioHandle::new();
         let audio_message = PlayAudioMessage {
             handle_id: handle.id().clone(),
-            path,
+            url,
             auto_play: options.auto_play,
             is_loop: options.is_loop,
         };
@@ -208,8 +207,6 @@ impl AudioMixer {
 /// Private methods
 impl AudioMixer {
     fn new() -> Self {
-        log::info!("initializing AUDIO_MIXER");
-
         setup_iframe_to_parent_listener();
 
         // once initialized broadcast context available
@@ -222,12 +219,12 @@ impl AudioMixer {
         .forget();
 
         Self {
-            kind: match is_iframe() {
+            kind: match use_iframe_audio() {
                 true => AudioMixerKind::Iframe(AudioMixerIframe::new()),
                 false => AudioMixerKind::Top(AudioMixerTop::new()),
             },
             callbacks: Default::default(),
-            context_available: RefCell::new(false),
+            context_available: RefCell::new(false), // corresponds to AudioMixerTop.already_played
             settings: Default::default(),
             rng: RefCell::new(thread_rng()),
             iframes: Default::default(),
@@ -284,7 +281,6 @@ impl AudioMixer {
     }
 
     pub(super) fn set_context_available(&self, available: bool) {
-        log::info!("set_context_available");
         *self.context_available.borrow_mut() = available;
     }
 }
@@ -344,7 +340,7 @@ pub(super) enum AudioMessageToTop {
 
 #[derive(Debug, Deserialize, Serialize)]
 pub(super) struct PlayAudioMessage {
-    pub path: String,
+    pub url: String,
     pub auto_play: bool,
     pub is_loop: bool,
     pub handle_id: AudioHandleId,
@@ -594,3 +590,11 @@ impl From<jig::AudioFeedbackNegative> for AudioPath<'_> {
         }))
     }
 }
+
+fn use_iframe_audio() -> bool {
+    // if local and is top window don't use iframe
+    if cfg!(feature = "local") && !is_iframe() {
+        return false;
+    }
+    cfg!(feature = "iframe_audio")
+}

frontend/apps/crates/components/src/audio/mixer/mixer_top.rs

@@ -1,24 +1,34 @@
 use super::{AudioHandleId, AudioMessageFromTop, AudioMessageToTop, PlayAudioMessage, AUDIO_MIXER};
-use awsm_web::audio::AudioMixer as AwsmAudioMixer;
-pub use awsm_web::audio::{
-    AudioClip, AudioClipOptions, AudioHandle as AwsmWebAudioHandle, AudioSource, Id,
-    WeakAudioHandle,
-};
-use std::{cell::RefCell, collections::HashMap, rc::Rc};
-use utils::prelude::*;
-
-//inherently cloneable, conceptually like it's wrapped in Rc itself
-#[derive(Clone)]
+use dominator::clone;
+use gloo_timers::future::TimeoutFuture;
+use itertools::Itertools;
+use std::{cell::RefCell, collections::HashMap};
+use utils::{js_wrappers::set_event_listener, prelude::*};
+use wasm_bindgen_futures::spawn_local;
+use web_sys::{AudioContext, Event, HtmlAudioElement};
+
+const EMPTY_AUDIO_URL: &str = "data:audio/mpeg;base64,//uQxAAAAAAAAAAAAAAAAAAAAAAASW5mbwAAAA8AAAABAAADQgD///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8AAAA6TEFNRTMuMTAwAc0AAAAAAAAAABSAJAJAQgAAgAAAA0LqRHv+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA//uQxAADwAABpAAAACAAADSAAAAETEFNRTMuMTAwVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV";
+
+thread_local! {
+    // using vec of tuples instead of hashmap because HtmlAudioElement doesn't implement Hash
+    static ENDED_CALLBACKS: RefCell<Vec<(HtmlAudioElement, Box<dyn FnMut()>)>> = Default::default();
+}
+
 pub struct AudioMixerTop {
-    pub(super) inner: Rc<AwsmAudioMixer>,
-    pub(super) awsm_handles: RefCell<HashMap<AudioHandleId, AwsmWebAudioHandle>>,
+    audio_context: AudioContext,
+    active: RefCell<HashMap<AudioHandleId, HtmlAudioElement>>,
+    inactive: RefCell<Vec<HtmlAudioElement>>,
+    already_played: RefCell<bool>,
 }
 
 impl AudioMixerTop {
     pub(super) fn new() -> Self {
+        let audio_context = create_audio_context();
         Self {
-            inner: Rc::new(AwsmAudioMixer::new(None)),
-            awsm_handles: Default::default(),
+            audio_context,
+            active: Default::default(),
+            inactive: Default::default(),
+            already_played: RefCell::new(false),
         }
     }
 
@@ -40,55 +50,123 @@ impl AudioMixerTop {
                 self.handle_dropped(handle_id);
             }
             AudioMessageToTop::PauseAll => {
-                self.inner.pause_all();
+                for (_, el) in self.active.borrow().iter() {
+                    let _ = el.pause();
+                }
             }
             AudioMessageToTop::PlayAll => {
-                self.inner.play_all();
+                for (_, el) in self.active.borrow().iter() {
+                    let _ = el.play();
+                }
             }
             AudioMessageToTop::BroadcastContextAvailable => {
                 self.broadcast_context_available_request();
             }
         }
     }
 
+    fn init_if_not_ready(&self) {
+        if !*self.already_played.borrow() {
+            *self.already_played.borrow_mut() = true;
+            *self.inactive.borrow_mut() = init_empty_audio_elements(10, &self.audio_context);
+        }
+    }
+
     fn play<F: FnMut() + 'static>(&self, audio_message: PlayAudioMessage, on_ended: F) {
-        let awsm_handle = self
-            .inner
-            .play_on_ended(
-                AudioSource::Url(audio_message.path.clone()),
-                audio_message.is_loop,
-                on_ended,
-            )
-            .unwrap_ji();
-        self.awsm_handles
-            .borrow_mut()
-            .insert(audio_message.handle_id, awsm_handle);
+        self.init_if_not_ready();
+
+        // Unwrapping. Should never exceed number of items in pool
+        let el = self.inactive.borrow_mut().pop().unwrap_ji();
+        el.set_src(&audio_message.url);
+        el.set_loop(audio_message.is_loop);
+        ENDED_CALLBACKS.with(|ended_callbacks| {
+            ended_callbacks
+                .borrow_mut()
+                .push((el.clone(), Box::new(on_ended)));
+        });
+
+        let _ = el.play();
+        self.active.borrow_mut().insert(audio_message.handle_id, el);
     }
 
     fn handle_dropped(&self, handle_id: AudioHandleId) {
-        let mut awsm_handles = self.awsm_handles.borrow_mut();
-        awsm_handles.remove(&handle_id);
+        let mut active = self.active.borrow_mut();
+        let el = active.remove(&handle_id);
+        if let Some(el) = &el {
+            spawn_local(clone!(el => async move {
+                // wait for next cycle as ended_callbacks is currently locked because handle_dropped is called from within a callback
+                TimeoutFuture::new(0).await;
+                ENDED_CALLBACKS.with(clone!(el => move |ended_callbacks| {
+                    let mut ended_callbacks = ended_callbacks.borrow_mut();
+                    if let Some(index) = ended_callbacks.iter().position(|(el2, _)| el2 == &el) {
+                        let _ = ended_callbacks.remove(index);
+                    }
+                }));
+            }));
+        }
+        if let Some(el) = el {
+            self.inactive.borrow_mut().push(el);
+        }
     }
 
     fn pause_handle_called(&self, handle_id: AudioHandleId) {
-        let awsm_handles = self.awsm_handles.borrow();
-        if let Some(audio) = awsm_handles.get(&handle_id) {
-            audio.pause();
+        let active = self.active.borrow();
+        if let Some(audio) = active.get(&handle_id) {
+            let _ = audio.pause();
         }
     }
 
     fn play_handle_called(&self, handle_id: AudioHandleId) {
-        let awsm_handles = self.awsm_handles.borrow();
-        if let Some(audio) = awsm_handles.get(&handle_id) {
-            audio.play();
+        let active = self.active.borrow();
+        if let Some(audio) = active.get(&handle_id) {
+            let _ = audio.play();
         }
     }
 
     fn broadcast_context_available_request(&self) {
-        let available = self.inner.context_available();
+        let available = *self.already_played.borrow();
         AUDIO_MIXER.with(|mixer| {
             mixer.set_context_available(available);
             mixer.message_all_iframes(AudioMessageFromTop::ContextAvailable(available));
         });
     }
 }
+
+fn init_empty_audio_elements(count: usize, context: &AudioContext) -> Vec<HtmlAudioElement> {
+    (0..count)
+        .map(|_| create_audio_element_on_context(context))
+        .collect_vec()
+}
+
+fn create_audio_element_on_context(context: &AudioContext) -> HtmlAudioElement {
+    let el = HtmlAudioElement::new().unwrap_ji();
+    el.set_src(EMPTY_AUDIO_URL);
+    el.set_cross_origin(Some("anonymous"));
+    let track = context.create_media_element_source(&el).unwrap_ji();
+    let _ = track.connect_with_audio_node(&context.destination());
+    el.load();
+    set_event_listener(
+        &el,
+        "ended",
+        Box::new(clone!(el => move |_: Event| {
+            ENDED_CALLBACKS.with(clone!(el => move |ended_callbacks| {
+                let mut ended_callbacks = ended_callbacks.borrow_mut();
+                let callback = ended_callbacks.iter_mut().find_map(|(el2, callback)| {
+                    if el2 == &el {
+                        Some(callback)
+                    } else {
+                        None
+                    }
+                });
+                if let Some(callback) = callback {
+                    (callback)();
+                }
+            }));
+        })),
+    );
+    el
+}
+
+fn create_audio_context() -> AudioContext {
+    AudioContext::new().unwrap_ji()
+}