From 20f8c3e792de60492d41aebca509f21af108797c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 2 Feb 2025 04:15:36 +0900 Subject: [PATCH] Audio engine observer fixes (#571) --- .../AudioDeviceModuleDelegateAdapter.swift | 16 +++---- .../LiveKit/Audio/AudioEngineObserver.swift | 47 ++++++++++++------- .../Audio/DefaultAudioSessionObserver.swift | 13 ++--- Sources/LiveKit/Protocols/NextInvokable.swift | 2 +- Sources/LiveKit/Track/AudioManager.swift | 13 +++++ Tests/LiveKitTests/AudioEngineTests.swift | 6 ++- 6 files changed, 62 insertions(+), 35 deletions(-) diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift index 1c3710e95..67aa02c43 100644 --- a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -40,49 +40,49 @@ class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate func audioDeviceModule(_: LKRTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) { guard let audioManager else { return } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() entryPoint?.engineDidCreate(engine) } func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() entryPoint?.engineWillStart(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() entryPoint?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() entryPoint?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine) { guard let audioManager else { return } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() entryPoint?.engineWillRelease(engine) } func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode?, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false } func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode?, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } - let entryPoint = audioManager._state.engineObservers.buildChain() + let entryPoint = audioManager.buildEngineObserverChain() return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false } } diff --git a/Sources/LiveKit/Audio/AudioEngineObserver.swift b/Sources/LiveKit/Audio/AudioEngineObserver.swift index b3f6cc9d5..8f91df087 100644 --- a/Sources/LiveKit/Audio/AudioEngineObserver.swift +++ b/Sources/LiveKit/Audio/AudioEngineObserver.swift @@ -18,7 +18,8 @@ import AVFAudio /// Do not retain the engine object. public protocol AudioEngineObserver: NextInvokable, Sendable { - func setNext(_ handler: any AudioEngineObserver) + associatedtype Next = any AudioEngineObserver + var next: (any AudioEngineObserver)? { get set } func engineDidCreate(_ engine: AVAudioEngine) func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) @@ -39,25 +40,35 @@ public protocol AudioEngineObserver: NextInvokable, Sendable { /// Default implementation to make it optional. public extension AudioEngineObserver { - func engineDidCreate(_: AVAudioEngine) {} - func engineWillEnable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} - func engineWillStart(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} - func engineDidStop(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} - func engineDidDisable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} - func engineWillRelease(_: AVAudioEngine) {} - - func engineWillConnectOutput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode?, format _: AVAudioFormat) -> Bool { false } - func engineWillConnectInput(_: AVAudioEngine, src _: AVAudioNode?, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } -} + func engineDidCreate(_ engine: AVAudioEngine) { + next?.engineDidCreate(engine) + } + + func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func engineWillStart(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + next?.engineWillStart(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } -extension [any AudioEngineObserver] { - func buildChain() -> Element? { - guard let first else { return nil } + func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + next?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } - for i in 0 ..< count - 1 { - self[i].setNext(self[i + 1]) - } + func engineWillRelease(_ engine: AVAudioEngine) { + next?.engineWillRelease(engine) + } + + func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode?, format: AVAudioFormat) -> Bool { + next?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false + } - return first + func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + next?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false } } diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift index 959ef429f..4e3d924f4 100644 --- a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -24,7 +24,7 @@ internal import LiveKitWebRTC @_implementationOnly import LiveKitWebRTC #endif -public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { +public class DefaultAudioSessionObserver: AudioEngineObserver, Loggable, @unchecked Sendable { struct State { var isSessionActive = false var next: (any AudioEngineObserver)? @@ -36,7 +36,12 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { let _state = StateSync(State()) - init() { + public var next: (any AudioEngineObserver)? { + get { _state.next } + set { _state.mutate { $0.next = newValue } } + } + + public init() { // Backward compatibility with `customConfigureAudioSessionFunc`. _state.onDidMutate = { new_, old_ in if let config_func = AudioManager.shared._state.customConfigureFunc, @@ -51,10 +56,6 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { } } - public func setNext(_ nextHandler: any AudioEngineObserver) { - _state.mutate { $0.next = nextHandler } - } - public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { if AudioManager.shared._state.customConfigureFunc == nil { log("Configuring audio session...") diff --git a/Sources/LiveKit/Protocols/NextInvokable.swift b/Sources/LiveKit/Protocols/NextInvokable.swift index 4dc516155..232e6480f 100644 --- a/Sources/LiveKit/Protocols/NextInvokable.swift +++ b/Sources/LiveKit/Protocols/NextInvokable.swift @@ -18,5 +18,5 @@ import Foundation public protocol NextInvokable { associatedtype Next - func setNext(_ handler: Next) + var next: Next? { get set } } diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index ad45ebf86..3d40db2df 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -341,3 +341,16 @@ public extension AudioManager { renderPreProcessingDelegateAdapter.remove(delegate: delegate) } } + +extension AudioManager { + func buildEngineObserverChain() -> (any AudioEngineObserver)? { + var objects = _state.engineObservers + guard !objects.isEmpty else { return nil } + + for i in 0 ..< objects.count - 1 { + objects[i].next = objects[i + 1] + } + + return objects.first + } +} diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index c481d304f..a2227902e 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -282,9 +282,10 @@ class AudioEngineTests: XCTestCase { } final class SineWaveNodeHook: AudioEngineObserver { + var next: (any LiveKit.AudioEngineObserver)? + let sineWaveNode = SineWaveSourceNode() - func setNext(_: any LiveKit.AudioEngineObserver) {} func engineDidCreate(_ engine: AVAudioEngine) { engine.attach(sineWaveNode) } @@ -301,6 +302,8 @@ final class SineWaveNodeHook: AudioEngineObserver { } final class PlayerNodeHook: AudioEngineObserver { + var next: (any LiveKit.AudioEngineObserver)? + public let playerNode = AVAudioPlayerNode() public let playerMixerNode = AVAudioMixerNode() public let playerNodeFormat: AVAudioFormat @@ -309,7 +312,6 @@ final class PlayerNodeHook: AudioEngineObserver { self.playerNodeFormat = playerNodeFormat } - func setNext(_: any LiveKit.AudioEngineObserver) {} public func engineDidCreate(_ engine: AVAudioEngine) { engine.attach(playerNode) engine.attach(playerMixerNode)