-
Notifications
You must be signed in to change notification settings - Fork 114
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
AVAudioEngine version AudioDeviceModule (#536)
Using lib from webrtc-sdk/webrtc#158. macOS will still use previous implementation at the moment. (Device selection etc not implemented yet) ### New features [Advanced other audio ducking](https://developer.apple.com/videos/play/wwdc2023/10235/?time=199) Lower audio from other apps dynamically when voice activity is detected. [Muted talker detection](https://developer.apple.com/videos/play/wwdc2023/10235/?time=475) Detect voice activity even mic is off after calling `AudioManager.shared.prepareRecording()`. Provide custom AVAudioNode configuration for input and output through `onEngineWillConnectInput` and `onEngineWillConnectOutput` handlers. Early recording initialization with `AudioManager.shared.prepareRecording()` without mic indicator turning on. Early mic input buffer capturing with `AudioManager.shared.startLocalRecording()` without the need for Room and connection. ### Improvements Correct mic muting logic without the requirement of restarting whole audio unit (previous implementation). Correct audio session configuration timing. AudioDeviceModule requests configuration when it's required, instead of counting tracks and configuring it in advance (previous implementation).
- Loading branch information
1 parent
81abc78
commit a489ddb
Showing
23 changed files
with
1,114 additions
and
205 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
88 changes: 88 additions & 0 deletions
88
Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| * Copyright 2025 LiveKit | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| import Foundation | ||
|
|
||
| #if swift(>=5.9) | ||
| internal import LiveKitWebRTC | ||
| #else | ||
| @_implementationOnly import LiveKitWebRTC | ||
| #endif | ||
|
|
||
| // Invoked on WebRTC's worker thread, do not block. | ||
| class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate { | ||
| weak var audioManager: AudioManager? | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, didReceiveSpeechActivityEvent speechActivityEvent: RTCSpeechActivityEvent) { | ||
| guard let audioManager else { return } | ||
| audioManager._state.onMutedSpeechActivity?(audioManager, speechActivityEvent.toLKType()) | ||
| } | ||
|
|
||
| func audioDeviceModuleDidUpdateDevices(_: LKRTCAudioDeviceModule) { | ||
| guard let audioManager else { return } | ||
| audioManager._state.onDevicesDidUpdate?(audioManager) | ||
| } | ||
|
|
||
| // Engine events | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) { | ||
| guard let audioManager else { return } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| entryPoint?.engineDidCreate(engine) | ||
| } | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { | ||
| guard let audioManager else { return } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) | ||
| } | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { | ||
| guard let audioManager else { return } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| entryPoint?.engineWillStart(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) | ||
| } | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { | ||
| guard let audioManager else { return } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| entryPoint?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) | ||
| } | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { | ||
| guard let audioManager else { return } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| entryPoint?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) | ||
| } | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine) { | ||
| guard let audioManager else { return } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| entryPoint?.engineWillRelease(engine) | ||
| } | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode?, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { | ||
| guard let audioManager else { return false } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false | ||
| } | ||
|
|
||
| func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode?, format: AVAudioFormat) -> Bool { | ||
| guard let audioManager else { return false } | ||
| let entryPoint = audioManager._state.engineObservers.buildChain() | ||
| return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| /* | ||
| * Copyright 2025 LiveKit | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| import AVFAudio | ||
|
|
||
| /// Do not retain the engine object. | ||
| public protocol AudioEngineObserver: NextInvokable, Sendable { | ||
| func setNext(_ handler: any AudioEngineObserver) | ||
|
|
||
| func engineDidCreate(_ engine: AVAudioEngine) | ||
| func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) | ||
| func engineWillStart(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) | ||
| func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) | ||
| func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) | ||
| func engineWillRelease(_ engine: AVAudioEngine) | ||
|
|
||
| /// Provide custom implementation for internal AVAudioEngine's output configuration. | ||
| /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. | ||
| /// Return true if custom implementation is provided, otherwise default implementation will be used. | ||
| func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode?, format: AVAudioFormat) -> Bool | ||
| /// Provide custom implementation for internal AVAudioEngine's input configuration. | ||
| /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. | ||
| /// Return true if custom implementation is provided, otherwise default implementation will be used. | ||
| func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool | ||
| } | ||
|
|
||
| /// Default implementation to make it optional. | ||
| public extension AudioEngineObserver { | ||
| func engineDidCreate(_: AVAudioEngine) {} | ||
| func engineWillEnable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} | ||
| func engineWillStart(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} | ||
| func engineDidStop(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} | ||
| func engineDidDisable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} | ||
| func engineWillRelease(_: AVAudioEngine) {} | ||
|
|
||
| func engineWillConnectOutput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode?, format _: AVAudioFormat) -> Bool { false } | ||
| func engineWillConnectInput(_: AVAudioEngine, src _: AVAudioNode?, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } | ||
| } | ||
|
|
||
| extension [any AudioEngineObserver] { | ||
| func buildChain() -> Element? { | ||
| guard let first else { return nil } | ||
|
|
||
| for i in 0 ..< count - 1 { | ||
| self[i].setNext(self[i + 1]) | ||
| } | ||
|
|
||
| return first | ||
| } | ||
| } |
127 changes: 127 additions & 0 deletions
127
Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| /* | ||
| * Copyright 2025 LiveKit | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #if os(iOS) || os(visionOS) || os(tvOS) | ||
|
|
||
| import AVFoundation | ||
|
|
||
| #if swift(>=5.9) | ||
| internal import LiveKitWebRTC | ||
| #else | ||
| @_implementationOnly import LiveKitWebRTC | ||
| #endif | ||
|
|
||
| public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { | ||
| struct State { | ||
| var isSessionActive = false | ||
| var next: (any AudioEngineObserver)? | ||
|
|
||
| // Used for backward compatibility with `customConfigureAudioSessionFunc`. | ||
| var isPlayoutEnabled: Bool = false | ||
| var isRecordingEnabled: Bool = false | ||
| } | ||
|
|
||
| let _state = StateSync(State()) | ||
|
|
||
| init() { | ||
| // Backward compatibility with `customConfigureAudioSessionFunc`. | ||
| _state.onDidMutate = { new_, old_ in | ||
| if let config_func = AudioManager.shared._state.customConfigureFunc, | ||
| new_.isPlayoutEnabled != old_.isPlayoutEnabled || | ||
| new_.isRecordingEnabled != old_.isRecordingEnabled | ||
| { | ||
| // Simulate state and invoke custom config func. | ||
| let old_state = AudioManager.State(localTracksCount: old_.isRecordingEnabled ? 1 : 0, remoteTracksCount: old_.isPlayoutEnabled ? 1 : 0) | ||
| let new_state = AudioManager.State(localTracksCount: new_.isRecordingEnabled ? 1 : 0, remoteTracksCount: new_.isPlayoutEnabled ? 1 : 0) | ||
| config_func(new_state, old_state) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public func setNext(_ nextHandler: any AudioEngineObserver) { | ||
| _state.mutate { $0.next = nextHandler } | ||
| } | ||
|
|
||
| public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { | ||
| if AudioManager.shared._state.customConfigureFunc == nil { | ||
| log("Configuring audio session...") | ||
| let session = LKRTCAudioSession.sharedInstance() | ||
| session.lockForConfiguration() | ||
| defer { session.unlockForConfiguration() } | ||
|
|
||
| let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback | ||
| do { | ||
| if _state.isSessionActive { | ||
| log("AudioSession deactivating due to category switch") | ||
| try session.setActive(false) // Deactivate first | ||
| _state.mutate { $0.isSessionActive = false } | ||
| } | ||
|
|
||
| log("AudioSession activating category to: \(config.category)") | ||
| try session.setConfiguration(config.toRTCType(), active: true) | ||
| _state.mutate { $0.isSessionActive = true } | ||
| } catch { | ||
| log("AudioSession failed to configure with error: \(error)", .error) | ||
| } | ||
|
|
||
| log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") | ||
| } | ||
|
|
||
| _state.mutate { | ||
| $0.isPlayoutEnabled = isPlayoutEnabled | ||
| $0.isRecordingEnabled = isRecordingEnabled | ||
| } | ||
|
|
||
| // Call next last | ||
| _state.next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) | ||
| } | ||
|
|
||
| public func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { | ||
| // Call next first | ||
| _state.next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) | ||
|
|
||
| _state.mutate { | ||
| $0.isPlayoutEnabled = isPlayoutEnabled | ||
| $0.isRecordingEnabled = isRecordingEnabled | ||
| } | ||
|
|
||
| if AudioManager.shared._state.customConfigureFunc == nil { | ||
| log("Configuring audio session...") | ||
| let session = LKRTCAudioSession.sharedInstance() | ||
| session.lockForConfiguration() | ||
| defer { session.unlockForConfiguration() } | ||
|
|
||
| do { | ||
| if isPlayoutEnabled, !isRecordingEnabled { | ||
| let config: AudioSessionConfiguration = .playback | ||
| log("AudioSession switching category to: \(config.category)") | ||
| try session.setConfiguration(config.toRTCType()) | ||
| } | ||
| if !isPlayoutEnabled, !isRecordingEnabled { | ||
| log("AudioSession deactivating") | ||
| try session.setActive(false) | ||
| _state.mutate { $0.isSessionActive = false } | ||
| } | ||
| } catch { | ||
| log("AudioSession failed to configure with error: \(error)", .error) | ||
| } | ||
|
|
||
| log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| /* | ||
| * Copyright 2025 LiveKit | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| import Foundation | ||
|
|
||
| public protocol NextInvokable { | ||
| associatedtype Next | ||
| func setNext(_ handler: Next) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.