From 622caa99b8628323613d3039e0d16a064d3fe7e1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:34:24 +0900 Subject: [PATCH 01/49] Tests --- Sources/LiveKit/Track/AudioManager.swift | 86 +++++++++++++---- Tests/LiveKitTests/AudioEngineTests.swift | 96 +++++++++++++++++++ .../Support/SinWaveSourceNode.swift | 58 +++++++++++ 3 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 Tests/LiveKitTests/AudioEngineTests.swift create mode 100644 Tests/LiveKitTests/Support/SinWaveSourceNode.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index de670b01f..b8a1c5499 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -68,6 +68,8 @@ public class AudioManager: Loggable { #endif public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void + public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void + public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ inputMixerNode: AVAudioMixerNode) -> Void #if os(iOS) || os(visionOS) || os(tvOS) @@ -208,13 +210,56 @@ public class AudioManager: Loggable { public var onDeviceUpdate: DeviceUpdateFunc? { didSet { - RTC.audioDeviceModule.setDevicesUpdatedHandler { [weak self] in + RTC.audioDeviceModule.setDevicesDidUpdateCallback { [weak self] in guard let self else { return } self.onDeviceUpdate?(self) } } } + public var onEngineWillConnectInput: OnEngineWillConnectInput? { + didSet { + RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, inputMixerNode in + guard let self else { return } + self.onEngineWillConnectInput?(self, engine, inputMixerNode) + } + } + } + + public var isManualRenderingMode: Bool { + get { RTC.audioDeviceModule.isManualRenderingMode } + set { + let result = RTC.audioDeviceModule.setManualRenderingMode(newValue) + if !result { + log("Failed to set manual rendering mode", .error) + } + } + } + + // MARK: Testing + + public func startPlayout() { + RTC.audioDeviceModule.initPlayout() + RTC.audioDeviceModule.startPlayout() + } + + public func stopPlayout() { + RTC.audioDeviceModule.stopPlayout() + } + + public func initRecording() { + RTC.audioDeviceModule.initRecording() + } + + public func startRecording() { + RTC.audioDeviceModule.initRecording() + RTC.audioDeviceModule.startRecording() + } + + public func stopRecording() { + RTC.audioDeviceModule.stopRecording() + } + // MARK: - Internal enum `Type` { @@ -224,19 +269,34 @@ public class AudioManager: Loggable { let state = StateSync(State()) - // MARK: - Private + init() { + RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + guard let self else { return } + self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - private let _configureRunner = SerialRunnerActor() + #if os(iOS) || os(visionOS) || os(tvOS) + self.log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + let config = LKRTCAudioSessionConfiguration.webRTC() + + if isRecordingEnabled { + config.category = AVAudioSession.Category.playAndRecord.rawValue + config.mode = AVAudioSession.Mode.videoChat.rawValue + config.categoryOptions = [.defaultToSpeaker, .allowBluetooth] + } else { + config.category = AVAudioSession.Category.playback.rawValue + config.mode = AVAudioSession.Mode.spokenAudio.rawValue + config.categoryOptions = [.mixWithOthers] + } - #if os(iOS) || os(visionOS) || os(tvOS) - private func _asyncConfigure(newState: State, oldState: State) async throws { - try await _configureRunner.run { - self.log("\(oldState) -> \(newState)") - let configureFunc = newState.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc - configureFunc(newState, oldState) + session.lockForConfiguration() + try? session.setConfiguration(config) + session.unlockForConfiguration() + #endif } } - #endif + + // MARK: - Private func trackDidStart(_ type: Type) async throws { let (newState, oldState) = state.mutate { state in @@ -245,9 +305,6 @@ public class AudioManager: Loggable { if type == .remote { state.remoteTracksCount += 1 } return (state, oldState) } - #if os(iOS) || os(visionOS) || os(tvOS) - try await _asyncConfigure(newState: newState, oldState: oldState) - #endif } func trackDidStop(_ type: Type) async throws { @@ -257,9 +314,6 @@ public class AudioManager: Loggable { if type == .remote { state.remoteTracksCount = max(state.remoteTracksCount - 1, 0) } return (state, oldState) } - #if os(iOS) || os(visionOS) || os(tvOS) - try await _asyncConfigure(newState: newState, oldState: oldState) - #endif } #if os(iOS) || os(visionOS) || os(tvOS) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift new file mode 100644 index 000000000..2005df332 --- /dev/null +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -0,0 +1,96 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFoundation +@testable import LiveKit +import LiveKitWebRTC +import XCTest + +class AudioEngineTests: XCTestCase { + override class func setUp() { + LiveKitSDK.setLoggerStandardOutput() + RTCSetMinDebugLogLevel(.info) + } + + override func tearDown() async throws {} + + // Test if mic is authorized. Only works on device. + func testMicAuthorized() async { + let status = AVCaptureDevice.authorizationStatus(for: .audio) + if status != .authorized { + let result = await AVCaptureDevice.requestAccess(for: .audio) + XCTAssert(result) + } + + XCTAssert(status == .authorized) + } + + // Test start generating local audio buffer without joining to room. + func testPrejoinLocalAudioBuffer() async throws { + // Set up expectation... + let didReceiveAudioFrame = expectation(description: "Did receive audio frame") + didReceiveAudioFrame.assertForOverFulfill = false + + // Start watching for audio frame... + let audioFrameWatcher = AudioTrackWatcher(id: "notifier01") { _ in + didReceiveAudioFrame.fulfill() + } + + let localMicTrack = LocalAudioTrack.createTrack() + // Attach audio frame watcher... + localMicTrack.add(audioRenderer: audioFrameWatcher) + + Task.detached { + print("Starting audio track in 3 seconds...") + try? await Task.sleep(for: .seconds(3)) + AudioManager.shared.startRecording() + } + + // Wait for audio frame... + print("Waiting for first audio frame...") + await fulfillment(of: [didReceiveAudioFrame], timeout: 10) + + // Remove audio frame watcher... + localMicTrack.remove(audioRenderer: audioFrameWatcher) + } + + // Test the manual rendering mode (no-device mode) of AVAudioEngine based AudioDeviceModule. + // In manual rendering, no device access will be initialized such as mic and speaker. + func testManualRenderingMode() async throws { + // Attach sin wave generator when engine requests input node... + // inputMixerNode will automatically convert to RTC's internal format (int16). + // AVAudioEngine.attach() retains the node. + AudioManager.shared.onEngineWillConnectInput = { _, engine, inputMixerNode in + let sin = SineWaveSourceNode() + engine.attach(sin) + engine.connect(sin, to: inputMixerNode, format: nil) + } + + // Set manual rendering mode... + AudioManager.shared.isManualRenderingMode = true + + // Check if manual rendering mode is set... + let isManualRenderingMode = AudioManager.shared.isManualRenderingMode + print("manualRenderingMode: \(isManualRenderingMode)") + XCTAssert(isManualRenderingMode) + + // Start rendering... + AudioManager.shared.startRecording() + + // Render for 10 seconds... + try? await Task.sleep(for: .seconds(10)) + } +} diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift new file mode 100644 index 000000000..b1fc632f7 --- /dev/null +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -0,0 +1,58 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +class SineWaveSourceNode: AVAudioSourceNode { + private let sampleRate: Double + private let frequency: Double + + init(frequency: Double = 440.0, sampleRate: Double = 48000.0) { + self.frequency = frequency + self.sampleRate = sampleRate + + let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1)! + + var currentPhase = 0.0 + let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate + + let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in + print("SineWaveSourceNode render block, frameCount: \(frameCount)") + + let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) + guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { + return kAudioUnitErr_InvalidParameter + } + + // Generate sine wave samples + for frame in 0 ..< Int(frameCount) { + ptr[frame] = Float(sin(currentPhase)) + + // Update the phase + currentPhase += phaseIncrement + + // Keep phase in [0, 2π] to prevent floating point errors + if currentPhase >= 2.0 * Double.pi { + currentPhase -= 2.0 * Double.pi + } + } + + return noErr + } + + super.init(format: format, renderBlock: renderBlock) + } +} From 9908d8eb94e3be6d90621230475536fea318643b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 7 Jan 2025 01:10:37 +0900 Subject: [PATCH 02/49] Update render test --- Tests/LiveKitTests/AudioEngineTests.swift | 22 +++++- .../LiveKitTests/Support/AudioRecorder.swift | 68 +++++++++++++++++++ .../Support/SinWaveSourceNode.swift | 10 +-- 3 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 Tests/LiveKitTests/Support/AudioRecorder.swift diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 2005df332..c9fde51ba 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -87,10 +87,26 @@ class AudioEngineTests: XCTestCase { print("manualRenderingMode: \(isManualRenderingMode)") XCTAssert(isManualRenderingMode) - // Start rendering... + let recorder = try AudioRecorder() + + let track = LocalAudioTrack.createTrack() + track.add(audioRenderer: recorder) + + // Start engine... AudioManager.shared.startRecording() - // Render for 10 seconds... - try? await Task.sleep(for: .seconds(10)) + // Render for 5 seconds... + try? await Task.sleep(for: .seconds(5)) + + recorder.close() + print("Written to: \(recorder.filePath)") + + // Stop engine + AudioManager.shared.stopRecording() + + // Play the recorded file... + let player = try AVAudioPlayer(contentsOf: recorder.filePath) + player.play() + try? await Task.sleep(for: .seconds(5)) } } diff --git a/Tests/LiveKitTests/Support/AudioRecorder.swift b/Tests/LiveKitTests/Support/AudioRecorder.swift new file mode 100644 index 000000000..641af9c25 --- /dev/null +++ b/Tests/LiveKitTests/Support/AudioRecorder.swift @@ -0,0 +1,68 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio +@testable import LiveKit + +// Used to save audio data for inspecting the correct format, etc. +class AudioRecorder { + public let sampleRate: Double + public let audioFile: AVAudioFile + public let filePath: URL + + init(sampleRate: Double = 16000, channels: Int = 1) throws { + self.sampleRate = sampleRate + + let settings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: sampleRate, + AVNumberOfChannelsKey: channels, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsNonInterleaved: false, + AVLinearPCMIsBigEndianKey: false, + ] + + let fileName = UUID().uuidString + ".wav" + let filePath = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(fileName) + self.filePath = filePath + + audioFile = try AVAudioFile(forWriting: filePath, + settings: settings, + commonFormat: .pcmFormatInt16, + interleaved: true) + } + + func write(pcmBuffer: AVAudioPCMBuffer) throws { + if #available(macOS 15.0, *) { + guard audioFile.isOpen else { return } + } + + try audioFile.write(from: pcmBuffer) + } + + func close() { + if #available(macOS 15.0, *) { + audioFile.close() + } + } +} + +extension AudioRecorder: AudioRenderer { + func render(pcmBuffer: AVAudioPCMBuffer) { + try? write(pcmBuffer: pcmBuffer) + } +} diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index b1fc632f7..1ab1ee53c 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -20,7 +20,7 @@ class SineWaveSourceNode: AVAudioSourceNode { private let sampleRate: Double private let frequency: Double - init(frequency: Double = 440.0, sampleRate: Double = 48000.0) { + init(frequency: Double = 400.0, sampleRate: Double = 48000.0) { self.frequency = frequency self.sampleRate = sampleRate @@ -30,8 +30,6 @@ class SineWaveSourceNode: AVAudioSourceNode { let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in - print("SineWaveSourceNode render block, frameCount: \(frameCount)") - let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { return kAudioUnitErr_InvalidParameter @@ -44,10 +42,8 @@ class SineWaveSourceNode: AVAudioSourceNode { // Update the phase currentPhase += phaseIncrement - // Keep phase in [0, 2π] to prevent floating point errors - if currentPhase >= 2.0 * Double.pi { - currentPhase -= 2.0 * Double.pi - } + // Keep phase within [0, 2π] range using fmod for stability + currentPhase = fmod(currentPhase, 2.0 * Double.pi) } return noErr From 3b9bbb3fc54ad350393a7913bffbb77ded678896 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:23:01 +0900 Subject: [PATCH 03/49] Backward compatible session config --- Sources/LiveKit/Track/AudioManager.swift | 54 ++++++++++--------- Tests/LiveKitTests/AudioEngineTests.swift | 20 ++++--- .../LiveKitTests/Support/AudioRecorder.swift | 4 +- .../Support/SinWaveSourceNode.swift | 1 + 4 files changed, 46 insertions(+), 33 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index b8a1c5499..c29e271f7 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -236,27 +236,43 @@ public class AudioManager: Loggable { } } - // MARK: Testing + // MARK: - Recording - public func startPlayout() { + /// Initialize recording (mic input) and pre-warm voice processing etc. + /// Mic permission is required and dialog will appear if not already granted. + public func prepareRecording() { + RTC.audioDeviceModule.initRecording() + } + + /// Starts mic input to the SDK even without any ``Room`` or a connection. + /// Audio buffers will flow into ``LocalAudioTrack/add(audioRenderer:)`` and ``capturePostProcessingDelegate``. + public func startLocalRecording() { + RTC.audioDeviceModule.initAndStartRecording() + } + + // MARK: Internal for testing + + func initPlayout() { RTC.audioDeviceModule.initPlayout() + } + + func startPlayout() { RTC.audioDeviceModule.startPlayout() } - public func stopPlayout() { + func stopPlayout() { RTC.audioDeviceModule.stopPlayout() } - public func initRecording() { + func initRecording() { RTC.audioDeviceModule.initRecording() } - public func startRecording() { - RTC.audioDeviceModule.initRecording() + func startRecording() { RTC.audioDeviceModule.startRecording() } - public func stopRecording() { + func stopRecording() { RTC.audioDeviceModule.stopRecording() } @@ -276,22 +292,10 @@ public class AudioManager: Loggable { #if os(iOS) || os(visionOS) || os(tvOS) self.log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - let config = LKRTCAudioSessionConfiguration.webRTC() - - if isRecordingEnabled { - config.category = AVAudioSession.Category.playAndRecord.rawValue - config.mode = AVAudioSession.Mode.videoChat.rawValue - config.categoryOptions = [.defaultToSpeaker, .allowBluetooth] - } else { - config.category = AVAudioSession.Category.playback.rawValue - config.mode = AVAudioSession.Mode.spokenAudio.rawValue - config.categoryOptions = [.mixWithOthers] - } - - session.lockForConfiguration() - try? session.setConfiguration(config) - session.unlockForConfiguration() + // Backward compatibility + let configureFunc = state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc + let simulatedState = AudioManager.State(localTracksCount: isRecordingEnabled ? 1 : 0, remoteTracksCount: isPlayoutEnabled ? 1 : 0) + configureFunc(simulatedState, AudioManager.State()) #endif } } @@ -299,7 +303,7 @@ public class AudioManager: Loggable { // MARK: - Private func trackDidStart(_ type: Type) async throws { - let (newState, oldState) = state.mutate { state in + state.mutate { state in let oldState = state if type == .local { state.localTracksCount += 1 } if type == .remote { state.remoteTracksCount += 1 } @@ -308,7 +312,7 @@ public class AudioManager: Loggable { } func trackDidStop(_ type: Type) async throws { - let (newState, oldState) = state.mutate { state in + state.mutate { state in let oldState = state if type == .local { state.localTracksCount = max(state.localTracksCount - 1, 0) } if type == .remote { state.remoteTracksCount = max(state.remoteTracksCount - 1, 0) } diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index c9fde51ba..5b869c020 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -38,6 +38,14 @@ class AudioEngineTests: XCTestCase { XCTAssert(status == .authorized) } + // Test if state transitions pass internal checks. + func testStates() async { + let adm = AudioManager.shared + adm.initPlayout() + adm.startPlayout() + adm.stopPlayout() + } + // Test start generating local audio buffer without joining to room. func testPrejoinLocalAudioBuffer() async throws { // Set up expectation... @@ -55,8 +63,8 @@ class AudioEngineTests: XCTestCase { Task.detached { print("Starting audio track in 3 seconds...") - try? await Task.sleep(for: .seconds(3)) - AudioManager.shared.startRecording() + try? await Task.sleep(nanoseconds: 3 * 1_000_000_000) + AudioManager.shared.startLocalRecording() } // Wait for audio frame... @@ -93,20 +101,20 @@ class AudioEngineTests: XCTestCase { track.add(audioRenderer: recorder) // Start engine... - AudioManager.shared.startRecording() + AudioManager.shared.startLocalRecording() // Render for 5 seconds... - try? await Task.sleep(for: .seconds(5)) + try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) recorder.close() print("Written to: \(recorder.filePath)") - + // Stop engine AudioManager.shared.stopRecording() // Play the recorded file... let player = try AVAudioPlayer(contentsOf: recorder.filePath) player.play() - try? await Task.sleep(for: .seconds(5)) + try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) } } diff --git a/Tests/LiveKitTests/Support/AudioRecorder.swift b/Tests/LiveKitTests/Support/AudioRecorder.swift index 641af9c25..de79a2038 100644 --- a/Tests/LiveKitTests/Support/AudioRecorder.swift +++ b/Tests/LiveKitTests/Support/AudioRecorder.swift @@ -47,7 +47,7 @@ class AudioRecorder { } func write(pcmBuffer: AVAudioPCMBuffer) throws { - if #available(macOS 15.0, *) { + if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { guard audioFile.isOpen else { return } } @@ -55,7 +55,7 @@ class AudioRecorder { } func close() { - if #available(macOS 15.0, *) { + if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { audioFile.close() } } diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index 1ab1ee53c..fa031d9bf 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -30,6 +30,7 @@ class SineWaveSourceNode: AVAudioSourceNode { let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in + print("AVAudioSourceNodeRenderBlock frameCount: \(frameCount)") let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { return kAudioUnitErr_InvalidParameter From b1f3ae1d3ab1b8588c9a63eafabfdc32a26ca976 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:38:07 +0900 Subject: [PATCH 04/49] .mixWithOthers by default --- Sources/LiveKit/Types/AudioSessionConfiguration.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/LiveKit/Types/AudioSessionConfiguration.swift b/Sources/LiveKit/Types/AudioSessionConfiguration.swift index 171da686b..89f855ea7 100644 --- a/Sources/LiveKit/Types/AudioSessionConfiguration.swift +++ b/Sources/LiveKit/Types/AudioSessionConfiguration.swift @@ -37,11 +37,11 @@ public extension AudioSessionConfiguration { mode: .spokenAudio) static let playAndRecordSpeaker = AudioSessionConfiguration(category: .playAndRecord, - categoryOptions: [.allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], + categoryOptions: [.mixWithOthers, .allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], mode: .videoChat) static let playAndRecordReceiver = AudioSessionConfiguration(category: .playAndRecord, - categoryOptions: [.allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], + categoryOptions: [.mixWithOthers, .allowBluetooth, .allowBluetoothA2DP, .allowAirPlay], mode: .voiceChat) } From b1871daf33fd5be57505e32a3028aff1ab503b72 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:36:32 +0900 Subject: [PATCH 05/49] Ducking config --- Sources/LiveKit/Track/AudioManager.swift | 11 +++++++++++ Tests/LiveKitTests/AudioEngineTests.swift | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index c29e271f7..eff4f08c0 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -236,6 +236,17 @@ public class AudioManager: Loggable { } } + public var isAdvancedDuckingEnabled: Bool { + get { RTC.audioDeviceModule.isAdvancedDuckingEnabled } + set { RTC.audioDeviceModule.isAdvancedDuckingEnabled = newValue } + } + + @available(iOS 17, macOS 14.0, visionOS 1.0, *) + public var duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingConfiguration.Level { + get { RTC.audioDeviceModule.duckingLevel } + set { RTC.audioDeviceModule.duckingLevel = newValue } + } + // MARK: - Recording /// Initialize recording (mic input) and pre-warm voice processing etc. diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 5b869c020..dee6ff8d4 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -46,6 +46,28 @@ class AudioEngineTests: XCTestCase { adm.stopPlayout() } + func testConfigureDucking() async { + AudioManager.shared.isAdvancedDuckingEnabled = false + XCTAssert(!AudioManager.shared.isAdvancedDuckingEnabled) + + AudioManager.shared.isAdvancedDuckingEnabled = true + XCTAssert(AudioManager.shared.isAdvancedDuckingEnabled) + + if #available(iOS 17, macOS 14.0, visionOS 1.0, *) { + AudioManager.shared.duckingLevel = .default + XCTAssert(AudioManager.shared.duckingLevel == .default) + + AudioManager.shared.duckingLevel = .min + XCTAssert(AudioManager.shared.duckingLevel == .min) + + AudioManager.shared.duckingLevel = .max + XCTAssert(AudioManager.shared.duckingLevel == .max) + + AudioManager.shared.duckingLevel = .mid + XCTAssert(AudioManager.shared.duckingLevel == .mid) + } + } + // Test start generating local audio buffer without joining to room. func testPrejoinLocalAudioBuffer() async throws { // Set up expectation... From be593c138623d38a525685d541349c6976ff4cdf Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 8 Jan 2025 15:33:51 +0900 Subject: [PATCH 06/49] Use 125.6422.12-exp.2 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index bf7183a9a..c7547e651 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.11"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 37e1c664d..82d48b771 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.11"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From 89c084c443dcd5b0bd859f61454d4b771eacc7cc Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 9 Jan 2025 06:03:17 +0900 Subject: [PATCH 07/49] Muted speech activity --- Sources/LiveKit/Track/AudioManager.swift | 12 +++++++ .../LiveKit/Types/SpeechActivityEvent.swift | 36 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 Sources/LiveKit/Types/SpeechActivityEvent.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index eff4f08c0..ca050aa76 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -71,6 +71,8 @@ public class AudioManager: Loggable { public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ inputMixerNode: AVAudioMixerNode) -> Void + public typealias OnSpeechActivityEvent = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void + #if os(iOS) || os(visionOS) || os(tvOS) public typealias ConfigureAudioSessionFunc = @Sendable (_ newState: State, @@ -226,6 +228,16 @@ public class AudioManager: Loggable { } } + // Invoked on internal thread, do not block. + public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? { + didSet { + RTC.audioDeviceModule.setSpeechActivityCallback { [weak self] event in + guard let self else { return } + self.onMutedSpeechActivityEvent?(self, event.toLKType()) + } + } + } + public var isManualRenderingMode: Bool { get { RTC.audioDeviceModule.isManualRenderingMode } set { diff --git a/Sources/LiveKit/Types/SpeechActivityEvent.swift b/Sources/LiveKit/Types/SpeechActivityEvent.swift new file mode 100644 index 000000000..98d59917f --- /dev/null +++ b/Sources/LiveKit/Types/SpeechActivityEvent.swift @@ -0,0 +1,36 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +public enum SpeechActivityEvent { + case started + case ended +} + +extension RTCSpeechActivityEvent { + func toLKType() -> SpeechActivityEvent { + switch self { + case .started: return .started + case .ended: return .ended + @unknown default: return .ended + } + } +} From 282cbc786046be18aafe042e57d930521b3d648f Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:53:36 +0900 Subject: [PATCH 08/49] Update node config methods --- Sources/LiveKit/Track/AudioManager.swift | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index ca050aa76..4b314db23 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -69,7 +69,16 @@ public class AudioManager: Loggable { public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void - public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ inputMixerNode: AVAudioMixerNode) -> Void + public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, + _ engine: AVAudioEngine, + _ src: AVAudioNode, + _ dst: AVAudioNode, + _ format: AVAudioFormat) -> Bool + public typealias OnEngineWillConnectOutput = (_ audioManager: AudioManager, + _ engine: AVAudioEngine, + _ src: AVAudioNode, + _ dst: AVAudioNode, + _ format: AVAudioFormat) -> Bool public typealias OnSpeechActivityEvent = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void @@ -221,9 +230,18 @@ public class AudioManager: Loggable { public var onEngineWillConnectInput: OnEngineWillConnectInput? { didSet { - RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, inputMixerNode in - guard let self else { return } - self.onEngineWillConnectInput?(self, engine, inputMixerNode) + RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, src, dst, format in + guard let self else { return false } + return self.onEngineWillConnectInput?(self, engine, src, dst, format) ?? false + } + } + } + + public var onEngineWillConnectOutput: OnEngineWillConnectOutput? { + didSet { + RTC.audioDeviceModule.setOnEngineWillConnectOutputCallback { [weak self] engine, src, dst, format in + guard let self else { return false } + return self.onEngineWillConnectOutput?(self, engine, src, dst, format) ?? false } } } From 8f705401f4b9d099b0bd2c26e8d0bb82e7aac943 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 01:19:33 +0900 Subject: [PATCH 09/49] Move audio buffer --- Sources/LiveKit/Track/AudioManager.swift | 33 -------------- Sources/LiveKit/Types/AudioBuffer.swift | 56 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 33 deletions(-) create mode 100644 Sources/LiveKit/Types/AudioBuffer.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 4b314db23..3cddde7df 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -24,39 +24,6 @@ internal import LiveKitWebRTC @_implementationOnly import LiveKitWebRTC #endif -// Wrapper for LKRTCAudioBuffer -@objc -public class LKAudioBuffer: NSObject { - private let _audioBuffer: LKRTCAudioBuffer - - @objc - public var channels: Int { _audioBuffer.channels } - - @objc - public var frames: Int { _audioBuffer.frames } - - @objc - public var framesPerBand: Int { _audioBuffer.framesPerBand } - - @objc - public var bands: Int { _audioBuffer.bands } - - @objc - @available(*, deprecated, renamed: "rawBuffer(forChannel:)") - public func rawBuffer(for channel: Int) -> UnsafeMutablePointer { - _audioBuffer.rawBuffer(forChannel: channel) - } - - @objc - public func rawBuffer(forChannel channel: Int) -> UnsafeMutablePointer { - _audioBuffer.rawBuffer(forChannel: channel) - } - - init(audioBuffer: LKRTCAudioBuffer) { - _audioBuffer = audioBuffer - } -} - // Audio Session Configuration related public class AudioManager: Loggable { // MARK: - Public diff --git a/Sources/LiveKit/Types/AudioBuffer.swift b/Sources/LiveKit/Types/AudioBuffer.swift new file mode 100644 index 000000000..63b39fd2f --- /dev/null +++ b/Sources/LiveKit/Types/AudioBuffer.swift @@ -0,0 +1,56 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +import Foundation + +// Wrapper for LKRTCAudioBuffer +@objc +public class LKAudioBuffer: NSObject { + private let _audioBuffer: LKRTCAudioBuffer + + @objc + public var channels: Int { _audioBuffer.channels } + + @objc + public var frames: Int { _audioBuffer.frames } + + @objc + public var framesPerBand: Int { _audioBuffer.framesPerBand } + + @objc + public var bands: Int { _audioBuffer.bands } + + @objc + @available(*, deprecated, renamed: "rawBuffer(forChannel:)") + public func rawBuffer(for channel: Int) -> UnsafeMutablePointer { + _audioBuffer.rawBuffer(forChannel: channel) + } + + @objc + public func rawBuffer(forChannel channel: Int) -> UnsafeMutablePointer { + _audioBuffer.rawBuffer(forChannel: channel) + } + + init(audioBuffer: LKRTCAudioBuffer) { + _audioBuffer = audioBuffer + } +} From 92e34065f12951ddbc7166c6ca9d61b544064e93 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 01:33:00 +0900 Subject: [PATCH 10/49] Update AudioManager.swift Docs --- Sources/LiveKit/Track/AudioManager.swift | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 3cddde7df..6a59b545f 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -195,6 +195,9 @@ public class AudioManager: Loggable { } } + /// Provide custom implementation for internal AVAudioEngine's input configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. public var onEngineWillConnectInput: OnEngineWillConnectInput? { didSet { RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, src, dst, format in @@ -204,6 +207,9 @@ public class AudioManager: Loggable { } } + /// Provide custom implementation for internal AVAudioEngine's output configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. public var onEngineWillConnectOutput: OnEngineWillConnectOutput? { didSet { RTC.audioDeviceModule.setOnEngineWillConnectOutputCallback { [weak self] engine, src, dst, format in @@ -213,7 +219,9 @@ public class AudioManager: Loggable { } } - // Invoked on internal thread, do not block. + /// Detect voice activity even if the mic is muted. + /// Internal audio engine must be initialized by calling ``prepareRecording()`` or + /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? { didSet { RTC.audioDeviceModule.setSpeechActivityCallback { [weak self] event in @@ -233,11 +241,14 @@ public class AudioManager: Loggable { } } + /// Enables advanced ducking which ducks other audio based on the presence of voice activity from local and remote chat participants. + /// Default: true. public var isAdvancedDuckingEnabled: Bool { get { RTC.audioDeviceModule.isAdvancedDuckingEnabled } set { RTC.audioDeviceModule.isAdvancedDuckingEnabled = newValue } } + /// The ducking(audio reducing) level of other audio. @available(iOS 17, macOS 14.0, visionOS 1.0, *) public var duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingConfiguration.Level { get { RTC.audioDeviceModule.duckingLevel } From 4b77f849a90a642faf77b2995cfdc1f0eb915d24 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:46:37 +0900 Subject: [PATCH 11/49] Use 125.6422.12-exp.3 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index c7547e651..31977d747 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 82d48b771..0b652d4df 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.2"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From 7cd4f29ae43e1863c1b185d2190eb8c6b03724c6 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:01:24 +0900 Subject: [PATCH 12/49] Fix tests --- Tests/LiveKitTests/AudioEngineTests.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index dee6ff8d4..5a33bd365 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -103,10 +103,11 @@ class AudioEngineTests: XCTestCase { // Attach sin wave generator when engine requests input node... // inputMixerNode will automatically convert to RTC's internal format (int16). // AVAudioEngine.attach() retains the node. - AudioManager.shared.onEngineWillConnectInput = { _, engine, inputMixerNode in + AudioManager.shared.onEngineWillConnectInput = { _, engine, _, dst, format in let sin = SineWaveSourceNode() engine.attach(sin) - engine.connect(sin, to: inputMixerNode, format: nil) + engine.connect(sin, to: dst, format: format) + return true } // Set manual rendering mode... From 130e1d219f75099eac3177981b0bdb8a9a87c99c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 12:15:20 +0900 Subject: [PATCH 13/49] Fix tests --- Tests/LiveKitTests/Support/AudioRecorder.swift | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Tests/LiveKitTests/Support/AudioRecorder.swift b/Tests/LiveKitTests/Support/AudioRecorder.swift index de79a2038..dc0cffe99 100644 --- a/Tests/LiveKitTests/Support/AudioRecorder.swift +++ b/Tests/LiveKitTests/Support/AudioRecorder.swift @@ -20,8 +20,8 @@ import AVFAudio // Used to save audio data for inspecting the correct format, etc. class AudioRecorder { public let sampleRate: Double - public let audioFile: AVAudioFile public let filePath: URL + private var audioFile: AVAudioFile? init(sampleRate: Double = 16000, channels: Int = 1) throws { self.sampleRate = sampleRate @@ -47,17 +47,12 @@ class AudioRecorder { } func write(pcmBuffer: AVAudioPCMBuffer) throws { - if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { - guard audioFile.isOpen else { return } - } - + guard let audioFile else { return } try audioFile.write(from: pcmBuffer) } func close() { - if #available(iOS 18, macOS 15.0, tvOS 18, visionOS 2.0, *) { - audioFile.close() - } + audioFile = nil } } From 874b3a4d37c20850d7885b2f6c0f67e179e2d7f0 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:51:29 +0900 Subject: [PATCH 14/49] AudioDuckingLevel type --- Sources/LiveKit/Track/AudioManager.swift | 6 ++--- Sources/LiveKit/Types/AudioDuckingLevel.swift | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 Sources/LiveKit/Types/AudioDuckingLevel.swift diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 6a59b545f..e4232afc6 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -250,9 +250,9 @@ public class AudioManager: Loggable { /// The ducking(audio reducing) level of other audio. @available(iOS 17, macOS 14.0, visionOS 1.0, *) - public var duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingConfiguration.Level { - get { RTC.audioDeviceModule.duckingLevel } - set { RTC.audioDeviceModule.duckingLevel = newValue } + public var duckingLevel: AudioDuckingLevel { + get { AudioDuckingLevel(rawValue: RTC.audioDeviceModule.duckingLevel) ?? .default } + set { RTC.audioDeviceModule.duckingLevel = newValue.rawValue } } // MARK: - Recording diff --git a/Sources/LiveKit/Types/AudioDuckingLevel.swift b/Sources/LiveKit/Types/AudioDuckingLevel.swift new file mode 100644 index 000000000..6caa49673 --- /dev/null +++ b/Sources/LiveKit/Types/AudioDuckingLevel.swift @@ -0,0 +1,22 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public enum AudioDuckingLevel: Int { + case `default` = 0 + case min = 10 + case mid = 20 + case max = 30 +} From 49c91ef8634ae401fe792f0147242c376329ac5e Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:34:51 +0900 Subject: [PATCH 15/49] Use 125.6422.12-exp.4 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 31977d747..753c73108 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 0b652d4df..2e8dddf83 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.3"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From 4b84621d2ff4f41b63badf2fe17201eb575dfe64 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:53:31 +0900 Subject: [PATCH 16/49] Fix Xcode 14.2 --- Sources/LiveKit/Track/AudioManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index e4232afc6..9397b1549 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -312,7 +312,7 @@ public class AudioManager: Loggable { #if os(iOS) || os(visionOS) || os(tvOS) self.log("Configuring audio session...") // Backward compatibility - let configureFunc = state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc + let configureFunc = self.state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc let simulatedState = AudioManager.State(localTracksCount: isRecordingEnabled ? 1 : 0, remoteTracksCount: isPlayoutEnabled ? 1 : 0) configureFunc(simulatedState, AudioManager.State()) #endif From 5a585a3b6203ae70975802cf421076f78852c9f6 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:14:26 +0900 Subject: [PATCH 17/49] Change session config timing --- Sources/LiveKit/Track/AudioManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 9397b1549..6a84fc5b4 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -305,7 +305,7 @@ public class AudioManager: Loggable { let state = StateSync(State()) init() { - RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + RTC.audioDeviceModule.setOnEngineWillEnableCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in guard let self else { return } self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") From a0103ad349ed465ed55051ea8cc32bb4f589b05c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:07:48 +0900 Subject: [PATCH 18/49] Update state tests --- Sources/LiveKit/Track/AudioManager.swift | 18 +++++++++++++++++- Tests/LiveKitTests/AudioEngineTests.swift | 19 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 6a84fc5b4..e2fbdcb63 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -269,7 +269,23 @@ public class AudioManager: Loggable { RTC.audioDeviceModule.initAndStartRecording() } - // MARK: Internal for testing + // MARK: - For testing + + var isPlayoutInitialized: Bool { + RTC.audioDeviceModule.isPlayoutInitialized + } + + var isPlaying: Bool { + RTC.audioDeviceModule.isPlaying + } + + var isRecordingInitialized: Bool { + RTC.audioDeviceModule.isRecordingInitialized + } + + var isRecording: Bool { + RTC.audioDeviceModule.isRecording + } func initPlayout() { RTC.audioDeviceModule.initPlayout() diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 5a33bd365..33e558baf 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -39,11 +39,28 @@ class AudioEngineTests: XCTestCase { } // Test if state transitions pass internal checks. - func testStates() async { + func testStateTransitions() async { let adm = AudioManager.shared + // Start Playout adm.initPlayout() + XCTAssert(adm.isPlayoutInitialized) adm.startPlayout() + XCTAssert(adm.isPlaying) + + // Start Recording + adm.initRecording() + XCTAssert(adm.isRecordingInitialized) + adm.startRecording() + XCTAssert(adm.isRecording) + + // Stop engine + adm.stopRecording() + XCTAssert(!adm.isRecording) + XCTAssert(!adm.isRecordingInitialized) + adm.stopPlayout() + XCTAssert(!adm.isPlaying) + XCTAssert(!adm.isPlayoutInitialized) } func testConfigureDucking() async { From 256b42aa1f0095b19752b619fd8079c945700fff Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Mon, 20 Jan 2025 23:56:01 +0900 Subject: [PATCH 19/49] P1 --- .../Audio/AudioManager+EngineObserver.swift | 45 ++++++++++++ Sources/LiveKit/Track/AudioManager.swift | 71 ++++++++++++++++--- Tests/LiveKitTests/AudioEngineTests.swift | 18 +++++ 3 files changed, 126 insertions(+), 8 deletions(-) create mode 100644 Sources/LiveKit/Audio/AudioManager+EngineObserver.swift diff --git a/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift b/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift new file mode 100644 index 000000000..662e30f0f --- /dev/null +++ b/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift @@ -0,0 +1,45 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +/// Do not retain the engine object. +@objc +public protocol AudioEngineObserver: Chainable { + @objc optional + func engineDidCreate(_ engine: AVAudioEngine) + + @objc optional + func engineWillEnable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineWillStart(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineDidStop(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineDidDisable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) + + @objc optional + func engineWillRelease(_ engine: AVAudioEngine) + + @objc optional + func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) + + @objc optional + func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) +} diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index e2fbdcb63..ea4c1ef06 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -35,6 +35,8 @@ public class AudioManager: Loggable { #endif public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void + + // Engine events public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, _ engine: AVAudioEngine, @@ -257,10 +259,12 @@ public class AudioManager: Loggable { // MARK: - Recording - /// Initialize recording (mic input) and pre-warm voice processing etc. + /// Keep recording initialized (mic input) and pre-warm voice processing etc. /// Mic permission is required and dialog will appear if not already granted. - public func prepareRecording() { - RTC.audioDeviceModule.initRecording() + /// This will per persisted accross Rooms and connections. + public var isRecordingAlwaysPrepared: Bool { + get { RTC.audioDeviceModule.isInitRecordingPersistentMode } + set { RTC.audioDeviceModule.isInitRecordingPersistentMode = newValue } } /// Starts mic input to the SDK even without any ``Room`` or a connection. @@ -320,19 +324,70 @@ public class AudioManager: Loggable { let state = StateSync(State()) + var isSessionActive: Bool = false + init() { RTC.audioDeviceModule.setOnEngineWillEnableCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in guard let self else { return } - self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") + self.log("OnEngineWillEnable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") #if os(iOS) || os(visionOS) || os(tvOS) self.log("Configuring audio session...") - // Backward compatibility - let configureFunc = self.state.customConfigureFunc ?? self.defaultConfigureAudioSessionFunc - let simulatedState = AudioManager.State(localTracksCount: isRecordingEnabled ? 1 : 0, remoteTracksCount: isPlayoutEnabled ? 1 : 0) - configureFunc(simulatedState, AudioManager.State()) + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback + do { + if isSessionActive { + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } else { + log("AudioSession activating category to: \(config.category)") + try session.setConfiguration(config.toRTCType(), active: true) + isSessionActive = true + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") #endif } + + RTC.audioDeviceModule.setOnEngineDidStopCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + guard let self else { return } + self.log("OnEngineDidDisable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") + + #if os(iOS) || os(visionOS) || os(tvOS) + self.log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + do { + if isPlayoutEnabled, !isRecordingEnabled { + let config: AudioSessionConfiguration = .playback + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } + if !isPlayoutEnabled, !isRecordingEnabled { + log("AudioSession deactivating") + try session.setActive(false) + isSessionActive = false + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + #endif + } + + RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in + guard let self else { return } + self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") + } } // MARK: - Private diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 33e558baf..4db9ac664 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -63,6 +63,24 @@ class AudioEngineTests: XCTestCase { XCTAssert(!adm.isPlayoutInitialized) } + func testRecordingAlwaysPreparedMode() async { + let adm = AudioManager.shared + + // Ensure initially not initialized. + XCTAssert(!adm.isRecordingInitialized) + + // Ensure recording is initialized after set to true. + adm.isRecordingAlwaysPrepared = true + XCTAssert(adm.isRecordingInitialized) + + adm.startRecording() + XCTAssert(adm.isRecordingInitialized) + + // Should be still initialized after stopRecording() is called. + adm.stopRecording() + XCTAssert(adm.isRecordingInitialized) + } + func testConfigureDucking() async { AudioManager.shared.isAdvancedDuckingEnabled = false XCTAssert(!AudioManager.shared.isAdvancedDuckingEnabled) From 3c9c0ddc7d5343b886f6efc03f0acd1d38444059 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:19:25 +0900 Subject: [PATCH 20/49] Chained engine observer --- .../AudioDeviceModuleDelegateAdapter.swift | 88 +++++++++++ .../LiveKit/Audio/AudioEngineObserver.swift | 63 ++++++++ .../Audio/AudioManager+EngineObserver.swift | 45 ------ .../Audio/DefaultAudioSessionObserver.swift | 89 ++++++++++++ Sources/LiveKit/Protocols/NextInvokable.swift | 22 +++ Sources/LiveKit/Track/AudioManager.swift | 137 +++--------------- 6 files changed, 283 insertions(+), 161 deletions(-) create mode 100644 Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift create mode 100644 Sources/LiveKit/Audio/AudioEngineObserver.swift delete mode 100644 Sources/LiveKit/Audio/AudioManager+EngineObserver.swift create mode 100644 Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift create mode 100644 Sources/LiveKit/Protocols/NextInvokable.swift diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift new file mode 100644 index 000000000..6f5a382cb --- /dev/null +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -0,0 +1,88 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +// Invoked on WebRTC's worker thread, do not block. +class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate { + weak var audioManager: AudioManager? + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didReceiveSpeechActivityEvent speechActivityEvent: RTCSpeechActivityEvent) { + guard let audioManager else { return } + audioManager.onMutedSpeechActivityEvent?(audioManager, speechActivityEvent.toLKType()) + } + + func audioDeviceModuleDidUpdateDevices(_: LKRTCAudioDeviceModule) { + guard let audioManager else { return } + audioManager.onDeviceUpdate?(audioManager) + } + + // Engine events + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineDidCreate(engine) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineWillStart(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine) { + guard let audioManager else { return } + let entryPoint = audioManager.state.engineObservers.buildChain() + entryPoint?.engineWillRelease(engine) + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureInputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + guard let audioManager else { return false } + let entryPoint = audioManager.state.engineObservers.buildChain() + return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false + } + + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + guard let audioManager else { return false } + let entryPoint = audioManager.state.engineObservers.buildChain() + return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false + } +} diff --git a/Sources/LiveKit/Audio/AudioEngineObserver.swift b/Sources/LiveKit/Audio/AudioEngineObserver.swift new file mode 100644 index 000000000..94c09b56c --- /dev/null +++ b/Sources/LiveKit/Audio/AudioEngineObserver.swift @@ -0,0 +1,63 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +/// Do not retain the engine object. +public protocol AudioEngineObserver: NextInvokable, Sendable { + func setNext(_ handler: any AudioEngineObserver) + + func engineDidCreate(_ engine: AVAudioEngine) + func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineWillStart(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) + func engineWillRelease(_ engine: AVAudioEngine) + + /// Provide custom implementation for internal AVAudioEngine's output configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. + func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool + /// Provide custom implementation for internal AVAudioEngine's input configuration. + /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. + /// Return true if custom implementation is provided, otherwise default implementation will be used. + func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool +} + +/// Default implementation to make it optional. +public extension AudioEngineObserver { + func engineDidCreate(_: AVAudioEngine) {} + func engineWillEnable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineWillStart(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineDidStop(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineDidDisable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} + func engineWillRelease(_: AVAudioEngine) {} + + func engineWillConnectOutput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } + func engineWillConnectInput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } +} + +extension [any AudioEngineObserver] { + func buildChain() -> Element? { + guard let first else { return nil } + + for i in 0 ..< count - 1 { + self[i].setNext(self[i + 1]) + } + + return first + } +} diff --git a/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift b/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift deleted file mode 100644 index 662e30f0f..000000000 --- a/Sources/LiveKit/Audio/AudioManager+EngineObserver.swift +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2025 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import AVFAudio - -/// Do not retain the engine object. -@objc -public protocol AudioEngineObserver: Chainable { - @objc optional - func engineDidCreate(_ engine: AVAudioEngine) - - @objc optional - func engineWillEnable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineWillStart(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineDidStop(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineDidDisable(_ engine: AVAudioEngine, playout: Bool, recording: Bool) - - @objc optional - func engineWillRelease(_ engine: AVAudioEngine) - - @objc optional - func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) - - @objc optional - func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -} diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift new file mode 100644 index 000000000..7bde7be0d --- /dev/null +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -0,0 +1,89 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFoundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { + var next: (any AudioEngineObserver)? + var isSessionActive = false + + public func setNext(_ handler: any AudioEngineObserver) { + next = handler + } + + public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + #if os(iOS) || os(visionOS) || os(tvOS) + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback + do { + if isSessionActive { + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } else { + log("AudioSession activating category to: \(config.category)") + try session.setConfiguration(config.toRTCType(), active: true) + isSessionActive = true + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + #endif + + // Call next last + next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } + + public func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + // Call next first + next?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + + #if os(iOS) || os(visionOS) || os(tvOS) + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + do { + if isPlayoutEnabled, !isRecordingEnabled { + let config: AudioSessionConfiguration = .playback + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } + if !isPlayoutEnabled, !isRecordingEnabled { + log("AudioSession deactivating") + try session.setActive(false) + isSessionActive = false + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + #endif + } +} diff --git a/Sources/LiveKit/Protocols/NextInvokable.swift b/Sources/LiveKit/Protocols/NextInvokable.swift new file mode 100644 index 000000000..4dc516155 --- /dev/null +++ b/Sources/LiveKit/Protocols/NextInvokable.swift @@ -0,0 +1,22 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +public protocol NextInvokable { + associatedtype Next + func setNext(_ handler: Next) +} diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 00c2cd622..48fb3c774 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -36,19 +36,6 @@ public class AudioManager: Loggable { public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void - // Engine events - public typealias OnEngineWillStart = (_ audioManager: AudioManager, _ engine: AVAudioEngine, _ playoutEnabled: Bool, _ recordingEnabled: Bool) -> Void - public typealias OnEngineWillConnectInput = (_ audioManager: AudioManager, - _ engine: AVAudioEngine, - _ src: AVAudioNode, - _ dst: AVAudioNode, - _ format: AVAudioFormat) -> Bool - public typealias OnEngineWillConnectOutput = (_ audioManager: AudioManager, - _ engine: AVAudioEngine, - _ src: AVAudioNode, - _ dst: AVAudioNode, - _ format: AVAudioFormat) -> Bool - public typealias OnSpeechActivityEvent = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void #if os(iOS) || os(visionOS) || os(tvOS) @@ -125,6 +112,8 @@ public class AudioManager: Loggable { public var sessionConfiguration: AudioSessionConfiguration? #endif + public var engineObservers = [any AudioEngineObserver]() + public var trackState: TrackState { switch (localTracksCount > 0, remoteTracksCount > 0) { case (true, false): return .localOnly @@ -195,50 +184,12 @@ public class AudioManager: Loggable { set { RTC.audioDeviceModule.inputDevice = newValue._ioDevice } } - public var onDeviceUpdate: DeviceUpdateFunc? { - didSet { - RTC.audioDeviceModule.setDevicesDidUpdateCallback { [weak self] in - guard let self else { return } - self.onDeviceUpdate?(self) - } - } - } - - /// Provide custom implementation for internal AVAudioEngine's input configuration. - /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. - /// Return true if custom implementation is provided, otherwise default implementation will be used. - public var onEngineWillConnectInput: OnEngineWillConnectInput? { - didSet { - RTC.audioDeviceModule.setOnEngineWillConnectInputCallback { [weak self] engine, src, dst, format in - guard let self else { return false } - return self.onEngineWillConnectInput?(self, engine, src, dst, format) ?? false - } - } - } - - /// Provide custom implementation for internal AVAudioEngine's output configuration. - /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. - /// Return true if custom implementation is provided, otherwise default implementation will be used. - public var onEngineWillConnectOutput: OnEngineWillConnectOutput? { - didSet { - RTC.audioDeviceModule.setOnEngineWillConnectOutputCallback { [weak self] engine, src, dst, format in - guard let self else { return false } - return self.onEngineWillConnectOutput?(self, engine, src, dst, format) ?? false - } - } - } + public var onDeviceUpdate: DeviceUpdateFunc? /// Detect voice activity even if the mic is muted. /// Internal audio engine must be initialized by calling ``prepareRecording()`` or /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. - public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? { - didSet { - RTC.audioDeviceModule.setSpeechActivityCallback { [weak self] event in - guard let self else { return } - self.onMutedSpeechActivityEvent?(self, event.toLKType()) - } - } - } + public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? public var isManualRenderingMode: Bool { get { RTC.audioDeviceModule.isManualRenderingMode } @@ -280,6 +231,17 @@ public class AudioManager: Loggable { RTC.audioDeviceModule.initAndStartRecording() } + /// Set a chain of ``AudioEngineObserver``s. + /// Defaults to having a single ``DefaultAudioSessionObserver`` initially. + /// + /// The first object will be invoked and is responsible for calling the next object. + /// See ``NextInvokable`` protocol for details. + /// + /// Objects set here will be retained. + public func set(engineObservers: [any AudioEngineObserver]) { + state.mutate { $0.engineObservers = engineObservers } + } + // MARK: - For testing var isPlayoutInitialized: Bool { @@ -329,72 +291,15 @@ public class AudioManager: Loggable { case remote } - let state = StateSync(State()) + let state: StateSync - var isSessionActive: Bool = false + let admDelegateAdapter: AudioDeviceModuleDelegateAdapter init() { - RTC.audioDeviceModule.setOnEngineWillEnableCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in - guard let self else { return } - self.log("OnEngineWillEnable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - - #if os(iOS) || os(visionOS) || os(tvOS) - self.log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { session.unlockForConfiguration() } - - let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback - do { - if isSessionActive { - log("AudioSession switching category to: \(config.category)") - try session.setConfiguration(config.toRTCType()) - } else { - log("AudioSession activating category to: \(config.category)") - try session.setConfiguration(config.toRTCType(), active: true) - isSessionActive = true - } - } catch { - log("AudioSession failed to configure with error: \(error)", .error) - } - - log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") - #endif - } - - RTC.audioDeviceModule.setOnEngineDidStopCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in - guard let self else { return } - self.log("OnEngineDidDisable isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - - #if os(iOS) || os(visionOS) || os(tvOS) - self.log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { session.unlockForConfiguration() } - - do { - if isPlayoutEnabled, !isRecordingEnabled { - let config: AudioSessionConfiguration = .playback - log("AudioSession switching category to: \(config.category)") - try session.setConfiguration(config.toRTCType()) - } - if !isPlayoutEnabled, !isRecordingEnabled { - log("AudioSession deactivating") - try session.setActive(false) - isSessionActive = false - } - } catch { - log("AudioSession failed to configure with error: \(error)", .error) - } - - log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") - #endif - } - - RTC.audioDeviceModule.setOnEngineWillStartCallback { [weak self] _, isPlayoutEnabled, isRecordingEnabled in - guard let self else { return } - self.log("OnEngineWillStart isPlayoutEnabled: \(isPlayoutEnabled), isRecordingEnabled: \(isRecordingEnabled)") - } + state = StateSync(State(engineObservers: [DefaultAudioSessionObserver()])) + admDelegateAdapter = AudioDeviceModuleDelegateAdapter() + admDelegateAdapter.audioManager = self + RTC.audioDeviceModule.observer = admDelegateAdapter } // MARK: - Private From 7e48b7b686a58a750cc2c9d5e88ec15225ff44d9 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:20:26 +0900 Subject: [PATCH 21/49] lib 125.6422.12-exp.5 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 753c73108..fd2b06a40 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.5"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 2e8dddf83..c038dc0c4 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.4"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.5"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From d3deb72f6c92eeb11b6153fe652901c3eacb5f4f Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 13:36:41 +0900 Subject: [PATCH 22/49] Update test --- .../AudioDeviceModuleDelegateAdapter.swift | 4 +-- Tests/LiveKitTests/AudioEngineTests.swift | 26 ++++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift index 6f5a382cb..53652f525 100644 --- a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -74,13 +74,13 @@ class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate entryPoint?.engineWillRelease(engine) } - func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureInputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } let entryPoint = audioManager.state.engineObservers.buildChain() return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false } - func audioDeviceModule(_: LKRTCAudioDeviceModule, engine : AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format : AVAudioFormat) -> Bool { + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } let entryPoint = audioManager.state.engineObservers.buildChain() return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 4db9ac664..9cb6d54d0 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -135,19 +135,13 @@ class AudioEngineTests: XCTestCase { // Test the manual rendering mode (no-device mode) of AVAudioEngine based AudioDeviceModule. // In manual rendering, no device access will be initialized such as mic and speaker. func testManualRenderingMode() async throws { - // Attach sin wave generator when engine requests input node... - // inputMixerNode will automatically convert to RTC's internal format (int16). - // AVAudioEngine.attach() retains the node. - AudioManager.shared.onEngineWillConnectInput = { _, engine, _, dst, format in - let sin = SineWaveSourceNode() - engine.attach(sin) - engine.connect(sin, to: dst, format: format) - return true - } - // Set manual rendering mode... AudioManager.shared.isManualRenderingMode = true + // Attach sine wave generator when engine requests input node. + // inputMixerNode will automatically convert to RTC's internal format (int16). + AudioManager.shared.set(engineObservers: [RewriteInputToSineWaveGenerator()]) + // Check if manual rendering mode is set... let isManualRenderingMode = AudioManager.shared.isManualRenderingMode print("manualRenderingMode: \(isManualRenderingMode)") @@ -176,3 +170,15 @@ class AudioEngineTests: XCTestCase { try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) } } + +final class RewriteInputToSineWaveGenerator: AudioEngineObserver { + func setNext(_: any LiveKit.AudioEngineObserver) {} + func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + print("engineWillConnectInput") + let sin = SineWaveSourceNode() + // AVAudioEngine.attach() retains the node. + engine.attach(sin) + engine.connect(sin, to: dst, format: format) + return true + } +} From aa8977b10a102449a0a654cbad34549d2e343a80 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:40:27 +0900 Subject: [PATCH 23/49] Fix test --- Tests/LiveKitTests/AudioEngineTests.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 9cb6d54d0..70f586ebb 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -27,6 +27,7 @@ class AudioEngineTests: XCTestCase { override func tearDown() async throws {} + #if !targetEnvironment(simulator) // Test if mic is authorized. Only works on device. func testMicAuthorized() async { let status = AVCaptureDevice.authorizationStatus(for: .audio) @@ -37,6 +38,7 @@ class AudioEngineTests: XCTestCase { XCTAssert(status == .authorized) } + #endif // Test if state transitions pass internal checks. func testStateTransitions() async { From 82323b4bfc4f836133f654b9fb3303d1388ae7b9 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 24 Jan 2025 00:37:22 +0900 Subject: [PATCH 24/49] Update manual render test --- Sources/LiveKit/Track/AudioManager.swift | 3 +- Tests/LiveKitTests/AudioEngineTests.swift | 7 +++-- .../Support/SinWaveSourceNode.swift | 29 ++++++++++++------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 48fb3c774..15212265e 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -293,11 +293,10 @@ public class AudioManager: Loggable { let state: StateSync - let admDelegateAdapter: AudioDeviceModuleDelegateAdapter + let admDelegateAdapter = AudioDeviceModuleDelegateAdapter() init() { state = StateSync(State(engineObservers: [DefaultAudioSessionObserver()])) - admDelegateAdapter = AudioDeviceModuleDelegateAdapter() admDelegateAdapter.audioManager = self RTC.audioDeviceModule.observer = admDelegateAdapter } diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 70f586ebb..62c0e0da8 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -139,7 +139,6 @@ class AudioEngineTests: XCTestCase { func testManualRenderingMode() async throws { // Set manual rendering mode... AudioManager.shared.isManualRenderingMode = true - // Attach sine wave generator when engine requests input node. // inputMixerNode will automatically convert to RTC's internal format (int16). AudioManager.shared.set(engineObservers: [RewriteInputToSineWaveGenerator()]) @@ -151,7 +150,10 @@ class AudioEngineTests: XCTestCase { let recorder = try AudioRecorder() - let track = LocalAudioTrack.createTrack() + // Note: AudioCaptureOptions will not be applied since track is not published. + let noProcessingOptions = AudioCaptureOptions(echoCancellation: false, noiseSuppression: false, autoGainControl: false, highpassFilter: false) + + let track = LocalAudioTrack.createTrack(options: noProcessingOptions) track.add(audioRenderer: recorder) // Start engine... @@ -178,7 +180,6 @@ final class RewriteInputToSineWaveGenerator: AudioEngineObserver { func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool { print("engineWillConnectInput") let sin = SineWaveSourceNode() - // AVAudioEngine.attach() retains the node. engine.attach(sin) engine.connect(sin, to: dst, format: format) return true diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index fa031d9bf..8913ac0a3 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -26,25 +26,32 @@ class SineWaveSourceNode: AVAudioSourceNode { let format = AVAudioFormat(standardFormatWithSampleRate: sampleRate, channels: 1)! - var currentPhase = 0.0 - let phaseIncrement = 2.0 * Double.pi * frequency / sampleRate + let twoPi = 2 * Float.pi + let amplitude: Float = 0.5 + var currentPhase: Float = 0.0 + let phaseIncrement: Float = (twoPi / Float(sampleRate)) * Float(frequency) let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in print("AVAudioSourceNodeRenderBlock frameCount: \(frameCount)") let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) - guard let ptr = ablPointer[0].mData?.assumingMemoryBound(to: Float.self) else { - return kAudioUnitErr_InvalidParameter - } // Generate sine wave samples for frame in 0 ..< Int(frameCount) { - ptr[frame] = Float(sin(currentPhase)) - - // Update the phase + // Get the signal value for this frame at time. + let value = sin(currentPhase) * amplitude + // Advance the phase for the next frame. currentPhase += phaseIncrement - - // Keep phase within [0, 2π] range using fmod for stability - currentPhase = fmod(currentPhase, 2.0 * Double.pi) + if currentPhase >= twoPi { + currentPhase -= twoPi + } + if currentPhase < 0.0 { + currentPhase += twoPi + } + // Set the same value on all channels (due to the inputFormat, there's only one channel though). + for buffer in ablPointer { + let buf: UnsafeMutableBufferPointer = UnsafeMutableBufferPointer(buffer) + buf[frame] = value + } } return noErr From d846a00b771919f4350c1b5524cbd4a898e94529 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 24 Jan 2025 00:53:57 +0900 Subject: [PATCH 25/49] no processing option for capture options --- Sources/LiveKit/Types/Options/AudioCaptureOptions.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift index 1e0680b0d..af5ad9999 100644 --- a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift @@ -24,6 +24,14 @@ internal import LiveKitWebRTC @objc public final class AudioCaptureOptions: NSObject, CaptureOptions, Sendable { + public static let noProcessing = AudioCaptureOptions( + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + typingNoiseDetection: false, + highpassFilter: false + ) + @objc public let echoCancellation: Bool From b6b51ef0256be8088a5e6c2e42c19ee1272f4775 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 24 Jan 2025 02:39:42 +0900 Subject: [PATCH 26/49] pcm buffer helper --- Sources/LiveKit/Convenience/AudioProcessing.swift | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Sources/LiveKit/Convenience/AudioProcessing.swift b/Sources/LiveKit/Convenience/AudioProcessing.swift index 8e9cfc73a..af8cd734c 100644 --- a/Sources/LiveKit/Convenience/AudioProcessing.swift +++ b/Sources/LiveKit/Convenience/AudioProcessing.swift @@ -56,6 +56,17 @@ public extension LKAudioBuffer { } } +public extension CMSampleBuffer { + func toAVAudioPCMBuffer() -> AVAudioPCMBuffer? { + let format = AVAudioFormat(cmAudioFormatDescription: formatDescription!) + let numSamples = AVAudioFrameCount(numSamples) + let pcmBuffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: numSamples)! + pcmBuffer.frameLength = numSamples + CMSampleBufferCopyPCMDataIntoAudioBufferList(self, at: 0, frameCount: Int32(numSamples), into: pcmBuffer.mutableAudioBufferList) + return pcmBuffer + } +} + public extension AVAudioPCMBuffer { /// Computes Peak and Linear Scale RMS Value (Average) for all channels. func audioLevels() -> [AudioLevel] { From c44b9e52e7208f4aca1cdbb18dec382892ba7335 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 24 Jan 2025 03:15:40 +0900 Subject: [PATCH 27/49] Implement --- .../Audio/AudioEngineAudioInputObserver.swift | 69 ++++++++++++++++ Sources/LiveKit/Track/AudioManager.swift | 2 + .../Track/Capturers/MacOSScreenCapturer.swift | 80 +++++++++++-------- .../Options/ScreenShareCaptureOptions.swift | 11 ++- 4 files changed, 128 insertions(+), 34 deletions(-) create mode 100644 Sources/LiveKit/Audio/AudioEngineAudioInputObserver.swift diff --git a/Sources/LiveKit/Audio/AudioEngineAudioInputObserver.swift b/Sources/LiveKit/Audio/AudioEngineAudioInputObserver.swift new file mode 100644 index 000000000..1ec278fea --- /dev/null +++ b/Sources/LiveKit/Audio/AudioEngineAudioInputObserver.swift @@ -0,0 +1,69 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFoundation + +#if swift(>=5.9) +internal import LiveKitWebRTC +#else +@_implementationOnly import LiveKitWebRTC +#endif + +public final class AudioEngineAudioInputObserver: AudioEngineObserver, Loggable { + public let playerNode = AVAudioPlayerNode() + public let playerMixerNode = AVAudioMixerNode() + public let micMixerNode = AVAudioMixerNode() + + // + let playerNodeFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, + sampleRate: 48000, + channels: 2, + interleaved: false) + + var next: (any AudioEngineObserver)? + + public init() {} + + public func setNext(_ handler: any AudioEngineObserver) { + next = handler + } + + public func engineDidCreate(_ engine: AVAudioEngine) { + engine.attach(playerNode) + engine.attach(playerMixerNode) + engine.attach(micMixerNode) + + micMixerNode.outputVolume = 0.0 + } + + public func engineWillRelease(_ engine: AVAudioEngine) { + engine.detach(playerNode) + engine.detach(playerMixerNode) + engine.detach(micMixerNode) + } + + public func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + // inputPlayer -> playerMixer -> mainMixer + engine.connect(playerNode, to: playerMixerNode, format: playerNodeFormat) + engine.connect(playerMixerNode, to: dst, format: format) + + // mic -> micMixer -> mainMixer + engine.connect(src, to: micMixerNode, format: format) + engine.connect(micMixerNode, to: dst, format: format) + + return true + } +} diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 15212265e..f4c09a583 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -295,6 +295,8 @@ public class AudioManager: Loggable { let admDelegateAdapter = AudioDeviceModuleDelegateAdapter() + public weak var screenShareAppAudioPlayerNode: AVAudioPlayerNode? + init() { state = StateSync(State(engineObservers: [DefaultAudioSessionObserver()])) admDelegateAdapter.audioManager = self diff --git a/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift b/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift index c92811ab2..994b60d7d 100644 --- a/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift +++ b/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift @@ -98,9 +98,16 @@ public class MacOSScreenCapturer: VideoCapturer { configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange configuration.showsCursor = options.showCursor + if #available(macOS 13.0, *) { + configuration.capturesAudio = options.appAudio + } + // Why does SCStream hold strong reference to delegate? let stream = SCStream(filter: filter, configuration: configuration, delegate: nil) try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: nil) + if #available(macOS 13.0, *) { + try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: nil) + } try await stream.startCapture() _screenCapturerState.mutate { $0.scStream = stream } @@ -200,7 +207,6 @@ extension MacOSScreenCapturer { @available(macOS 12.3, *) extension MacOSScreenCapturer: SCStreamOutput { public func stream(_: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, - of outputType: SCStreamOutputType) { guard case .started = captureState else { @@ -211,40 +217,50 @@ extension MacOSScreenCapturer: SCStreamOutput { // Return early if the sample buffer is invalid. guard sampleBuffer.isValid else { return } - guard case .screen = outputType else { return } - - // Retrieve the array of metadata attachments from the sample buffer. - guard let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, - createIfNecessary: false) as? [[SCStreamFrameInfo: Any]], - let attachments = attachmentsArray.first else { return } - - // Validate the status of the frame. If it isn't `.complete`, return nil. - guard let statusRawValue = attachments[SCStreamFrameInfo.status] as? Int, - let status = SCFrameStatus(rawValue: statusRawValue), - status == .complete else { return } - - // Retrieve the content rectangle, scale, and scale factor. - guard let contentRectDict = attachments[.contentRect], - let contentRect = CGRect(dictionaryRepresentation: contentRectDict as! CFDictionary), - // let contentScale = attachments[.contentScale] as? CGFloat, - let scaleFactor = attachments[.scaleFactor] as? CGFloat else { return } - - // Schedule resend timer - let newTimer = Task.detached(priority: .utility) { [weak self] in - while true { - try? await Task.sleep(nanoseconds: UInt64(1 * 1_000_000_000)) - if Task.isCancelled { break } - guard let self else { break } - try await self._capturePreviousFrame() + if case .audio = outputType { + if let pcm = sampleBuffer.toAVAudioPCMBuffer(), + let node = AudioManager.shared.screenShareAppAudioPlayerNode, + let engine = node.engine, engine.isRunning + { + node.scheduleBuffer(pcm) + if !node.isPlaying { + node.play() + } + } + } else if case .screen = outputType { + // Retrieve the array of metadata attachments from the sample buffer. + guard let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, + createIfNecessary: false) as? [[SCStreamFrameInfo: Any]], + let attachments = attachmentsArray.first else { return } + + // Validate the status of the frame. If it isn't `.complete`, return nil. + guard let statusRawValue = attachments[SCStreamFrameInfo.status] as? Int, + let status = SCFrameStatus(rawValue: statusRawValue), + status == .complete else { return } + + // Retrieve the content rectangle, scale, and scale factor. + guard let contentRectDict = attachments[.contentRect], + let contentRect = CGRect(dictionaryRepresentation: contentRectDict as! CFDictionary), + // let contentScale = attachments[.contentScale] as? CGFloat, + let scaleFactor = attachments[.scaleFactor] as? CGFloat else { return } + + // Schedule resend timer + let newTimer = Task.detached(priority: .utility) { [weak self] in + while true { + try? await Task.sleep(nanoseconds: UInt64(1 * 1_000_000_000)) + if Task.isCancelled { break } + guard let self else { break } + try await self._capturePreviousFrame() + } } - } - _screenCapturerState.mutate { - $0.resendTimer?.cancel() - $0.resendTimer = newTimer - } + _screenCapturerState.mutate { + $0.resendTimer?.cancel() + $0.resendTimer = newTimer + } - capture(sampleBuffer, contentRect: contentRect, scaleFactor: scaleFactor) + capture(sampleBuffer, contentRect: contentRect, scaleFactor: scaleFactor) + } } } diff --git a/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift b/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift index e894f42c5..0414e9ca2 100644 --- a/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift @@ -34,17 +34,22 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen @objc public let includeCurrentApplication: Bool + @objc + public let appAudio: Bool + public init(dimensions: Dimensions = .h1080_169, fps: Int = 30, showCursor: Bool = true, useBroadcastExtension: Bool = false, - includeCurrentApplication: Bool = false) + includeCurrentApplication: Bool = false, + appAudio: Bool = true) { self.dimensions = dimensions self.fps = fps self.showCursor = showCursor self.useBroadcastExtension = useBroadcastExtension self.includeCurrentApplication = includeCurrentApplication + self.appAudio = appAudio } // MARK: - Equal @@ -55,7 +60,8 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen fps == other.fps && showCursor == other.showCursor && useBroadcastExtension == other.useBroadcastExtension && - includeCurrentApplication == other.includeCurrentApplication + includeCurrentApplication == other.includeCurrentApplication && + appAudio == other.appAudio } override public var hash: Int { @@ -65,6 +71,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen hasher.combine(showCursor) hasher.combine(useBroadcastExtension) hasher.combine(includeCurrentApplication) + hasher.combine(appAudio) return hasher.finalize() } } From 0e05ca0acc13932125193cae009ade3beb69a03b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 28 Jan 2025 12:02:01 +0900 Subject: [PATCH 28/49] Runtime agc / bypass vp toggle --- .../AudioDeviceModuleDelegateAdapter.swift | 4 +-- .../LiveKit/Audio/AudioEngineObserver.swift | 8 ++--- Sources/LiveKit/Track/AudioManager.swift | 36 +++++++++++++------ 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift index 53652f525..a93317c53 100644 --- a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -74,13 +74,13 @@ class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate entryPoint?.engineWillRelease(engine) } - func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode?, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } let entryPoint = audioManager.state.engineObservers.buildChain() return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false } - func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { + func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode?, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } let entryPoint = audioManager.state.engineObservers.buildChain() return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false diff --git a/Sources/LiveKit/Audio/AudioEngineObserver.swift b/Sources/LiveKit/Audio/AudioEngineObserver.swift index 94c09b56c..b3f6cc9d5 100644 --- a/Sources/LiveKit/Audio/AudioEngineObserver.swift +++ b/Sources/LiveKit/Audio/AudioEngineObserver.swift @@ -30,11 +30,11 @@ public protocol AudioEngineObserver: NextInvokable, Sendable { /// Provide custom implementation for internal AVAudioEngine's output configuration. /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. /// Return true if custom implementation is provided, otherwise default implementation will be used. - func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool + func engineWillConnectOutput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode?, format: AVAudioFormat) -> Bool /// Provide custom implementation for internal AVAudioEngine's input configuration. /// Buffers flow from `src` to `dst`. Preferred format to connect node is provided as `format`. /// Return true if custom implementation is provided, otherwise default implementation will be used. - func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool + func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool } /// Default implementation to make it optional. @@ -46,8 +46,8 @@ public extension AudioEngineObserver { func engineDidDisable(_: AVAudioEngine, isPlayoutEnabled _: Bool, isRecordingEnabled _: Bool) {} func engineWillRelease(_: AVAudioEngine) {} - func engineWillConnectOutput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } - func engineWillConnectInput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } + func engineWillConnectOutput(_: AVAudioEngine, src _: AVAudioNode, dst _: AVAudioNode?, format _: AVAudioFormat) -> Bool { false } + func engineWillConnectInput(_: AVAudioEngine, src _: AVAudioNode?, dst _: AVAudioNode, format _: AVAudioFormat) -> Bool { false } } extension [any AudioEngineObserver] { diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 15212265e..60eec7467 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -191,16 +191,6 @@ public class AudioManager: Loggable { /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? - public var isManualRenderingMode: Bool { - get { RTC.audioDeviceModule.isManualRenderingMode } - set { - let result = RTC.audioDeviceModule.setManualRenderingMode(newValue) - if !result { - log("Failed to set manual rendering mode", .error) - } - } - } - /// Enables advanced ducking which ducks other audio based on the presence of voice activity from local and remote chat participants. /// Default: true. public var isAdvancedDuckingEnabled: Bool { @@ -215,6 +205,32 @@ public class AudioManager: Loggable { set { RTC.audioDeviceModule.duckingLevel = newValue.rawValue } } + /// Bypass Voice-Processing I/O of internal AVAudioEngine. + /// It is valid to toggle this at runtime. + public var isVoiceProcessingBypassed: Bool { + get { RTC.audioDeviceModule.isVoiceProcessingBypassed } + set { RTC.audioDeviceModule.isVoiceProcessingBypassed = newValue } + } + + /// Bypass the Auto Gain Control of internal AVAudioEngine. + /// It is valid to toggle this at runtime. + public var isVoiceProcessingAGCEnabled: Bool { + get { RTC.audioDeviceModule.isVoiceProcessingAGCEnabled } + set { RTC.audioDeviceModule.isVoiceProcessingAGCEnabled = newValue } + } + + /// Enables manual-rendering (no-device) mode of AVAudioEngine. + /// Currently experimental. + public var isManualRenderingMode: Bool { + get { RTC.audioDeviceModule.isManualRenderingMode } + set { + let result = RTC.audioDeviceModule.setManualRenderingMode(newValue) + if !result { + log("Failed to set manual rendering mode", .error) + } + } + } + // MARK: - Recording /// Keep recording initialized (mic input) and pre-warm voice processing etc. From 8b35ab691c5dca766c4a3a1183d92e5eb44e3a45 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 28 Jan 2025 12:05:10 +0900 Subject: [PATCH 29/49] Runtime bypass is valid now --- Sources/LiveKit/Core/RTC.swift | 16 +--------------- Sources/LiveKit/Core/Room.swift | 7 +++---- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Sources/LiveKit/Core/RTC.swift b/Sources/LiveKit/Core/RTC.swift index 1d658d73c..e1632cee7 100644 --- a/Sources/LiveKit/Core/RTC.swift +++ b/Sources/LiveKit/Core/RTC.swift @@ -50,19 +50,6 @@ private class VideoEncoderFactorySimulcast: LKRTCVideoEncoderFactorySimulcast { } class RTC { - private static var _bypassVoiceProcessing: Bool = false - private static var _peerConnectionFactoryInitialized = false - - static var bypassVoiceProcessing: Bool { - get { _bypassVoiceProcessing } - set { - if _peerConnectionFactoryInitialized { - logger.log("Warning: Setting bypassVoiceProcessing after PeerConnectionFactory initialization has no effect. Set it at application launch.", .warning, type: Room.self) - } - _bypassVoiceProcessing = newValue - } - } - static let h264BaselineLevel5CodecInfo: LKRTCVideoCodecInfo = { // this should never happen guard let profileLevelId = LKRTCH264ProfileLevelId(profile: .constrainedBaseline, level: .level5) else { @@ -100,8 +87,7 @@ class RTC { logger.log("Initializing PeerConnectionFactory...", type: Room.self) - _peerConnectionFactoryInitialized = true - return LKRTCPeerConnectionFactory(bypassVoiceProcessing: bypassVoiceProcessing, + return LKRTCPeerConnectionFactory(bypassVoiceProcessing: false, encoderFactory: encoderFactory, decoderFactory: decoderFactory, audioProcessingModule: audioProcessingModule) diff --git a/Sources/LiveKit/Core/Room.swift b/Sources/LiveKit/Core/Room.swift index 96ed89908..da800b788 100644 --- a/Sources/LiveKit/Core/Room.swift +++ b/Sources/LiveKit/Core/Room.swift @@ -518,12 +518,11 @@ extension Room: AppStateDelegate { public extension Room { /// Set this to true to bypass initialization of voice processing. - /// Must be set before RTCPeerConnectionFactory gets initialized. - /// The most reliable place to set this is in your application's initialization process. + @available(*, deprecated, renamed: "AudioManager.shared.isVoiceProcessingBypassed") @objc static var bypassVoiceProcessing: Bool { - get { RTC.bypassVoiceProcessing } - set { RTC.bypassVoiceProcessing = newValue } + get { AudioManager.shared.isVoiceProcessingBypassed } + set { AudioManager.shared.isVoiceProcessingBypassed = newValue } } } From a28b9c8b6788f3d0b3446151803c40b9b0d3c06c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 28 Jan 2025 12:06:15 +0900 Subject: [PATCH 30/49] Use 125.6422.12-exp.6 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index fd2b06a40..0bfb31b77 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.5"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.6"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index c038dc0c4..e753e4a6b 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.5"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.6"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From 9c18ff5bc2bf96e1e9d305c45967827e06af69e1 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 28 Jan 2025 12:15:42 +0900 Subject: [PATCH 31/49] Fix manual render test --- Tests/LiveKitTests/AudioEngineTests.swift | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 62c0e0da8..97d86bb87 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -151,9 +151,7 @@ class AudioEngineTests: XCTestCase { let recorder = try AudioRecorder() // Note: AudioCaptureOptions will not be applied since track is not published. - let noProcessingOptions = AudioCaptureOptions(echoCancellation: false, noiseSuppression: false, autoGainControl: false, highpassFilter: false) - - let track = LocalAudioTrack.createTrack(options: noProcessingOptions) + let track = LocalAudioTrack.createTrack(options: .noProcessing) track.add(audioRenderer: recorder) // Start engine... @@ -177,7 +175,7 @@ class AudioEngineTests: XCTestCase { final class RewriteInputToSineWaveGenerator: AudioEngineObserver { func setNext(_: any LiveKit.AudioEngineObserver) {} - func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool { print("engineWillConnectInput") let sin = SineWaveSourceNode() engine.attach(sin) From 89e0620e26c49412913cf9c350d7e8347fc968ac Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 29 Jan 2025 21:57:58 +0900 Subject: [PATCH 32/49] Update manual render tests --- Tests/LiveKitTests/AudioEngineTests.swift | 116 ++++++++++++++++++++-- 1 file changed, 109 insertions(+), 7 deletions(-) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 97d86bb87..921fe2700 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -136,12 +136,12 @@ class AudioEngineTests: XCTestCase { // Test the manual rendering mode (no-device mode) of AVAudioEngine based AudioDeviceModule. // In manual rendering, no device access will be initialized such as mic and speaker. - func testManualRenderingMode() async throws { + func testManualRenderingModeSineGenerator() async throws { // Set manual rendering mode... AudioManager.shared.isManualRenderingMode = true // Attach sine wave generator when engine requests input node. // inputMixerNode will automatically convert to RTC's internal format (int16). - AudioManager.shared.set(engineObservers: [RewriteInputToSineWaveGenerator()]) + AudioManager.shared.set(engineObservers: [SineWaveNodeHook()]) // Check if manual rendering mode is set... let isManualRenderingMode = AudioManager.shared.isManualRenderingMode @@ -169,17 +169,119 @@ class AudioEngineTests: XCTestCase { // Play the recorded file... let player = try AVAudioPlayer(contentsOf: recorder.filePath) player.play() - try? await Task.sleep(nanoseconds: 5 * 1_000_000_000) + while player.isPlaying { + try? await Task.sleep(nanoseconds: 1 * 100_000_000) // 10ms + } + } + + func testManualRenderingModeAudioFile() async throws { + // Sample audio + let url = URL(string: "https://github.com/rafaelreis-hotmart/Audio-Sample-files/raw/refs/heads/master/sample.wav")! + + print("Downloading sample audio from \(url)...") + let (downloadedLocalUrl, _) = try await URLSession.shared.downloadBackport(from: url) + + // Move the file to a new temporary location with a more descriptive name, if desired + let tempLocalUrl = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString).appendingPathExtension("wav") + try FileManager.default.moveItem(at: downloadedLocalUrl, to: tempLocalUrl) + print("Original file: \(tempLocalUrl)") + + let audioFile = try AVAudioFile(forReading: tempLocalUrl) + let audioFileFormat = audioFile.processingFormat // AVAudioFormat object + + print("Sample Rate: \(audioFileFormat.sampleRate)") + print("Channel Count: \(audioFileFormat.channelCount)") + print("Common Format: \(audioFileFormat.commonFormat)") + print("Interleaved: \(audioFileFormat.isInterleaved)") + + // Set manual rendering mode... + AudioManager.shared.isManualRenderingMode = true + + let playerNodeHook = PlayerNodeHook(playerNodeFormat: audioFileFormat) + AudioManager.shared.set(engineObservers: [playerNodeHook]) + + // Check if manual rendering mode is set... + let isManualRenderingMode = AudioManager.shared.isManualRenderingMode + print("manualRenderingMode: \(isManualRenderingMode)") + XCTAssert(isManualRenderingMode) + + let recorder = try AudioRecorder() + + // Note: AudioCaptureOptions will not be applied since track is not published. + let track = LocalAudioTrack.createTrack(options: .noProcessing) + track.add(audioRenderer: recorder) + + // Start engine... + AudioManager.shared.startLocalRecording() + + let scheduleAndPlayTask = Task { + print("Will scheduleFile") + await playerNodeHook.playerNode.scheduleFile(audioFile, at: nil) + print("Did scheduleFile") + } + + // Wait for audio file to be consumed... + playerNodeHook.playerNode.play() + await scheduleAndPlayTask.value + + recorder.close() + print("Processed file: \(recorder.filePath)") + + // Stop engine + AudioManager.shared.stopRecording() + + // Play the recorded file... + let player = try AVAudioPlayer(contentsOf: recorder.filePath) + player.play() + while player.isPlaying { + try? await Task.sleep(nanoseconds: 1 * 100_000_000) // 10ms + } } } -final class RewriteInputToSineWaveGenerator: AudioEngineObserver { +final class SineWaveNodeHook: AudioEngineObserver { + let sineWaveNode = SineWaveSourceNode() + func setNext(_: any LiveKit.AudioEngineObserver) {} + func engineDidCreate(_ engine: AVAudioEngine) { + engine.attach(sineWaveNode) + } + + func engineWillRelease(_ engine: AVAudioEngine) { + engine.detach(sineWaveNode) + } + + func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + print("engineWillConnectInput") + engine.connect(sineWaveNode, to: dst, format: format) + return true + } +} + +final class PlayerNodeHook: AudioEngineObserver { + public let playerNode = AVAudioPlayerNode() + public let playerMixerNode = AVAudioMixerNode() + public let playerNodeFormat: AVAudioFormat + + init(playerNodeFormat: AVAudioFormat) { + self.playerNodeFormat = playerNodeFormat + } + + func setNext(_: any LiveKit.AudioEngineObserver) {} + public func engineDidCreate(_ engine: AVAudioEngine) { + engine.attach(playerNode) + engine.attach(playerMixerNode) + } + + public func engineWillRelease(_ engine: AVAudioEngine) { + engine.detach(playerNode) + engine.detach(playerMixerNode) + } + func engineWillConnectInput(_ engine: AVAudioEngine, src _: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool { print("engineWillConnectInput") - let sin = SineWaveSourceNode() - engine.attach(sin) - engine.connect(sin, to: dst, format: format) + engine.connect(playerNode, to: playerMixerNode, format: playerNodeFormat) + engine.connect(playerMixerNode, to: dst, format: format) return true } } From f88456ed7d5e655c3bdee40553cbb3e4d4692534 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 29 Jan 2025 22:09:53 +0900 Subject: [PATCH 33/49] Use 125.6422.12 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- Tests/LiveKitTests/AudioEngineTests.swift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Package.swift b/Package.swift index 0bfb31b77..478c4c453 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.6"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index e753e4a6b..866554d04 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12-exp.6"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 921fe2700..5607e4369 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -14,7 +14,7 @@ * limitations under the License. */ -import AVFoundation +@preconcurrency import AVFoundation @testable import LiveKit import LiveKitWebRTC import XCTest From 5ad8bd513b5fb8cc84909c8601555b7c0c64d77a Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 29 Jan 2025 22:55:29 +0900 Subject: [PATCH 34/49] make state sync sendable --- Sources/LiveKit/Support/StateSync.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/LiveKit/Support/StateSync.swift b/Sources/LiveKit/Support/StateSync.swift index 9c7c2ae70..3ba4955af 100644 --- a/Sources/LiveKit/Support/StateSync.swift +++ b/Sources/LiveKit/Support/StateSync.swift @@ -18,7 +18,7 @@ import Combine import Foundation @dynamicMemberLookup -public final class StateSync { +public final class StateSync: @unchecked Sendable { // MARK: - Types public typealias OnDidMutate = (_ newState: State, _ oldState: State) -> Void From c5ee683b46a5902b20d80055bd2aa1dd785df447 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 29 Jan 2025 23:29:51 +0900 Subject: [PATCH 35/49] Backward compatibility for custom config func --- .../Audio/DefaultAudioSessionObserver.swift | 131 +++++++++++------- Sources/LiveKit/Track/AudioManager.swift | 8 +- 2 files changed, 91 insertions(+), 48 deletions(-) diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift index 7bde7be0d..58e6fec54 100644 --- a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -14,6 +14,8 @@ * limitations under the License. */ +#if os(iOS) || os(visionOS) || os(tvOS) + import AVFoundation #if swift(>=5.9) @@ -23,67 +25,102 @@ internal import LiveKitWebRTC #endif public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { - var next: (any AudioEngineObserver)? - var isSessionActive = false + struct State { + var isSessionActive = false + var next: (any AudioEngineObserver)? + + // Used for backward compatibility with `customConfigureAudioSessionFunc`. + var isPlayoutEnabled: Bool = false + var isRecordingEnabled: Bool = false + } + + let _state = StateSync(State()) + + init() { + // Backward compatibility with `customConfigureAudioSessionFunc`. + _state.onDidMutate = { new_, old_ in + if let config_func = AudioManager.shared.state.customConfigureFunc, + new_.isPlayoutEnabled != old_.isPlayoutEnabled || + new_.isRecordingEnabled != old_.isRecordingEnabled + { + // Simulate state and invoke custom config func. + let old_state = AudioManager.State(localTracksCount: old_.isRecordingEnabled ? 1 : 0, remoteTracksCount: old_.isPlayoutEnabled ? 1 : 0) + let new_state = AudioManager.State(localTracksCount: new_.isRecordingEnabled ? 1 : 0, remoteTracksCount: new_.isPlayoutEnabled ? 1 : 0) + config_func(new_state, old_state) + } + } + } - public func setNext(_ handler: any AudioEngineObserver) { - next = handler + public func setNext(_ nextHandler: any AudioEngineObserver) { + _state.mutate { $0.next = nextHandler } } public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { - #if os(iOS) || os(visionOS) || os(tvOS) - log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { session.unlockForConfiguration() } - - let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback - do { - if isSessionActive { - log("AudioSession switching category to: \(config.category)") - try session.setConfiguration(config.toRTCType()) - } else { - log("AudioSession activating category to: \(config.category)") - try session.setConfiguration(config.toRTCType(), active: true) - isSessionActive = true + if AudioManager.shared.state.customConfigureFunc == nil { + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback + do { + if _state.isSessionActive { + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } else { + log("AudioSession activating category to: \(config.category)") + try session.setConfiguration(config.toRTCType(), active: true) + _state.mutate { $0.isSessionActive = true } + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) } - } catch { - log("AudioSession failed to configure with error: \(error)", .error) + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") } - log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") - #endif + _state.mutate { + $0.isPlayoutEnabled = isPlayoutEnabled + $0.isRecordingEnabled = isRecordingEnabled + } // Call next last - next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + _state.next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } public func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { // Call next first - next?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) - - #if os(iOS) || os(visionOS) || os(tvOS) - log("Configuring audio session...") - let session = LKRTCAudioSession.sharedInstance() - session.lockForConfiguration() - defer { session.unlockForConfiguration() } - - do { - if isPlayoutEnabled, !isRecordingEnabled { - let config: AudioSessionConfiguration = .playback - log("AudioSession switching category to: \(config.category)") - try session.setConfiguration(config.toRTCType()) - } - if !isPlayoutEnabled, !isRecordingEnabled { - log("AudioSession deactivating") - try session.setActive(false) - isSessionActive = false - } - } catch { - log("AudioSession failed to configure with error: \(error)", .error) + _state.next?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + + _state.mutate { + $0.isPlayoutEnabled = isPlayoutEnabled + $0.isRecordingEnabled = isRecordingEnabled } - log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") - #endif + if AudioManager.shared.state.customConfigureFunc == nil { + log("Configuring audio session...") + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + defer { session.unlockForConfiguration() } + + do { + if isPlayoutEnabled, !isRecordingEnabled { + let config: AudioSessionConfiguration = .playback + log("AudioSession switching category to: \(config.category)") + try session.setConfiguration(config.toRTCType()) + } + if !isPlayoutEnabled, !isRecordingEnabled { + log("AudioSession deactivating") + try session.setActive(false) + _state.mutate { $0.isSessionActive = false } + } + } catch { + log("AudioSession failed to configure with error: \(error)", .error) + } + + log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") + } } } + +#endif diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 60eec7467..b6833ad61 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -52,6 +52,7 @@ public class AudioManager: Loggable { /// - ``isSpeakerOutputPreferred`` /// /// If you want to revert to default behavior, set this to `nil`. + @available(*, deprecated, message: "Use `set(engineObservers:)` instead. See `DefaultAudioSessionObserver` for example.") public var customConfigureAudioSessionFunc: ConfigureAudioSessionFunc? { get { state.customConfigureFunc } set { state.mutate { $0.customConfigureFunc = newValue } } @@ -312,7 +313,12 @@ public class AudioManager: Loggable { let admDelegateAdapter = AudioDeviceModuleDelegateAdapter() init() { - state = StateSync(State(engineObservers: [DefaultAudioSessionObserver()])) + #if os(iOS) || os(visionOS) || os(tvOS) + let engineObservers: [any AudioEngineObserver] = [DefaultAudioSessionObserver()] + #else + let engineObservers: [any AudioEngineObserver] = [] + #endif + state = StateSync(State(engineObservers: engineObservers)) admDelegateAdapter.audioManager = self RTC.audioDeviceModule.observer = admDelegateAdapter } From 38c563b6074f8f4a03f17f587700391c7131f30a Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 29 Jan 2025 23:52:04 +0900 Subject: [PATCH 36/49] Strip unused code --- Sources/LiveKit/Track/AudioManager.swift | 120 ++---------------- .../LiveKit/Track/Local/LocalAudioTrack.swift | 10 -- .../Track/Remote/RemoteAudioTrack.swift | 10 -- 3 files changed, 12 insertions(+), 128 deletions(-) diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index b6833ad61..d1a3987e7 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -34,12 +34,13 @@ public class AudioManager: Loggable { public static let shared = AudioManager() #endif - public typealias DeviceUpdateFunc = (_ audioManager: AudioManager) -> Void + public typealias OnDevicesDidUpdate = (_ audioManager: AudioManager) -> Void - public typealias OnSpeechActivityEvent = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void + public typealias OnSpeechActivity = (_ audioManager: AudioManager, _ event: SpeechActivityEvent) -> Void #if os(iOS) || os(visionOS) || os(tvOS) + @available(*, deprecated) public typealias ConfigureAudioSessionFunc = @Sendable (_ newState: State, _ oldState: State) -> Void @@ -80,40 +81,26 @@ public class AudioManager: Loggable { get { state.sessionConfiguration } set { state.mutate { $0.sessionConfiguration = newValue } } } - #endif + @available(*, deprecated) public enum TrackState { case none case localOnly case remoteOnly case localAndRemote } + #endif - public struct State: Equatable, Sendable { - // Only consider State mutated when public vars change - public static func == (lhs: AudioManager.State, rhs: AudioManager.State) -> Bool { - var isEqual = lhs.localTracksCount == rhs.localTracksCount && - lhs.remoteTracksCount == rhs.remoteTracksCount - - #if os(iOS) || os(visionOS) || os(tvOS) - isEqual = isEqual && - lhs.isSpeakerOutputPreferred == rhs.isSpeakerOutputPreferred && - lhs.sessionConfiguration == rhs.sessionConfiguration - #endif - - return isEqual - } + public struct State: Sendable { + public var engineObservers = [any AudioEngineObserver]() + #if os(iOS) || os(visionOS) || os(tvOS) + // Keep this var within State so it's protected by UnfairLock public var localTracksCount: Int = 0 public var remoteTracksCount: Int = 0 public var isSpeakerOutputPreferred: Bool = true - #if os(iOS) || os(visionOS) || os(tvOS) - // Keep this var within State so it's protected by UnfairLock public var customConfigureFunc: ConfigureAudioSessionFunc? public var sessionConfiguration: AudioSessionConfiguration? - #endif - - public var engineObservers = [any AudioEngineObserver]() public var trackState: TrackState { switch (localTracksCount > 0, remoteTracksCount > 0) { @@ -123,6 +110,7 @@ public class AudioManager: Loggable { default: return .none } } + #endif } // MARK: - AudioProcessingModule @@ -185,12 +173,12 @@ public class AudioManager: Loggable { set { RTC.audioDeviceModule.inputDevice = newValue._ioDevice } } - public var onDeviceUpdate: DeviceUpdateFunc? + public var onDeviceUpdate: OnDevicesDidUpdate? /// Detect voice activity even if the mic is muted. /// Internal audio engine must be initialized by calling ``prepareRecording()`` or /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. - public var onMutedSpeechActivityEvent: OnSpeechActivityEvent? + public var onMutedSpeechActivityEvent: OnSpeechActivity? /// Enables advanced ducking which ducks other audio based on the presence of voice activity from local and remote chat participants. /// Default: true. @@ -303,11 +291,6 @@ public class AudioManager: Loggable { // MARK: - Internal - enum `Type` { - case local - case remote - } - let state: StateSync let admDelegateAdapter = AudioDeviceModuleDelegateAdapter() @@ -322,85 +305,6 @@ public class AudioManager: Loggable { admDelegateAdapter.audioManager = self RTC.audioDeviceModule.observer = admDelegateAdapter } - - // MARK: - Private - - func trackDidStart(_ type: Type) async throws { - state.mutate { state in - let oldState = state - if type == .local { state.localTracksCount += 1 } - if type == .remote { state.remoteTracksCount += 1 } - return (state, oldState) - } - } - - func trackDidStop(_ type: Type) async throws { - state.mutate { state in - let oldState = state - if type == .local { state.localTracksCount = max(state.localTracksCount - 1, 0) } - if type == .remote { state.remoteTracksCount = max(state.remoteTracksCount - 1, 0) } - return (state, oldState) - } - } - - #if os(iOS) || os(visionOS) || os(tvOS) - /// The default implementation when audio session configuration is requested by the SDK. - /// Configure the `RTCAudioSession` of `WebRTC` framework. - /// - /// > Note: It is recommended to use `RTCAudioSessionConfiguration.webRTC()` to obtain an instance of `RTCAudioSessionConfiguration` instead of instantiating directly. - /// - /// - Parameters: - /// - configuration: A configured RTCAudioSessionConfiguration - /// - setActive: passing true/false will call `AVAudioSession.setActive` internally - public func defaultConfigureAudioSessionFunc(newState: State, oldState: State) { - // Lazily computed config - let computeConfiguration: (() -> AudioSessionConfiguration) = { - switch newState.trackState { - case .none: - // Use .soloAmbient configuration - return .soloAmbient - case .remoteOnly where newState.isSpeakerOutputPreferred: - // Use .playback configuration with spoken audio - return .playback - default: - // Use .playAndRecord configuration - return newState.isSpeakerOutputPreferred ? .playAndRecordSpeaker : .playAndRecordReceiver - } - } - - let configuration = newState.sessionConfiguration ?? computeConfiguration() - - var setActive: Bool? - if newState.trackState != .none, oldState.trackState == .none { - // activate audio session when there is any local/remote audio track - setActive = true - } else if newState.trackState == .none, oldState.trackState != .none { - // deactivate audio session when there are no more local/remote audio tracks - setActive = false - } - - let session = LKRTCAudioSession.sharedInstance() - // Check if needs setConfiguration - guard configuration != session.toAudioSessionConfiguration() else { - log("Skipping configure audio session, no changes") - return - } - - session.lockForConfiguration() - defer { session.unlockForConfiguration() } - - do { - log("Configuring audio session: \(String(describing: configuration))") - if let setActive { - try session.setConfiguration(configuration.toRTCType(), active: setActive) - } else { - try session.setConfiguration(configuration.toRTCType()) - } - } catch { - log("Failed to configure audio session with error: \(error)", .error) - } - } - #endif } public extension AudioManager { diff --git a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift index 35eaae63c..5f64a5481 100644 --- a/Sources/LiveKit/Track/Local/LocalAudioTrack.swift +++ b/Sources/LiveKit/Track/Local/LocalAudioTrack.swift @@ -78,16 +78,6 @@ public class LocalAudioTrack: Track, LocalTrack, AudioTrack { public func unmute() async throws { try await super._unmute() } - - // MARK: - Internal - - override func startCapture() async throws { - try await AudioManager.shared.trackDidStart(.local) - } - - override func stopCapture() async throws { - try await AudioManager.shared.trackDidStop(.local) - } } public extension LocalAudioTrack { diff --git a/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift b/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift index 4b0776482..a62066e7c 100644 --- a/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift +++ b/Sources/LiveKit/Track/Remote/RemoteAudioTrack.swift @@ -75,14 +75,4 @@ public class RemoteAudioTrack: Track, RemoteTrack, AudioTrack { audioTrack.remove(_adapter) } } - - // MARK: - Internal - - override func startCapture() async throws { - try await AudioManager.shared.trackDidStart(.remote) - } - - override func stopCapture() async throws { - try await AudioManager.shared.trackDidStop(.remote) - } } From ce79ac62478c1b0e465d9748bdc117fe885c7aca Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 29 Jan 2025 23:57:32 +0900 Subject: [PATCH 37/49] Refactoring --- .../AudioDeviceModuleDelegateAdapter.swift | 16 ++++++------- .../Audio/DefaultAudioSessionObserver.swift | 6 ++--- Sources/LiveKit/Track/AudioManager.swift | 24 +++++++++---------- Tests/LiveKitTests/TrackTests.swift | 4 +--- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift index a93317c53..e2878ef38 100644 --- a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -40,49 +40,49 @@ class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate func audioDeviceModule(_: LKRTCAudioDeviceModule, didCreateEngine engine: AVAudioEngine) { guard let audioManager else { return } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() entryPoint?.engineDidCreate(engine) } func audioDeviceModule(_: LKRTCAudioDeviceModule, willEnableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() entryPoint?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, willStartEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() entryPoint?.engineWillStart(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, didStopEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() entryPoint?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, didDisableEngine engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { guard let audioManager else { return } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() entryPoint?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } func audioDeviceModule(_: LKRTCAudioDeviceModule, willReleaseEngine engine: AVAudioEngine) { guard let audioManager else { return } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() entryPoint?.engineWillRelease(engine) } func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureInputFromSource src: AVAudioNode?, toDestination dst: AVAudioNode, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() return entryPoint?.engineWillConnectInput(engine, src: src, dst: dst, format: format) ?? false } func audioDeviceModule(_: LKRTCAudioDeviceModule, engine: AVAudioEngine, configureOutputFromSource src: AVAudioNode, toDestination dst: AVAudioNode?, format: AVAudioFormat) -> Bool { guard let audioManager else { return false } - let entryPoint = audioManager.state.engineObservers.buildChain() + let entryPoint = audioManager._state.engineObservers.buildChain() return entryPoint?.engineWillConnectOutput(engine, src: src, dst: dst, format: format) ?? false } } diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift index 58e6fec54..930e7ffde 100644 --- a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -39,7 +39,7 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { init() { // Backward compatibility with `customConfigureAudioSessionFunc`. _state.onDidMutate = { new_, old_ in - if let config_func = AudioManager.shared.state.customConfigureFunc, + if let config_func = AudioManager.shared._state.customConfigureFunc, new_.isPlayoutEnabled != old_.isPlayoutEnabled || new_.isRecordingEnabled != old_.isRecordingEnabled { @@ -56,7 +56,7 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { } public func engineWillEnable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { - if AudioManager.shared.state.customConfigureFunc == nil { + if AudioManager.shared._state.customConfigureFunc == nil { log("Configuring audio session...") let session = LKRTCAudioSession.sharedInstance() session.lockForConfiguration() @@ -97,7 +97,7 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { $0.isRecordingEnabled = isRecordingEnabled } - if AudioManager.shared.state.customConfigureFunc == nil { + if AudioManager.shared._state.customConfigureFunc == nil { log("Configuring audio session...") let session = LKRTCAudioSession.sharedInstance() session.lockForConfiguration() diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index d1a3987e7..f1abce0a4 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -55,8 +55,8 @@ public class AudioManager: Loggable { /// If you want to revert to default behavior, set this to `nil`. @available(*, deprecated, message: "Use `set(engineObservers:)` instead. See `DefaultAudioSessionObserver` for example.") public var customConfigureAudioSessionFunc: ConfigureAudioSessionFunc? { - get { state.customConfigureFunc } - set { state.mutate { $0.customConfigureFunc = newValue } } + get { _state.customConfigureFunc } + set { _state.mutate { $0.customConfigureFunc = newValue } } } /// Determines whether the device's built-in speaker or receiver is preferred for audio output. @@ -67,8 +67,8 @@ public class AudioManager: Loggable { /// /// This property is ignored if ``customConfigureAudioSessionFunc`` is set. public var isSpeakerOutputPreferred: Bool { - get { state.isSpeakerOutputPreferred } - set { state.mutate { $0.isSpeakerOutputPreferred = newValue } } + get { _state.isSpeakerOutputPreferred } + set { _state.mutate { $0.isSpeakerOutputPreferred = newValue } } } /// Specifies a fixed configuration for the audio session, overriding dynamic adjustments. @@ -78,8 +78,8 @@ public class AudioManager: Loggable { /// /// This property is ignored if ``customConfigureAudioSessionFunc`` is set. public var sessionConfiguration: AudioSessionConfiguration? { - get { state.sessionConfiguration } - set { state.mutate { $0.sessionConfiguration = newValue } } + get { _state.sessionConfiguration } + set { _state.mutate { $0.sessionConfiguration = newValue } } } @available(*, deprecated) @@ -244,7 +244,7 @@ public class AudioManager: Loggable { /// /// Objects set here will be retained. public func set(engineObservers: [any AudioEngineObserver]) { - state.mutate { $0.engineObservers = engineObservers } + _state.mutate { $0.engineObservers = engineObservers } } // MARK: - For testing @@ -291,9 +291,9 @@ public class AudioManager: Loggable { // MARK: - Internal - let state: StateSync + let _state: StateSync - let admDelegateAdapter = AudioDeviceModuleDelegateAdapter() + let _admDelegateAdapter = AudioDeviceModuleDelegateAdapter() init() { #if os(iOS) || os(visionOS) || os(tvOS) @@ -301,9 +301,9 @@ public class AudioManager: Loggable { #else let engineObservers: [any AudioEngineObserver] = [] #endif - state = StateSync(State(engineObservers: engineObservers)) - admDelegateAdapter.audioManager = self - RTC.audioDeviceModule.observer = admDelegateAdapter + _state = StateSync(State(engineObservers: engineObservers)) + _admDelegateAdapter.audioManager = self + RTC.audioDeviceModule.observer = _admDelegateAdapter } } diff --git a/Tests/LiveKitTests/TrackTests.swift b/Tests/LiveKitTests/TrackTests.swift index 55bae6534..04559e6f7 100644 --- a/Tests/LiveKitTests/TrackTests.swift +++ b/Tests/LiveKitTests/TrackTests.swift @@ -29,12 +29,10 @@ class TestTrack: LocalAudioTrack { override func startCapture() async throws { try? await Task.sleep(nanoseconds: UInt64(Double.random(in: 0.0 ... 1.0) * 1_000_000)) - try await AudioManager.shared.trackDidStart(.local) } override func stopCapture() async throws { try? await Task.sleep(nanoseconds: UInt64(Double.random(in: 0.0 ... 1.0) * 1_000_000)) - try await AudioManager.shared.trackDidStop(.local) } } @@ -71,7 +69,7 @@ class TrackTests: XCTestCase { AudioManager.shared.customConfigureAudioSessionFunc = nil - XCTAssertEqual(AudioManager.shared.state.localTracksCount, 0, "localTracksCount should be 0") + XCTAssertEqual(AudioManager.shared._state.localTracksCount, 0, "localTracksCount should be 0") } #endif } From e57a5ad65fa4bd89cfbe4ec91162078854290472 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 00:53:17 +0900 Subject: [PATCH 38/49] Fix tests --- .../Support/SinWaveSourceNode.swift | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index 8913ac0a3..2389fde86 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -33,25 +33,30 @@ class SineWaveSourceNode: AVAudioSourceNode { let renderBlock: AVAudioSourceNodeRenderBlock = { _, _, frameCount, audioBufferList in print("AVAudioSourceNodeRenderBlock frameCount: \(frameCount)") - let ablPointer = UnsafeMutableAudioBufferListPointer(audioBufferList) + + // Accessing the AudioBufferList manually + let audioBuffers = audioBufferList.pointee + + // Assuming a single channel setup + guard audioBuffers.mNumberBuffers > 0 else { + return noErr + } + + let audioBuffer = audioBuffers.mBuffers // Access first buffer + guard let dataPointer = audioBuffer.mData?.assumingMemoryBound(to: Float.self) else { + return noErr + } + + let bufferPointer = UnsafeMutableBufferPointer(start: dataPointer, count: Int(frameCount)) // Generate sine wave samples - for frame in 0 ..< Int(frameCount) { - // Get the signal value for this frame at time. + for frame in 0 ..< bufferPointer.count { let value = sin(currentPhase) * amplitude - // Advance the phase for the next frame. currentPhase += phaseIncrement - if currentPhase >= twoPi { - currentPhase -= twoPi - } - if currentPhase < 0.0 { - currentPhase += twoPi - } - // Set the same value on all channels (due to the inputFormat, there's only one channel though). - for buffer in ablPointer { - let buf: UnsafeMutableBufferPointer = UnsafeMutableBufferPointer(buffer) - buf[frame] = value - } + if currentPhase >= twoPi { currentPhase -= twoPi } + if currentPhase < 0.0 { currentPhase += twoPi } + + bufferPointer[frame] = value } return noErr From 107e239a6e853762d70b0df2ec06e952443e0919 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 02:04:32 +0900 Subject: [PATCH 39/49] Rename onMutedSpeechActivity --- Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift | 2 +- Sources/LiveKit/Track/AudioManager.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift index e2878ef38..73e19d1d6 100644 --- a/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift +++ b/Sources/LiveKit/Audio/AudioDeviceModuleDelegateAdapter.swift @@ -28,7 +28,7 @@ class AudioDeviceModuleDelegateAdapter: NSObject, LKRTCAudioDeviceModuleDelegate func audioDeviceModule(_: LKRTCAudioDeviceModule, didReceiveSpeechActivityEvent speechActivityEvent: RTCSpeechActivityEvent) { guard let audioManager else { return } - audioManager.onMutedSpeechActivityEvent?(audioManager, speechActivityEvent.toLKType()) + audioManager.onMutedSpeechActivity?(audioManager, speechActivityEvent.toLKType()) } func audioDeviceModuleDidUpdateDevices(_: LKRTCAudioDeviceModule) { diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index f1abce0a4..5caa8fd75 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -178,7 +178,7 @@ public class AudioManager: Loggable { /// Detect voice activity even if the mic is muted. /// Internal audio engine must be initialized by calling ``prepareRecording()`` or /// connecting to a room and subscribing to a remote audio track or publishing a local audio track. - public var onMutedSpeechActivityEvent: OnSpeechActivity? + public var onMutedSpeechActivity: OnSpeechActivity? /// Enables advanced ducking which ducks other audio based on the presence of voice activity from local and remote chat participants. /// Default: true. From 6820281f8a357da101c0687fa60efd6497d0d101 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 04:15:45 +0900 Subject: [PATCH 40/49] Squashed commit of the following: commit facb3ce1fe5e794ef1037d4a537ca3738d6644c2 Author: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu Jan 23 23:20:43 2025 +0900 Move debug string commit 81afdd11b4b4f013c1f5957b1c784d21cc4181fc Author: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu Jan 23 23:12:19 2025 +0900 Revert "temp use local lib" This reverts commit b1b79d22712df7b44ecceaeb7b54bb2c39fcb7db. commit c978bd31c6673f273b5f0a3f55645e6e114c5f9d Author: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu Jan 23 23:12:04 2025 +0900 Tests commit b1b79d22712df7b44ecceaeb7b54bb2c39fcb7db Author: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu Jan 23 17:15:23 2025 +0900 temp use local lib --- .../Extensions/CustomStringConvertible.swift | 11 ++++ Tests/LiveKitTests/AudioProcessingTests.swift | 59 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/Sources/LiveKit/Extensions/CustomStringConvertible.swift b/Sources/LiveKit/Extensions/CustomStringConvertible.swift index 79b09f409..b34d1dabd 100644 --- a/Sources/LiveKit/Extensions/CustomStringConvertible.swift +++ b/Sources/LiveKit/Extensions/CustomStringConvertible.swift @@ -178,3 +178,14 @@ extension AVCaptureDevice.Format { return "Format(\(values.joined(separator: ", ")))" } } + +extension LKRTCAudioProcessingConfig { + func toDebugString() -> String { + "RTCAudioProcessingConfig(" + + "isEchoCancellationEnabled: \(isEchoCancellationEnabled), " + + "isNoiseSuppressionEnabled: \(isNoiseSuppressionEnabled), " + + "isAutoGainControl1Enabled: \(isAutoGainControl1Enabled), " + + "isHighpassFilterEnabled: \(isHighpassFilterEnabled)" + + ")" + } +} diff --git a/Tests/LiveKitTests/AudioProcessingTests.swift b/Tests/LiveKitTests/AudioProcessingTests.swift index 0078552aa..c6fa0c1c9 100644 --- a/Tests/LiveKitTests/AudioProcessingTests.swift +++ b/Tests/LiveKitTests/AudioProcessingTests.swift @@ -17,6 +17,7 @@ import Accelerate import AVFoundation import Foundation +import LiveKitWebRTC @testable import LiveKit import XCTest @@ -61,4 +62,62 @@ class AudioProcessingTests: XCTestCase, AudioCustomProcessingDelegate { try await Task.sleep(nanoseconds: ns) } } + + func testOptionsAppliedToAudioProcessingModule() async throws { + try await withRooms([RoomTestingOptions(canPublish: true)]) { rooms in + // Alias to Room1 + let room1 = rooms[0] + + let allOnOptions = AudioCaptureOptions( + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + highpassFilter: true + ) + + let allOffOptions = AudioCaptureOptions( + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + highpassFilter: false + ) + + let pub1 = try await room1.localParticipant.setMicrophone(enabled: true, captureOptions: allOnOptions) + guard let pub1 else { + XCTFail("Publication is nil") + return + } + + let ns = UInt64(3 * 1_000_000_000) + try await Task.sleep(nanoseconds: ns) + + // Directly read config from the apm + let allOnConfigResult = RTC.audioProcessingModule.config + print("Config result for all on: \(allOnConfigResult.toDebugString()))") + XCTAssert(allOnConfigResult.isEchoCancellationEnabled) + XCTAssert(allOnConfigResult.isNoiseSuppressionEnabled) + XCTAssert(allOnConfigResult.isAutoGainControl1Enabled) + XCTAssert(allOnConfigResult.isHighpassFilterEnabled) + + try await room1.localParticipant.unpublish(publication: pub1) + + let pub2 = try await room1.localParticipant.setMicrophone(enabled: true, captureOptions: allOffOptions) + guard let pub2 else { + XCTFail("Publication is nil") + return + } + + try await Task.sleep(nanoseconds: ns) + + // Directly read config from the apm + let allOffConfigResult = RTC.audioProcessingModule.config + print("Config result for all off: \(allOffConfigResult.toDebugString())") + XCTAssert(!allOffConfigResult.isEchoCancellationEnabled) + XCTAssert(!allOffConfigResult.isNoiseSuppressionEnabled) + XCTAssert(!allOffConfigResult.isAutoGainControl1Enabled) + XCTAssert(!allOffConfigResult.isHighpassFilterEnabled) + + try await room1.localParticipant.unpublish(publication: pub2) + } + } } From 77c7cfe809712c971a542fa9f209270986760d79 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 04:25:15 +0900 Subject: [PATCH 41/49] Use 125.6422.13 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 478c4c453..c94142cde 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.13"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 866554d04..5ba012bca 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.12"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.13"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From d44a090d88677d96395a839409667a6299c14a78 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 04:13:42 +0900 Subject: [PATCH 42/49] Disable RTC audio options by default --- .../Types/Options/AudioCaptureOptions.swift | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift index af5ad9999..3588f16fe 100644 --- a/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/AudioCaptureOptions.swift @@ -26,32 +26,38 @@ internal import LiveKitWebRTC public final class AudioCaptureOptions: NSObject, CaptureOptions, Sendable { public static let noProcessing = AudioCaptureOptions( echoCancellation: false, - noiseSuppression: false, autoGainControl: false, - typingNoiseDetection: false, - highpassFilter: false + noiseSuppression: false, + highpassFilter: false, + typingNoiseDetection: false ) + /// Whether to enable software (WebRTC's) echo cancellation. + /// By default, Apple's voice processing is already enabled. + /// See ``AudioManager/isVoiceProcessingBypassed`` for details. @objc public let echoCancellation: Bool - @objc - public let noiseSuppression: Bool - + /// Whether to enable software (WebRTC's) gain control. + /// By default, Apple's gain control is already enabled. + /// See ``AudioManager/isVoiceProcessingAGCEnabled`` for details. @objc public let autoGainControl: Bool @objc - public let typingNoiseDetection: Bool + public let noiseSuppression: Bool @objc public let highpassFilter: Bool - public init(echoCancellation: Bool = true, - noiseSuppression: Bool = true, - autoGainControl: Bool = true, - typingNoiseDetection: Bool = true, - highpassFilter: Bool = true) + @objc + public let typingNoiseDetection: Bool + + public init(echoCancellation: Bool = false, + autoGainControl: Bool = false, + noiseSuppression: Bool = false, + highpassFilter: Bool = false, + typingNoiseDetection: Bool = false) { self.echoCancellation = echoCancellation self.noiseSuppression = noiseSuppression From 3c6d6b572d29230e3045bd4733dfaa43ac34c59f Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 04:36:07 +0900 Subject: [PATCH 43/49] Fix apm tests --- Tests/LiveKitTests/AudioProcessingTests.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Tests/LiveKitTests/AudioProcessingTests.swift b/Tests/LiveKitTests/AudioProcessingTests.swift index c6fa0c1c9..62c03911d 100644 --- a/Tests/LiveKitTests/AudioProcessingTests.swift +++ b/Tests/LiveKitTests/AudioProcessingTests.swift @@ -64,21 +64,24 @@ class AudioProcessingTests: XCTestCase, AudioCustomProcessingDelegate { } func testOptionsAppliedToAudioProcessingModule() async throws { + // Disable Apple VPIO. + AudioManager.shared.isVoiceProcessingBypassed = true + try await withRooms([RoomTestingOptions(canPublish: true)]) { rooms in // Alias to Room1 let room1 = rooms[0] let allOnOptions = AudioCaptureOptions( echoCancellation: true, - noiseSuppression: true, autoGainControl: true, + noiseSuppression: true, highpassFilter: true ) let allOffOptions = AudioCaptureOptions( echoCancellation: false, - noiseSuppression: false, autoGainControl: false, + noiseSuppression: false, highpassFilter: false ) From 75ed189257655ef9517875fc4726f6a7eef94f96 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:08:42 +0900 Subject: [PATCH 44/49] config tests --- .../Audio/DefaultAudioSessionObserver.swift | 10 ++++++++ Tests/LiveKitTests/AudioEngineTests.swift | 23 +++++++++++++++++++ .../Support/SinWaveSourceNode.swift | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift index 930e7ffde..5a3a42020 100644 --- a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -121,6 +121,16 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") } } + + public func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + _state.mutate { + $0.isPlayoutEnabled = isPlayoutEnabled + $0.isRecordingEnabled = isRecordingEnabled + } + + // Call next last + _state.next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + } } #endif diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 5607e4369..52f09fc80 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -237,6 +237,29 @@ class AudioEngineTests: XCTestCase { try? await Task.sleep(nanoseconds: 1 * 100_000_000) // 10ms } } + + func testBackwardCompatibility() async throws { + // Don't use device for this test + AudioManager.shared.isManualRenderingMode = true + + var trackState: AudioManager.TrackState = .none + AudioManager.shared.customConfigureAudioSessionFunc = { newState, oldState in + print("New trackState: \(newState.trackState), Old trackState: \(oldState.trackState)") + trackState = newState.trackState + } + + XCTAssert(trackState == .none) + + AudioManager.shared.initPlayout() + XCTAssert(trackState == .remoteOnly) + + AudioManager.shared.initRecording() + XCTAssert(trackState == .localAndRemote) + + AudioManager.shared.stopPlayout() + AudioManager.shared.stopRecording() + XCTAssert(trackState == .none) + } } final class SineWaveNodeHook: AudioEngineObserver { diff --git a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift index 2389fde86..258a720b3 100644 --- a/Tests/LiveKitTests/Support/SinWaveSourceNode.swift +++ b/Tests/LiveKitTests/Support/SinWaveSourceNode.swift @@ -16,7 +16,7 @@ import AVFAudio -class SineWaveSourceNode: AVAudioSourceNode { +class SineWaveSourceNode: AVAudioSourceNode, @unchecked Sendable { private let sampleRate: Double private let frequency: Double From 58318ea11061675f7f3d522486c6bcec0e85d9b9 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:21:12 +0900 Subject: [PATCH 45/49] Update default config logic --- .../Audio/DefaultAudioSessionObserver.swift | 27 +++++++------------ Tests/LiveKitTests/AudioEngineTests.swift | 27 ++++++++++++++----- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift index 5a3a42020..959ef429f 100644 --- a/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift +++ b/Sources/LiveKit/Audio/DefaultAudioSessionObserver.swift @@ -65,13 +65,14 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { let config: AudioSessionConfiguration = isRecordingEnabled ? .playAndRecordSpeaker : .playback do { if _state.isSessionActive { - log("AudioSession switching category to: \(config.category)") - try session.setConfiguration(config.toRTCType()) - } else { - log("AudioSession activating category to: \(config.category)") - try session.setConfiguration(config.toRTCType(), active: true) - _state.mutate { $0.isSessionActive = true } + log("AudioSession deactivating due to category switch") + try session.setActive(false) // Deactivate first + _state.mutate { $0.isSessionActive = false } } + + log("AudioSession activating category to: \(config.category)") + try session.setConfiguration(config.toRTCType(), active: true) + _state.mutate { $0.isSessionActive = true } } catch { log("AudioSession failed to configure with error: \(error)", .error) } @@ -88,9 +89,9 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { _state.next?.engineWillEnable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) } - public func engineDidStop(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { + public func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { // Call next first - _state.next?.engineDidStop(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) + _state.next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) _state.mutate { $0.isPlayoutEnabled = isPlayoutEnabled @@ -121,16 +122,6 @@ public final class DefaultAudioSessionObserver: AudioEngineObserver, Loggable { log("AudioSession activationCount: \(session.activationCount), webRTCSessionCount: \(session.webRTCSessionCount)") } } - - public func engineDidDisable(_ engine: AVAudioEngine, isPlayoutEnabled: Bool, isRecordingEnabled: Bool) { - _state.mutate { - $0.isPlayoutEnabled = isPlayoutEnabled - $0.isRecordingEnabled = isRecordingEnabled - } - - // Call next last - _state.next?.engineDidDisable(engine, isPlayoutEnabled: isPlayoutEnabled, isRecordingEnabled: isRecordingEnabled) - } } #endif diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 52f09fc80..46059ef93 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -238,28 +238,43 @@ class AudioEngineTests: XCTestCase { } } + #if os(iOS) || os(visionOS) || os(tvOS) func testBackwardCompatibility() async throws { - // Don't use device for this test - AudioManager.shared.isManualRenderingMode = true - var trackState: AudioManager.TrackState = .none AudioManager.shared.customConfigureAudioSessionFunc = { newState, oldState in print("New trackState: \(newState.trackState), Old trackState: \(oldState.trackState)") trackState = newState.trackState } - + + // Configure session since we are setting a empty config func. + let config = AudioSessionConfiguration.playAndRecordSpeaker + let session = LKRTCAudioSession.sharedInstance() + session.lockForConfiguration() + try session.setConfiguration(config.toRTCType(), active: true) + session.unlockForConfiguration() + XCTAssert(trackState == .none) - + AudioManager.shared.initPlayout() XCTAssert(trackState == .remoteOnly) AudioManager.shared.initRecording() XCTAssert(trackState == .localAndRemote) - AudioManager.shared.stopPlayout() AudioManager.shared.stopRecording() + XCTAssert(trackState == .remoteOnly) + + AudioManager.shared.stopPlayout() XCTAssert(trackState == .none) } + + func testDefaultAudioSessionConfiguration() async throws { + AudioManager.shared.initPlayout() + AudioManager.shared.initRecording() + AudioManager.shared.stopRecording() + AudioManager.shared.stopPlayout() + } + #endif } final class SineWaveNodeHook: AudioEngineObserver { From 3a595db1e475982e01b5748ef86a79a289f4b273 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:22:03 +0900 Subject: [PATCH 46/49] Use 125.6422.14 --- Package.swift | 2 +- Package@swift-5.9.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index c94142cde..bdabfd3ba 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.13"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.14"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift index 5ba012bca..8ad03927e 100644 --- a/Package@swift-5.9.swift +++ b/Package@swift-5.9.swift @@ -20,7 +20,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.13"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "125.6422.14"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation From c2441e6c691a63ba279628539f06a93bbea7f971 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 30 Jan 2025 19:42:29 +0900 Subject: [PATCH 47/49] Update tests --- Tests/LiveKitTests/AudioEngineTests.swift | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Tests/LiveKitTests/AudioEngineTests.swift b/Tests/LiveKitTests/AudioEngineTests.swift index 46059ef93..c481d304f 100644 --- a/Tests/LiveKitTests/AudioEngineTests.swift +++ b/Tests/LiveKitTests/AudioEngineTests.swift @@ -240,10 +240,14 @@ class AudioEngineTests: XCTestCase { #if os(iOS) || os(visionOS) || os(tvOS) func testBackwardCompatibility() async throws { - var trackState: AudioManager.TrackState = .none + struct TestState { + var trackState: AudioManager.TrackState = .none + } + let _testState = StateSync(TestState()) + AudioManager.shared.customConfigureAudioSessionFunc = { newState, oldState in print("New trackState: \(newState.trackState), Old trackState: \(oldState.trackState)") - trackState = newState.trackState + _testState.mutate { $0.trackState = newState.trackState } } // Configure session since we are setting a empty config func. @@ -253,19 +257,19 @@ class AudioEngineTests: XCTestCase { try session.setConfiguration(config.toRTCType(), active: true) session.unlockForConfiguration() - XCTAssert(trackState == .none) + XCTAssert(_testState.trackState == .none) AudioManager.shared.initPlayout() - XCTAssert(trackState == .remoteOnly) + XCTAssert(_testState.trackState == .remoteOnly) AudioManager.shared.initRecording() - XCTAssert(trackState == .localAndRemote) + XCTAssert(_testState.trackState == .localAndRemote) AudioManager.shared.stopRecording() - XCTAssert(trackState == .remoteOnly) + XCTAssert(_testState.trackState == .remoteOnly) AudioManager.shared.stopPlayout() - XCTAssert(trackState == .none) + XCTAssert(_testState.trackState == .none) } func testDefaultAudioSessionConfiguration() async throws { From 18056e75c7a1820bbe79bea89f96652e99150c0b Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Sun, 2 Feb 2025 18:40:39 +0900 Subject: [PATCH 48/49] Ports --- Sources/LiveKit/Audio/AudioPortManager.swift | 36 ++++++++++++ ....swift => DefaultAudioInputObserver.swift} | 57 +++++++++++++++---- Sources/LiveKit/Track/AudioManager.swift | 8 ++- .../Track/Capturers/MacOSScreenCapturer.swift | 14 ++--- 4 files changed, 95 insertions(+), 20 deletions(-) create mode 100644 Sources/LiveKit/Audio/AudioPortManager.swift rename Sources/LiveKit/Audio/{AudioEngineAudioInputObserver.swift => DefaultAudioInputObserver.swift} (53%) diff --git a/Sources/LiveKit/Audio/AudioPortManager.swift b/Sources/LiveKit/Audio/AudioPortManager.swift new file mode 100644 index 000000000..65ecb52e8 --- /dev/null +++ b/Sources/LiveKit/Audio/AudioPortManager.swift @@ -0,0 +1,36 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFAudio + +public class AudioPortManager { + struct State { + var ports: [String: AVAudioPlayerNode] = [:] + } + + let _state = StateSync(State()) + + func node(for portId: String) -> AVAudioPlayerNode { + _state.mutate { + if let r = $0.ports[portId] { + return r + } + let r = AVAudioPlayerNode() + $0.ports[portId] = r + return r + } + } +} diff --git a/Sources/LiveKit/Audio/AudioEngineAudioInputObserver.swift b/Sources/LiveKit/Audio/DefaultAudioInputObserver.swift similarity index 53% rename from Sources/LiveKit/Audio/AudioEngineAudioInputObserver.swift rename to Sources/LiveKit/Audio/DefaultAudioInputObserver.swift index 1ec278fea..958f68b1f 100644 --- a/Sources/LiveKit/Audio/AudioEngineAudioInputObserver.swift +++ b/Sources/LiveKit/Audio/DefaultAudioInputObserver.swift @@ -14,7 +14,7 @@ * limitations under the License. */ -import AVFoundation +@preconcurrency import AVFoundation #if swift(>=5.9) internal import LiveKitWebRTC @@ -22,47 +22,82 @@ internal import LiveKitWebRTC @_implementationOnly import LiveKitWebRTC #endif -public final class AudioEngineAudioInputObserver: AudioEngineObserver, Loggable { - public let playerNode = AVAudioPlayerNode() - public let playerMixerNode = AVAudioMixerNode() - public let micMixerNode = AVAudioMixerNode() +enum AudioPort: Sendable { + case defaultInput + case custom(AVAudioPlayerNode) +} +public final class DefaultAudioInputObserver: AudioEngineObserver, Loggable { // let playerNodeFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48000, channels: 2, interleaved: false) - var next: (any AudioEngineObserver)? + struct State { + var next: (any AudioEngineObserver)? + public let playerNode: AVAudioPlayerNode + public let playerMixerNode = AVAudioMixerNode() + public let micMixerNode = AVAudioMixerNode() + } + + let _state: StateSync - public init() {} + public var next: (any AudioEngineObserver)? { + get { _state.next } + set { _state.mutate { $0.next = newValue } } + } + + public init(playerNode: AVAudioPlayerNode) { + _state = StateSync(State(playerNode: playerNode)) + } public func setNext(_ handler: any AudioEngineObserver) { next = handler } public func engineDidCreate(_ engine: AVAudioEngine) { + let (playerNode, playerMixerNode, micMixerNode) = _state.read { + ($0.playerNode, $0.playerMixerNode, $0.micMixerNode) + } + engine.attach(playerNode) engine.attach(playerMixerNode) engine.attach(micMixerNode) micMixerNode.outputVolume = 0.0 + + // Invoke next + next?.engineDidCreate(engine) } public func engineWillRelease(_ engine: AVAudioEngine) { + // Invoke next + next?.engineWillRelease(engine) + + let (playerNode, playerMixerNode, micMixerNode) = _state.read { + ($0.playerNode, $0.playerMixerNode, $0.micMixerNode) + } + engine.detach(playerNode) engine.detach(playerMixerNode) engine.detach(micMixerNode) } - public func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + public func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool { + let (playerNode, playerMixerNode, micMixerNode) = _state.read { + ($0.playerNode, $0.playerMixerNode, $0.micMixerNode) + } + // inputPlayer -> playerMixer -> mainMixer engine.connect(playerNode, to: playerMixerNode, format: playerNodeFormat) engine.connect(playerMixerNode, to: dst, format: format) - // mic -> micMixer -> mainMixer - engine.connect(src, to: micMixerNode, format: format) - engine.connect(micMixerNode, to: dst, format: format) + if let src { + // mic -> micMixer -> mainMixer + engine.connect(src, to: micMixerNode, format: format) + engine.connect(micMixerNode, to: dst, format: format) + } return true } diff --git a/Sources/LiveKit/Track/AudioManager.swift b/Sources/LiveKit/Track/AudioManager.swift index 3d40db2df..baae1284d 100644 --- a/Sources/LiveKit/Track/AudioManager.swift +++ b/Sources/LiveKit/Track/AudioManager.swift @@ -255,6 +255,8 @@ public class AudioManager: Loggable { _state.mutate { $0.engineObservers = engineObservers } } + public let ports = AudioPortManager() + // MARK: - For testing var isPlayoutInitialized: Bool { @@ -305,9 +307,11 @@ public class AudioManager: Loggable { init() { #if os(iOS) || os(visionOS) || os(tvOS) - let engineObservers: [any AudioEngineObserver] = [DefaultAudioSessionObserver()] + let playerNode = ports.node(for: "default") + let engineObservers: [any AudioEngineObserver] = [DefaultAudioSessionObserver(), DefaultAudioInputObserver(playerNode: playerNode)] #else - let engineObservers: [any AudioEngineObserver] = [] + let playerNode = ports.node(for: "default") + let engineObservers: [any AudioEngineObserver] = [DefaultAudioInputObserver(playerNode: playerNode)] #endif _state = StateSync(State(engineObservers: engineObservers)) _admDelegateAdapter.audioManager = self diff --git a/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift b/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift index 994b60d7d..763f030d3 100644 --- a/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift +++ b/Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift @@ -218,13 +218,13 @@ extension MacOSScreenCapturer: SCStreamOutput { guard sampleBuffer.isValid else { return } if case .audio = outputType { - if let pcm = sampleBuffer.toAVAudioPCMBuffer(), - let node = AudioManager.shared.screenShareAppAudioPlayerNode, - let engine = node.engine, engine.isRunning - { - node.scheduleBuffer(pcm) - if !node.isPlaying { - node.play() + if let pcm = sampleBuffer.toAVAudioPCMBuffer() { + let node = AudioManager.shared.ports.node(for: "default") + if let engine = node.engine, engine.isRunning { + node.scheduleBuffer(pcm) + if !node.isPlaying { + node.play() + } } } } else if case .screen = outputType { From d690238c1ee0c958ec87f2b91ef2199d6f869d35 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 4 Feb 2025 17:03:37 +0900 Subject: [PATCH 49/49] Fix --- Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift b/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift index 89a143768..32a0e1f57 100644 --- a/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift +++ b/Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift @@ -52,7 +52,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen public init(dimensions: Dimensions = .h1080_169, fps: Int = 30, showCursor: Bool = true, - appAudio: Bool = true) + appAudio: Bool = true, useBroadcastExtension: Bool = defaultToBroadcastExtension, includeCurrentApplication: Bool = false) {