macOS screen share audio

Sources/LiveKit/Audio/AudioPortManager.swift

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import AVFAudio
+
+public class AudioPortManager {
+    struct State {
+        var ports: [String: AVAudioPlayerNode] = [:]
+    }
+
+    let _state = StateSync(State())
+
+    func node(for portId: String) -> AVAudioPlayerNode {
+        _state.mutate {
+            if let r = $0.ports[portId] {
+                return r
+            }
+            let r = AVAudioPlayerNode()
+            $0.ports[portId] = r
+            return r
+        }
+    }
+}

Sources/LiveKit/Audio/DefaultAudioInputObserver.swift

@@ -0,0 +1,104 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@preconcurrency import AVFoundation
+
+#if swift(>=5.9)
+internal import LiveKitWebRTC
+#else
+@_implementationOnly import LiveKitWebRTC
+#endif
+
+enum AudioPort: Sendable {
+    case defaultInput
+    case custom(AVAudioPlayerNode)
+}
+
+public final class DefaultAudioInputObserver: AudioEngineObserver, Loggable {
+    // <AVAudioFormat 0x600003055180:  2 ch,  48000 Hz, Float32, deinterleaved>
+    let playerNodeFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32,
+                                         sampleRate: 48000,
+                                         channels: 2,
+                                         interleaved: false)
+
+    struct State {
+        var next: (any AudioEngineObserver)?
+        public let playerNode: AVAudioPlayerNode
+        public let playerMixerNode = AVAudioMixerNode()
+        public let micMixerNode = AVAudioMixerNode()
+    }
+
+    let _state: StateSync<State>
+
+    public var next: (any AudioEngineObserver)? {
+        get { _state.next }
+        set { _state.mutate { $0.next = newValue } }
+    }
+
+    public init(playerNode: AVAudioPlayerNode) {
+        _state = StateSync(State(playerNode: playerNode))
+    }
+
+    public func setNext(_ handler: any AudioEngineObserver) {
+        next = handler
+    }
+
+    public func engineDidCreate(_ engine: AVAudioEngine) {
+        let (playerNode, playerMixerNode, micMixerNode) = _state.read {
+            ($0.playerNode, $0.playerMixerNode, $0.micMixerNode)
+        }
+
+        engine.attach(playerNode)
+        engine.attach(playerMixerNode)
+        engine.attach(micMixerNode)
+
+        micMixerNode.outputVolume = 0.0
+
+        // Invoke next
+        next?.engineDidCreate(engine)
+    }
+
+    public func engineWillRelease(_ engine: AVAudioEngine) {
+        // Invoke next
+        next?.engineWillRelease(engine)
+
+        let (playerNode, playerMixerNode, micMixerNode) = _state.read {
+            ($0.playerNode, $0.playerMixerNode, $0.micMixerNode)
+        }
+
+        engine.detach(playerNode)
+        engine.detach(playerMixerNode)
+        engine.detach(micMixerNode)
+    }
+
+    public func engineWillConnectInput(_ engine: AVAudioEngine, src: AVAudioNode?, dst: AVAudioNode, format: AVAudioFormat) -> Bool {
+        let (playerNode, playerMixerNode, micMixerNode) = _state.read {
+            ($0.playerNode, $0.playerMixerNode, $0.micMixerNode)
+        }
+
+        // inputPlayer -> playerMixer -> mainMixer
+        engine.connect(playerNode, to: playerMixerNode, format: playerNodeFormat)
+        engine.connect(playerMixerNode, to: dst, format: format)
+
+        if let src {
+            // mic -> micMixer -> mainMixer
+            engine.connect(src, to: micMixerNode, format: format)
+            engine.connect(micMixerNode, to: dst, format: format)
+        }
+
+        return true
+    }
+}

Sources/LiveKit/Convenience/AudioProcessing.swift

@@ -56,6 +56,17 @@ public extension LKAudioBuffer {
     }
 }
 
+public extension CMSampleBuffer {
+    func toAVAudioPCMBuffer() -> AVAudioPCMBuffer? {
+        let format = AVAudioFormat(cmAudioFormatDescription: formatDescription!)
+        let numSamples = AVAudioFrameCount(numSamples)
+        let pcmBuffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: numSamples)!
+        pcmBuffer.frameLength = numSamples
+        CMSampleBufferCopyPCMDataIntoAudioBufferList(self, at: 0, frameCount: Int32(numSamples), into: pcmBuffer.mutableAudioBufferList)
+        return pcmBuffer
+    }
+}
+
 public extension AVAudioPCMBuffer {
     /// Computes Peak and Linear Scale RMS Value (Average) for all channels.
     func audioLevels() -> [AudioLevel] {

Sources/LiveKit/Track/AudioManager.swift

@@ -101,10 +101,10 @@
         public var localTracksCount: Int = 0
         public var remoteTracksCount: Int = 0
         public var isSpeakerOutputPreferred: Bool = true
         public var customConfigureFunc: ConfigureAudioSessionFunc?
         public var sessionConfiguration: AudioSessionConfiguration?
 
         public var trackState: TrackState {
             switch (localTracksCount > 0, remoteTracksCount > 0) {
             case (true, false): return .localOnly
             case (false, true): return .remoteOnly
@@ -255,6 +255,8 @@
         _state.mutate { $0.engineObservers = engineObservers }
     }
 
+    public let ports = AudioPortManager()
+
     // MARK: - For testing
 
     var isPlayoutInitialized: Bool {
@@ -305,9 +307,11 @@
 
     init() {
         #if os(iOS) || os(visionOS) || os(tvOS)
-        let engineObservers: [any AudioEngineObserver] = [DefaultAudioSessionObserver()]
+        let playerNode = ports.node(for: "default")
+        let engineObservers: [any AudioEngineObserver] = [DefaultAudioSessionObserver(), DefaultAudioInputObserver(playerNode: playerNode)]
         #else
-        let engineObservers: [any AudioEngineObserver] = []
+        let playerNode = ports.node(for: "default")
+        let engineObservers: [any AudioEngineObserver] = [DefaultAudioInputObserver(playerNode: playerNode)]
         #endif
         _state = StateSync(State(engineObservers: engineObservers))
         _admDelegateAdapter.audioManager = self

Sources/LiveKit/Track/Capturers/MacOSScreenCapturer.swift

@@ -98,9 +98,16 @@ public class MacOSScreenCapturer: VideoCapturer {
         configuration.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
         configuration.showsCursor = options.showCursor
 
+        if #available(macOS 13.0, *) {
+            configuration.capturesAudio = options.appAudio
+        }
+
         // Why does SCStream hold strong reference to delegate?
         let stream = SCStream(filter: filter, configuration: configuration, delegate: nil)
         try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: nil)
+        if #available(macOS 13.0, *) {
+            try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: nil)
+        }
         try await stream.startCapture()
 
         _screenCapturerState.mutate { $0.scStream = stream }
@@ -200,7 +207,6 @@ extension MacOSScreenCapturer {
 @available(macOS 12.3, *)
 extension MacOSScreenCapturer: SCStreamOutput {
     public func stream(_: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer,
-
                        of outputType: SCStreamOutputType)
     {
         guard case .started = captureState else {
@@ -211,40 +217,50 @@ extension MacOSScreenCapturer: SCStreamOutput {
         // Return early if the sample buffer is invalid.
         guard sampleBuffer.isValid else { return }
 
-        guard case .screen = outputType else { return }
-
-        // Retrieve the array of metadata attachments from the sample buffer.
-        guard let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer,
-                                                                             createIfNecessary: false) as? [[SCStreamFrameInfo: Any]],
-            let attachments = attachmentsArray.first else { return }
-
-        // Validate the status of the frame. If it isn't `.complete`, return nil.
-        guard let statusRawValue = attachments[SCStreamFrameInfo.status] as? Int,
-              let status = SCFrameStatus(rawValue: statusRawValue),
-              status == .complete else { return }
-
-        // Retrieve the content rectangle, scale, and scale factor.
-        guard let contentRectDict = attachments[.contentRect],
-              let contentRect = CGRect(dictionaryRepresentation: contentRectDict as! CFDictionary),
-              // let contentScale = attachments[.contentScale] as? CGFloat,
-              let scaleFactor = attachments[.scaleFactor] as? CGFloat else { return }
-
-        // Schedule resend timer
-        let newTimer = Task.detached(priority: .utility) { [weak self] in
-            while true {
-                try? await Task.sleep(nanoseconds: UInt64(1 * 1_000_000_000))
-                if Task.isCancelled { break }
-                guard let self else { break }
-                try await self._capturePreviousFrame()
+        if case .audio = outputType {
+            if let pcm = sampleBuffer.toAVAudioPCMBuffer() {
+                let node = AudioManager.shared.ports.node(for: "default")
+                if let engine = node.engine, engine.isRunning {
+                    node.scheduleBuffer(pcm)
+                    if !node.isPlaying {
+                        node.play()
+                    }
+                }
+            }
+        } else if case .screen = outputType {
+            // Retrieve the array of metadata attachments from the sample buffer.
+            guard let attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer,
+                                                                                 createIfNecessary: false) as? [[SCStreamFrameInfo: Any]],
+                let attachments = attachmentsArray.first else { return }
+
+            // Validate the status of the frame. If it isn't `.complete`, return nil.
+            guard let statusRawValue = attachments[SCStreamFrameInfo.status] as? Int,
+                  let status = SCFrameStatus(rawValue: statusRawValue),
+                  status == .complete else { return }
+
+            // Retrieve the content rectangle, scale, and scale factor.
+            guard let contentRectDict = attachments[.contentRect],
+                  let contentRect = CGRect(dictionaryRepresentation: contentRectDict as! CFDictionary),
+                  // let contentScale = attachments[.contentScale] as? CGFloat,
+                  let scaleFactor = attachments[.scaleFactor] as? CGFloat else { return }
+
+            // Schedule resend timer
+            let newTimer = Task.detached(priority: .utility) { [weak self] in
+                while true {
+                    try? await Task.sleep(nanoseconds: UInt64(1 * 1_000_000_000))
+                    if Task.isCancelled { break }
+                    guard let self else { break }
+                    try await self._capturePreviousFrame()
+                }
             }
-        }
 
-        _screenCapturerState.mutate {
-            $0.resendTimer?.cancel()
-            $0.resendTimer = newTimer
-        }
+            _screenCapturerState.mutate {
+                $0.resendTimer?.cancel()
+                $0.resendTimer = newTimer
+            }
 
-        capture(sampleBuffer, contentRect: contentRect, scaleFactor: scaleFactor)
+            capture(sampleBuffer, contentRect: contentRect, scaleFactor: scaleFactor)
+        }
     }
 }
 

Sources/LiveKit/Types/Options/ScreenShareCaptureOptions.swift

@@ -28,6 +28,9 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen
     @objc
     public let showCursor: Bool
 
+    @objc
+    public let appAudio: Bool
+
     /// Use broadcast extension for screen capture (iOS only).
     ///
     /// If a broadcast extension has been properly configured, this defaults to `true`.
@@ -49,12 +52,14 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen
     public init(dimensions: Dimensions = .h1080_169,
                 fps: Int = 30,
                 showCursor: Bool = true,
+                appAudio: Bool = true,
                 useBroadcastExtension: Bool = defaultToBroadcastExtension,
                 includeCurrentApplication: Bool = false)
     {
         self.dimensions = dimensions
         self.fps = fps
         self.showCursor = showCursor
+        self.appAudio = appAudio
         self.useBroadcastExtension = useBroadcastExtension
         self.includeCurrentApplication = includeCurrentApplication
     }
@@ -66,6 +71,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen
         return dimensions == other.dimensions &&
             fps == other.fps &&
             showCursor == other.showCursor &&
+            appAudio == other.appAudio &&
             useBroadcastExtension == other.useBroadcastExtension &&
             includeCurrentApplication == other.includeCurrentApplication
     }
@@ -75,6 +81,7 @@ public final class ScreenShareCaptureOptions: NSObject, VideoCaptureOptions, Sen
         hasher.combine(dimensions)
         hasher.combine(fps)
         hasher.combine(showCursor)
+        hasher.combine(appAudio)
         hasher.combine(useBroadcastExtension)
         hasher.combine(includeCurrentApplication)
         return hasher.finalize()