fix: fix issue that we can't grab windows in other spaces from macos …

…12.2

src/api-wrappers/CGWindowID.swift

@@ -19,11 +19,63 @@ extension CGWindowID {
         return CGSCopySpacesForWindows(cgsMainConnectionId, CGSSpaceMask.all.rawValue, [self] as CFArray) as! [CGSSpaceID]
     }
 
-    func screenshot() -> CGImage? {
-        // CGSHWCaptureWindowList
+    // fullscreen has multiple windows
+    // e.g. Notes.app has a toolbar window and a main window
+    // We need to composite these window images
+    func fullScreenshot(_ win: Window) -> CGImage? {
+        var height: CGFloat = 0;
+        var width: CGFloat = 0;
+        var imageMap = [(CGWindow, CGImage)]()
+        var maxWidthWindowId: CGWindowID = 0
+        let screen = Spaces.spaceFrameMap.first { $0.0 == win.spaceId }!.1
+        var windowsInSpaces = Spaces.windowsInSpaces([win.spaceId]) // The returned windows are sorted from highest to lowest according to the z-index
+        var windowsToCapture: [CGWindow] = []
+        // find current app's window in the fullscreen space
+        for item in windowsInSpaces {
+            let cgWin = CGWindowListCopyWindowInfo([.excludeDesktopElements, .optionIncludingWindow], item) as! [CGWindow]
+            guard cgWin.first!.isNotMenubarOrOthers(),
+                  cgWin.first!.ownerPID() == win.application.runningApplication.processIdentifier,
+                  cgWin.first!.bounds() != nil,
+                  let bounds = CGRect(dictionaryRepresentation: cgWin.first!.bounds()!), bounds.height > 0, bounds.width > 0 else { continue }
+            windowsToCapture.append(cgWin.first!)
+        }
+        // Drawing images from lowest to highest base on the z-index
+        windowsToCapture = windowsToCapture.reversed()
+        for item in windowsToCapture {
+            let bounds = CGRect(dictionaryRepresentation: item.bounds()!)
+            if width < bounds!.width {
+                maxWidthWindowId = item.id()!
+            }
+            var windowId = item.id()!
+            let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage]
+            imageMap.append((item, list.first!))
+        }
+        let bytesPerRow = imageMap.first { $0.0.id()! == maxWidthWindowId }!.1.bytesPerRow
+        var context = CGContext.init(data: nil,
+                width: Int(screen.frame.width),
+                height: Int(screen.frame.height),
+                bitsPerComponent: 8,
+                bytesPerRow: bytesPerRow,
+                space: imageMap.first!.1.colorSpace!,
+                bitmapInfo: imageMap.first!.1.bitmapInfo.rawValue)
+        // composite these window images
+        for item in imageMap {
+            let bounds = CGRect(dictionaryRepresentation: item.0.bounds()!)
+            // Convert the coordinate system, the origin of window is top-left, the image is bottom-left
+            // so we need to convert y-index
+            context?.draw(item.1, in: CGRect.init(x: bounds!.origin.x, y: screen.frame.height - bounds!.height - bounds!.origin.y, width: bounds!.width, height: bounds!.height))
+        }
+        return context?.makeImage()
+    }
+
+    func screenshot(_ win: Window) -> CGImage? {
         var windowId_ = self
-        let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId_, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage]
-        return list.first
+        if win.isFullscreen {
+            return fullScreenshot(win)
+        } else {
+            let list = CGSHWCaptureWindowList(cgsMainConnectionId, &windowId_, 1, [.ignoreGlobalClipShape, .nominalResolution]).takeRetainedValue() as! [CGImage]
+            return list.first
+        }
 
 //        // CGWindowListCreateImage
 //        return CGWindowListCreateImage(.null, .optionIncludingWindow, self, [.boundsIgnoreFraming, .bestResolution])

src/api-wrappers/PrivateApis.swift

@@ -122,10 +122,15 @@ func CGSCopyWindowsWithOptionsAndTags(_ cid: CGSConnectionID, _ owner: UInt32, _
 func CGSManagedDisplayGetCurrentSpace(_ cid: CGSConnectionID, _ displayUuid: CFString) -> CGSSpaceID
 
 // adds the provided windows to the provided spaces
-// * macOS 10.10+
+// * macOS 10.10-12.2
 @_silgen_name("CGSAddWindowsToSpaces")
 func CGSAddWindowsToSpaces(_ cid: CGSConnectionID, _ windows: NSArray, _ spaces: NSArray) -> Void
 
+// Move the given windows (CGWindowIDs) to the given space (CGSSpaceID)
+// * macOS 10.10+
+@_silgen_name("CGSMoveWindowsToManagedSpace")
+func CGSMoveWindowsToManagedSpace(_ cid: CGSConnectionID, _ windows: NSArray, _ space: CGSSpaceID) -> Void
+
 // remove the provided windows from the provided spaces
 // * macOS 10.10+
 @_silgen_name("CGSRemoveWindowsFromSpaces")

src/logic/Application.swift

@@ -140,6 +140,42 @@ class Application: NSObject {
         return windows
     }
 
+    func getOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) -> [Window] {
+        var otherSpaceWindows: [Window] = []
+        for winId in windowsOnlyOnOtherSpaces {
+            let cgWinArray = CGWindowListCopyWindowInfo([.excludeDesktopElements, .optionIncludingWindow], winId) as! [CGWindow]
+            // get current app's windows only on other space
+            guard runningApplication.processIdentifier == cgWinArray.first!.ownerPID()
+                          && cgWinArray.first!.id() != nil
+                          && cgWinArray.first!.isNotMenubarOrOthers()
+                          && cgWinArray.first!.bounds() != nil
+                          && CGRect(dictionaryRepresentation: cgWinArray.first!.bounds()!)!.width > 100
+                          && CGRect(dictionaryRepresentation: cgWinArray.first!.bounds()!)!.height > 100
+                    else { continue }
+            guard let capture = CGWindowListCreateImage(CGRect.null, .optionIncludingWindow, cgWinArray.first!.id()!, .boundsIgnoreFraming) else { continue }
+            let win = Window(self, cgWinArray.first!)
+            Windows.appendAndUpdateFocus(win)
+            otherSpaceWindows.append(win)
+        }
+        return otherSpaceWindows
+    }
+
+    func addOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) {
+        if runningApplication.isFinishedLaunching && runningApplication.activationPolicy != .prohibited {
+            retryAxCallUntilTimeout { [weak self] in
+                guard let self = self else { return }
+                DispatchQueue.main.async { [weak self] in
+                    guard let self = self else { return }
+                    var windows = self.getOtherSpaceWindows(windowsOnlyOnOtherSpaces)
+                    if let window = self.addWindowslessAppsIfNeeded() {
+                        windows.append(contentsOf: window)
+                    }
+                    App.app.refreshOpenUi(windows)
+                }
+            }
+        }
+    }
+
     func addWindowslessAppsIfNeeded() -> [Window]? {
         if !Preferences.hideWindowlessApps &&
                runningApplication.activationPolicy == .regular &&

src/logic/Applications.swift

@@ -14,6 +14,14 @@ class Applications {
         _ = group.wait(wallTimeout: .now() + .seconds(2))
     }
 
+    static func addOtherSpaceWindows(_ windowsOnlyOnOtherSpaces: [CGWindowID]) {
+        for app in list {
+            app.wasLaunchedBeforeAltTab = true
+            guard app.runningApplication.isFinishedLaunching else { continue }
+            app.addOtherSpaceWindows(windowsOnlyOnOtherSpaces)
+        }
+    }
+
     static func initialDiscovery() {
         addInitialRunningApplications()
         addInitialRunningApplicationsWindows()
@@ -32,10 +40,11 @@ class Applications {
             let windowsOnOtherSpaces = Spaces.windowsInSpaces(otherSpaces)
             let windowsOnlyOnOtherSpaces = Array(Set(windowsOnOtherSpaces).subtracting(windowsOnCurrentSpace))
             if windowsOnlyOnOtherSpaces.count > 0 {
-                // on initial launch, we use private APIs to bring windows from other spaces into the current space, observe them, then remove them from the current space
-                CGSAddWindowsToSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId])
-                Applications.observeNewWindowsBlocking()
-                CGSRemoveWindowsFromSpaces(cgsMainConnectionId, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId])
+                // Currently we add those window in other space without AXUIElement init
+                // We don't need to get the AXUIElement until we focus these windows.
+                // when we need to focus these windows, we use the helper window to take us to that space,
+                // then get the AXUIElement, and finally focus that window.
+                Applications.addOtherSpaceWindows(windowsOnlyOnOtherSpaces)
             }
         }
     }
@@ -115,7 +124,13 @@ class Applications {
             // Here I consider there is an edge case where AltTab is starting up and this program has been loading, then it is possible that axUiElement.windows() will throw an exception.
             // I'm not quite sure if this happens, but even if it does, then after restarting this application, AltTab captures its window without any problem. I think this happens rarely.
             let allWindows = CGWindow.windows(.optionAll)
-            guard let winApp = (allWindows.first { app.processIdentifier == $0.ownerPID() && $0.isNotMenubarOrOthers() && $0.id() != nil && $0.title() != nil}) else {
+            guard let winApp = (allWindows.first { app.processIdentifier == $0.ownerPID()
+                    && $0.isNotMenubarOrOthers()
+                    && $0.id() != nil
+                    && $0.bounds() != nil
+                    && CGRect(dictionaryRepresentation: $0.bounds()!)!.width > 0
+                    && CGRect(dictionaryRepresentation: $0.bounds()!)!.height > 0
+            }) else {
                 return false
             }
             return true

src/logic/HelperWindow.swift

@@ -0,0 +1,18 @@
+import Cocoa
+/**
+ * we use this window to help us switch to another space
+ */
+class HelperWindow: NSWindow {
+    var canBecomeKey_ = true
+    override var canBecomeKey: Bool { canBecomeKey_ }
+    convenience init() {
+        self.init(contentRect: .zero, styleMask: [.borderless], backing: .buffered, defer: false)
+        setupWindow()
+    }
+
+    private func setupWindow() {
+        isReleasedWhenClosed = false
+        hidesOnDeactivate = false
+        title = "Helper Window"
+    }
+}

src/logic/Spaces.swift

@@ -6,14 +6,25 @@ class Spaces {
     static var visibleSpaces = [CGSSpaceID]()
     static var screenSpacesMap = [ScreenUuid: [CGSSpaceID]]()
     static var idsAndIndexes = [(CGSSpaceID, SpaceIndex)]()
+    static var spaceFrameMap = [(CGSSpaceID, NSScreen)]()
 
     static func observeSpaceChanges() {
         NSWorkspace.shared.notificationCenter.addObserver(forName: NSWorkspace.activeSpaceDidChangeNotification, object: nil, queue: nil, using: { _ in
             debugPrint("OS event", "activeSpaceDidChangeNotification")
             refreshAllIdsAndIndexes()
             updateCurrentSpace()
             // if UI was kept open during Space transition, the Spaces may be obsolete; we refresh them
-            Windows.list.forEachAsync { $0.updatesWindowSpace() }
+            Windows.list.forEachAsync {
+                $0.updatesWindowSpace()
+                // we need to set up AXUIElement for those invisiable window launched before AltTab when space changed
+                if $0.axUiElement == nil && $0.spaceId == currentSpaceId && !$0.isWindowlessApp {
+                    do {
+                        try $0.getAxUiElementAndObserveEvents()
+                    } catch {
+                        debugPrint("can not setUpMissingInfoForOtherWindows for", $0.application.runningApplication.bundleIdentifier)
+                    }
+                }
+            }
         })
         NSWorkspace.shared.notificationCenter.addObserver(forName: NSApplication.didChangeScreenParametersNotification, object: nil, queue: nil, using: { _ in
             debugPrint("OS event", "didChangeScreenParametersNotification")
@@ -47,15 +58,18 @@ class Spaces {
         idsAndIndexes.removeAll()
         screenSpacesMap.removeAll()
         visibleSpaces.removeAll()
+        spaceFrameMap.removeAll()
         var spaceIndex = SpaceIndex(1)
         (CGSCopyManagedDisplaySpaces(cgsMainConnectionId) as! [NSDictionary]).forEach { (screen: NSDictionary) in
             var display = screen["Display Identifier"] as! ScreenUuid
             if display as String == "Main", let mainUuid = NSScreen.main?.uuid() {
                 display = mainUuid
             }
+            let nsScreen = NSScreen.screens.first { $0.uuid() == display } as! NSScreen
             (screen["Spaces"] as! [NSDictionary]).forEach { (space: NSDictionary) in
                 let spaceId = space["id64"] as! CGSSpaceID
                 idsAndIndexes.append((spaceId, spaceIndex))
+                spaceFrameMap.append((spaceId, nsScreen))
                 screenSpacesMap[display, default: []].append(spaceId)
                 spaceIndex += 1
             }

src/logic/Window.swift

@@ -60,6 +60,44 @@ class Window {
         debugPrint("Adding app-window", title ?? "nil", application.runningApplication.bundleIdentifier ?? "nil")
     }
 
+    // we init a window when this window is not at current active space
+    init(_ application: Application, _ cgWindow: CGWindow) {
+        self.application = application
+        self.title = bestEffortTitle(cgWindow.title())
+        self.cgWindowId = cgWindow.id()!
+        let bounds = CGRect(dictionaryRepresentation: cgWindow.bounds()!)
+        self.size = bounds!.size
+        self.position = bounds!.origin
+        updatesWindowSpace()
+        if CGSSpaceGetType(cgsMainConnectionId, self.spaceId) == .fullscreen {
+            self.isFullscreen = true
+        } else {
+            self.isFullscreen = false
+        }
+        self.isMinimized = false
+        if !Preferences.hideThumbnails {
+            refreshThumbnail()
+        }
+        application.removeWindowslessAppWindow()
+        debugPrint("Adding app-window", title ?? "nil", application.runningApplication.bundleIdentifier ?? "nil")
+    }
+
+    func getAxUiElementAndObserveEvents() throws {
+        guard let windows = try self.application.axUiElement?.windows(), windows.count > 0 else {
+            debugPrint("try to getAxUiElementAndObserveEvents nil", self.application.runningApplication.bundleIdentifier, self.title, self.spaceId)
+            return
+        }
+        for win in windows {
+            if try cgWindowId == win.cgWindowId() {
+                self.axUiElement = win
+                self.isMinimized = try self.axUiElement.isMinimized()
+                self.observeEvents()
+                return
+            }
+        }
+        debugPrint("try to getAxUiElementAndObserveEvents failed", self.application.runningApplication.bundleIdentifier, self.title, self.spaceId)
+    }
+
     deinit {
         debugPrint("Deinit window", title ?? "nil", application.runningApplication.bundleIdentifier ?? "nil")
     }
@@ -83,13 +121,13 @@ class Window {
     }
 
     func refreshThumbnail() {
-        guard let cgImage = cgWindowId.screenshot() else { return }
+        guard let cgImage = cgWindowId.screenshot(self) else { return }
         thumbnail = NSImage(cgImage: cgImage, size: NSSize(width: cgImage.width, height: cgImage.height))
         thumbnailFullSize = thumbnail!.size
     }
 
     func close() {
-        if isWindowlessApp { return }
+        if isWindowlessApp || self.axUiElement == nil { return }
         BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in
             guard let self = self else { return }
             if self.isFullscreen {
@@ -102,7 +140,7 @@ class Window {
     }
 
     func minDemin() {
-        if isWindowlessApp { return }
+        if isWindowlessApp || self.axUiElement == nil { return }
         BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in
             guard let self = self else { return }
             if self.isFullscreen {
@@ -119,7 +157,7 @@ class Window {
     }
 
     func toggleFullscreen() {
-        if isWindowlessApp { return }
+        if isWindowlessApp || self.axUiElement == nil { return }
         BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in
             guard let self = self else { return }
             self.axUiElement.setAttribute(kAXFullscreenAttribute, !self.isFullscreen)
@@ -145,13 +183,43 @@ class Window {
         }
     }
 
+    func gotoSpace(_ spaceId: CGSSpaceID) {
+        CGSMoveWindowsToManagedSpace(cgsMainConnectionId, [App.app.helperWindow.windowNumber] as NSArray, spaceId)
+        App.app.showHelperWindow()
+    }
+
     func focus() {
         if isWindowlessApp {
             if let bundleID = application.runningApplication.bundleIdentifier {
                 NSWorkspace.shared.launchApplication(withBundleIdentifier: bundleID, additionalEventParamDescriptor: nil, launchIdentifier: nil)
             } else {
                 application.runningApplication.activate(options: .activateIgnoringOtherApps)
             }
+        } else if self.axUiElement == nil {
+            // the window is in other space, we can not get AXUIElement before we go to it's space
+            // when we want to focus the window, it means we want to go to the space and focus the window
+            // so we just go to that space by using helperWindow, then we get AXUIElement and focus the window
+            gotoSpace(self.spaceId)
+            do {
+                retryAxCallUntilTimeout { [weak self] in
+                    guard let self = self else { return }
+                    try self.getAxUiElementAndObserveEvents()
+                    if self.axUiElement == nil {
+                        // retry until we get axUiElement
+                        throw AxError.runtimeError
+                    }
+                    BackgroundWork.accessibilityCommandsQueue.asyncWithCap { [weak self] in
+                        guard let self = self else { return }
+                        var psn = ProcessSerialNumber()
+                        GetProcessForPID(self.application.pid, &psn)
+                        _SLPSSetFrontProcessWithOptions(&psn, self.cgWindowId, .userGenerated)
+                        self.makeKeyWindow(psn)
+                        self.axUiElement.focusWindow()
+                    }
+                }
+            } catch {
+                debugPrint("can not setUpMissingInfoForCurrentWindow for", self.application.runningApplication.bundleIdentifier)
+            }
         } else {
             // macOS bug: when switching to a System Preferences window in another space, it switches to that space,
             // but quickly switches back to another window in that space

src/ui/App.swift

@@ -19,6 +19,7 @@ class App: AppCenterApplication, NSApplicationDelegate {
     static var app: App!
     var thumbnailsPanel: ThumbnailsPanel!
     var preferencesWindow: PreferencesWindow!
+    var helperWindow: HelperWindow!
     var feedbackWindow: FeedbackWindow!
     var isFirstSummon = true
     var appIsBeingUsed = false
@@ -59,6 +60,7 @@ class App: AppCenterApplication, NSApplicationDelegate {
             Spaces.initialDiscovery()
             Applications.initialDiscovery()
             self.preferencesWindow = PreferencesWindow()
+            self.helperWindow = HelperWindow()
             self.feedbackWindow = FeedbackWindow()
             KeyboardEvents.addEventHandlers()
             MouseEvents.observe()
@@ -158,6 +160,11 @@ class App: AppCenterApplication, NSApplicationDelegate {
         showSecondaryWindow(preferencesWindow)
     }
 
+    // focus the helper window and we can go to that space
+    @objc func showHelperWindow() {
+        showSecondaryWindow(helperWindow)
+    }
+
     func showSecondaryWindow(_ window: NSWindow?) {
         if let window = window {
             NSScreen.preferred().repositionPanel(window, .appleCentered)