Merge branch 'main' into xip-multivolume-expasion

# Conflicts:
#	Sources/XcodesKit/XcodeInstaller.swift
#	Sources/xcodes/main.swift
#	Tests/XcodesKitTests/XcodesKitTests.swift

.github/workflows/ci.yml

@@ -2,10 +2,10 @@ name: CI
 on: [push, pull_request]
 jobs:
   test:
-    runs-on: macOS-10.15
+    runs-on: macOS-11
     steps:
     - uses: actions/checkout@v3
     - name: Run tests
       env: 
-        DEVELOPER_DIR: /Applications/Xcode_12.3.app
+        DEVELOPER_DIR: /Applications/Xcode_13.2.1.app
       run: swift test

Sources/Unxip/Unxip.swift

@@ -133,8 +133,9 @@ struct File {
             compressionStream.addTask {
                 try Task.checkCancellation()
                 let position = _position
-                let data = [UInt8](unsafeUninitializedCapacity: blockSize + blockSize / 16) { buffer, count in
-                    data[position..<min(position + blockSize, data.endIndex)].withUnsafeBufferPointer { data in
+                let end = min(position + blockSize, data.endIndex)
+                let data = [UInt8](unsafeUninitializedCapacity: (end - position) + (end - position) / 16) { buffer, count in
+                    data[position..<end].withUnsafeBufferPointer { data in
                         count = compression_encode_buffer(buffer.baseAddress!, buffer.count, data.baseAddress!, data.count, nil, COMPRESSION_LZFSE)
                         guard count < buffer.count else {
                             count = 0
@@ -365,7 +366,7 @@ public struct Unxip {
 
             // The assumption is that all directories are provided without trailing slashes
             func parentDirectory<S: StringProtocol>(of path: S) -> S.SubSequence {
-                return path[..<path.lastIndex(of: "/")!]
+                path[..<path.lastIndex(of: "/")!]
             }
 
             // https://bugs.swift.org/browse/SR-15816
@@ -384,9 +385,11 @@ public struct Unxip {
                 continue
             }
 
-            if let (original, task) = hardlinks[file.identifier] {
+            if let (original, originalTask) = hardlinks[file.identifier] {
+                let task = parentDirectoryTask(for: file)
+                assert(task != nil, file.name)
                 _ = taskStream.addRunningTask {
-                    await task.value
+                    _ = await (originalTask.value, task?.value)
                     warn(link(original, file.name), "linking")
                 }
                 continue
@@ -399,6 +402,7 @@ public struct Unxip {
                     let task = parentDirectoryTask(for: file)
                     assert(task != nil, file.name)
                     _ = taskStream.addRunningTask {
+                        await task?.value
                         warn(symlink(String(data: Data(file.data.map(Array.init).reduce([], +)), encoding: .utf8), file.name), "symlinking")
                         setStickyBit(on: file)
                     }

Sources/XcodesKit/Models.swift

@@ -50,6 +50,10 @@ public struct Xcode: Codable, Equatable {
     public let filename: String
     public let releaseDate: Date?
 
+    public var downloadPath: String {
+        return url.path
+    }
+
     public init(version: Version, url: URL, filename: String, releaseDate: Date?) {
         self.version =  version
         self.url = url

Sources/XcodesKit/URLRequest+Apple.swift

@@ -4,6 +4,7 @@ extension URL {
     static let download = URL(string: "https://developer.apple.com/download")!
     static let downloads = URL(string: "https://developer.apple.com/services-account/QH65B2/downloadws/listDownloads.action")!
     static let downloadXcode = URL(string: "https://developer.apple.com/devcenter/download.action")!
+    static let downloadADCAuth = URL(string: "https://developerservices2.apple.com/services/download")!
 }
 
 extension URLRequest {
@@ -25,4 +26,14 @@ extension URLRequest {
         request.allHTTPHeaderFields?["Accept"] = "*/*"
         return request
     }
+
+    // default to a known download path if none passed in
+    static func downloadADCAuth(path: String? = "/Developer_Tools/Xcode_14/Xcode_14.xip") -> URLRequest {
+        var components = URLComponents(url: .downloadADCAuth, resolvingAgainstBaseURL: false)!
+        components.queryItems = [URLQueryItem(name: "path", value: path)]
+        var request = URLRequest(url: components.url!)
+        request.allHTTPHeaderFields = request.allHTTPHeaderFields ?? [:]
+        request.allHTTPHeaderFields?["Accept"] = "*/*"
+        return request
+    }
 }

Sources/XcodesKit/Version.swift

@@ -1,3 +1,3 @@
 import Version
 
-public let version = Version("0.20.0")!
+public let version = Version("1.0.0")!

Sources/XcodesKit/XcodeInstaller.swift

@@ -82,7 +82,7 @@ public final class XcodeInstaller {
         case downloading(version: String, progress: String?, willInstall: Bool)
         case unarchiving(experimentalUnxip: Bool)
         case moving(destination: String)
-        case trashingArchive(archiveName: String)
+        case cleaningArchive(archiveName: String, shouldDelete: Bool)
         case checkingSecurity
         case finishing
 
@@ -114,7 +114,10 @@ public final class XcodeInstaller {
                     """
             case .moving(let destination):
                 return "Moving Xcode to \(destination)"
-            case .trashingArchive(let archiveName):
+            case .cleaningArchive(let archiveName, let shouldDelete):
+                if shouldDelete {
+                    return "Deleting Xcode archive \(archiveName)"
+                }
                 return "Moving Xcode archive \(archiveName) to the Trash"
             case .checkingSecurity:
                 return "Checking security assessment and code signing"
@@ -128,7 +131,7 @@ public final class XcodeInstaller {
             case .downloading:      return 1
             case .unarchiving:      return 2
             case .moving:           return 3
-            case .trashingArchive:  return 4
+            case .cleaningArchive:  return 4
             case .checkingSecurity: return 5
             case .finishing:        return 6
             }
@@ -163,22 +166,22 @@ public final class XcodeInstaller {
         case aria2(Path)
     }
 
-    public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, experimentalUnxip: Bool = false, shouldExpandXipInplace: Bool) -> Promise<Void> {
+    public func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, experimentalUnxip: Bool = false, shouldExpandXipInplace: Bool, emptyTrash: Bool, noSuperuser: Bool) -> Promise<Void> {
         return firstly { () -> Promise<InstalledXcode> in
-            return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: 0, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace)
+            return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: 0, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace, emptyTrash: emptyTrash, noSuperuser: noSuperuser)
         }
         .done { xcode in
             Current.logging.log("\nXcode \(xcode.version.descriptionWithoutBuildMetadata) has been installed to \(xcode.path.string)".green)
             Current.shell.exit(0)
         }
     }
 
-    private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, attemptNumber: Int, experimentalUnxip: Bool, shouldExpandXipInplace: Bool) -> Promise<InstalledXcode> {
+    private func install(_ installationType: InstallationType, dataSource: DataSource, downloader: Downloader, destination: Path, attemptNumber: Int, experimentalUnxip: Bool, shouldExpandXipInplace: Bool, emptyTrash: Bool, noSuperuser: Bool) -> Promise<InstalledXcode> {
         return firstly { () -> Promise<(Xcode, URL)> in
             return self.getXcodeArchive(installationType, dataSource: dataSource, downloader: downloader, destination: destination, willInstall: true)
         }
         .then { xcode, url -> Promise<InstalledXcode> in
-            return self.installArchivedXcode(xcode, at: url, to: destination, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace)
+            return self.installArchivedXcode(xcode, at: url, to: destination, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace, emptyTrash: emptyTrash, noSuperuser: noSuperuser)
         }
         .recover { error -> Promise<InstalledXcode> in
             switch error {
@@ -195,7 +198,7 @@ public final class XcodeInstaller {
                         Current.logging.log(error.legibleLocalizedDescription.red)
                         Current.logging.log("Removing damaged XIP and re-attempting installation.\n")
                         try Current.files.removeItem(at: damagedXIPURL)
-                        return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: attemptNumber + 1, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace)
+                        return self.install(installationType, dataSource: dataSource, downloader: downloader, destination: destination, attemptNumber: attemptNumber + 1, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: shouldExpandXipInplace, emptyTrash: emptyTrash, noSuperuser: noSuperuser)
                     }
                 }
             default:
@@ -287,7 +290,15 @@ public final class XcodeInstaller {
 
     private func downloadXcode(version: Version, dataSource: DataSource, downloader: Downloader, willInstall: Bool) -> Promise<(Xcode, URL)> {
         return firstly { () -> Promise<Version> in
-            loginIfNeeded().map { version }
+            if dataSource == .apple {
+                return loginIfNeeded().map { version }
+            } else {
+                guard let xcode = self.xcodeList.availableXcodes.first(withVersion: version) else {
+                    throw Error.unavailableVersion(version)
+                }
+
+                return validateADCSession(path: xcode.downloadPath).map { version }
+            }
         }
         .then { version -> Promise<Version> in
             if self.xcodeList.shouldUpdate {
@@ -297,14 +308,6 @@ public final class XcodeInstaller {
                 return Promise.value(version)
             }
         }
-        .then { version -> Promise<Version> in
-            // This request would've already been made if the Apple data source were being used.
-            // That's not the case for the Xcode Releases data source.
-            // We need the cookies from its response in order to download Xcodes though,
-            // so perform it here first just to be sure.
-            Current.network.dataTask(with: URLRequest.downloads)
-                .map { _ in version }
-        }
         .then { version -> Promise<(Xcode, URL)> in
             guard let xcode = self.xcodeList.availableXcodes.first(withVersion: version) else {
                 throw Error.unavailableVersion(version)
@@ -334,7 +337,11 @@ public final class XcodeInstaller {
                 .map { return (xcode, $0) }
         }
     }
-
+
+    func validateADCSession(path: String) -> Promise<Void> {
+        return Current.network.dataTask(with: URLRequest.downloadADCAuth(path: path)).asVoid()
+    }
+
     func loginIfNeeded(withUsername providedUsername: String? = nil, shouldPromptForPassword: Bool = false) -> Promise<Void> {
         return firstly { () -> Promise<Void> in
             return Current.network.validateSession()
@@ -528,15 +535,7 @@ public final class XcodeInstaller {
         }
     }
 
-    public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path, experimentalUnxip: Bool = false, shouldExpandXipInplace: Bool) -> Promise<InstalledXcode> {
-        let passwordInput = {
-            Promise<String> { seal in
-                Current.logging.log("xcodes requires superuser privileges in order to finish installation.")
-                guard let password = Current.shell.readSecureLine(prompt: "macOS User Password: ") else { seal.reject(Error.missingSudoerPassword); return }
-                seal.fulfill(password + "\n")
-            }
-        }
-
+    public func installArchivedXcode(_ xcode: Xcode, at archiveURL: URL, to destination: Path, experimentalUnxip: Bool = false, shouldExpandXipInplace: Bool, emptyTrash: Bool, noSuperuser: Bool) -> Promise<InstalledXcode> {
         return firstly { () -> Promise<InstalledXcode> in
             let destinationURL = destination.join("Xcode-\(xcode.version.descriptionWithoutBuildMetadata).app").url
             switch archiveURL.pathExtension {
@@ -556,15 +555,38 @@ public final class XcodeInstaller {
             }
         }
         .then { xcode -> Promise<InstalledXcode> in
-            Current.logging.log(InstallationStep.trashingArchive(archiveName: archiveURL.lastPathComponent).description)
-            try Current.files.trashItem(at: archiveURL)
+            Current.logging.log(InstallationStep.cleaningArchive(archiveName: archiveURL.lastPathComponent, shouldDelete: emptyTrash).description)
+            if emptyTrash {
+                try Current.files.removeItem(at: archiveURL)
+            }
+            else {
+                try Current.files.trashItem(at: archiveURL)
+            }
             Current.logging.log(InstallationStep.checkingSecurity.description)
 
             return when(fulfilled: self.verifySecurityAssessment(of: xcode),
                                    self.verifySigningCertificate(of: xcode.path.url))
                 .map { xcode }
         }
         .then { xcode -> Promise<InstalledXcode> in
+            if noSuperuser {
+                Current.logging.log(InstallationStep.finishing.description)
+                Current.logging.log("Skipping asking for superuser privileges.")
+                return Promise.value(xcode)
+            }
+            return self.postInstallXcode(xcode)
+        }
+    }
+
+    public func postInstallXcode(_ xcode: InstalledXcode) -> Promise<InstalledXcode> {
+        let passwordInput = {
+            Promise<String> { seal in
+                Current.logging.log("xcodes requires superuser privileges in order to finish installation.")
+                guard let password = Current.shell.readSecureLine(prompt: "macOS User Password: ") else { seal.reject(Error.missingSudoerPassword); return }
+                seal.fulfill(password + "\n")
+            }
+        }
+        return firstly { () -> Promise<InstalledXcode> in
             Current.logging.log(InstallationStep.finishing.description)
 
             return self.enableDeveloperMode(passwordInput: passwordInput).map { xcode }
@@ -577,7 +599,7 @@ public final class XcodeInstaller {
         }
     }
 
-    public func uninstallXcode(_ versionString: String, directory: Path) -> Promise<Void> {
+    public func uninstallXcode(_ versionString: String, directory: Path, emptyTrash: Bool) -> Promise<Void> {
         return firstly { () -> Promise<InstalledXcode> in
             guard let version = Version(xcodeVersion: versionString) else {
                 Current.logging.log(Error.invalidVersion(versionString).legibleLocalizedDescription)
@@ -591,11 +613,17 @@ public final class XcodeInstaller {
 
             return Promise.value(installedXcode)
         }
-        .map { ($0, try Current.files.trashItem(at: $0.path.url)) }
-        .then { (installedXcode, trashURL) -> Promise<(InstalledXcode, URL)> in
+        .map { installedXcode -> (InstalledXcode, URL?) in
+            if emptyTrash {
+                try Current.files.removeItem(at: installedXcode.path.url)
+                return (installedXcode, nil)
+            }
+            return (installedXcode, try Current.files.trashItem(at: installedXcode.path.url))
+        }
+        .then { (installedXcode, trashURL) -> Promise<(InstalledXcode, URL?)> in
             // If we just uninstalled the selected Xcode, try to select the latest installed version so things don't accidentally break
             Current.shell.xcodeSelectPrintPath()
-                .then { output -> Promise<(InstalledXcode, URL)> in
+                .then { output -> Promise<(InstalledXcode, URL?)> in
                     if output.out.hasPrefix(installedXcode.path.string),
                        let latestInstalledXcode = Current.files.installedXcodes(directory).sorted(by: { $0.version < $1.version }).last {
                         return selectXcodeAtPath(latestInstalledXcode.path.string)
@@ -610,17 +638,26 @@ public final class XcodeInstaller {
                 }
         }
         .done { (installedXcode, trashURL) in
-            Current.logging.log("Xcode \(installedXcode.version.appleDescription) moved to Trash: \(trashURL.path)".green)
+            if let trashURL = trashURL {
+                Current.logging.log("Xcode \(installedXcode.version.appleDescription) moved to Trash: \(trashURL.path)".green)
+            }
+            else {
+                Current.logging.log("Xcode \(installedXcode.version.appleDescription) deleted".green)
+            }
             Current.shell.exit(0)
         }
     }
 
     func update(dataSource: DataSource) -> Promise<[Xcode]> {
-        return firstly { () -> Promise<Void> in
-            loginIfNeeded()
-        }
-        .then { () -> Promise<[Xcode]> in
-            self.xcodeList.update(dataSource: dataSource)
+        if dataSource == .apple {
+            return firstly { () -> Promise<Void> in
+                loginIfNeeded()
+            }
+            .then { () -> Promise<[Xcode]> in
+                self.xcodeList.update(dataSource: dataSource)
+            }
+        } else {
+            return self.xcodeList.update(dataSource: dataSource)
         }
     }
 

Sources/xcodes/main.swift

@@ -185,6 +185,12 @@ struct Xcodes: ParsableCommand {
 
         @Flag(help: "Use the experimental unxip functionality. May speed up unarchiving by up to 2-3x.")
         var experimentalUnxip: Bool = false
+
+        @Flag(help: "Don't ask for superuser (root) permission. Some optional steps of the installation will be skipped.")
+        var noSuperuser: Bool = false
+
+        @Flag(help: "Completely delete Xcode .xip after installation, instead of keeping it on the user's Trash.")
+        var emptyTrash: Bool = false
 
         @Option(help: "The directory to install Xcode into. Defaults to /Applications.",
                 completion: .directory)
@@ -224,7 +230,7 @@ struct Xcodes: ParsableCommand {
 
             let destination = getDirectory(possibleDirectory: directory)
 
-            installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: expandXipInplace)
+            installer.install(installation, dataSource: globalDataSource.dataSource, downloader: downloader, destination: destination, experimentalUnxip: experimentalUnxip, shouldExpandXipInplace: expandXipInplace, emptyTrash: emptyTrash, noSuperuser: noSuperuser)
                 .done { Install.exit() }
                 .catch { error in
                     Install.processDownloadOrInstall(error: error)
@@ -348,6 +354,9 @@ struct Xcodes: ParsableCommand {
                   completion: .custom { _ in Current.files.installedXcodes(getDirectory(possibleDirectory: nil)).sorted { $0.version < $1.version }.map { $0.version.appleDescription } })
         var version: [String] = []
 
+        @Flag(help: "Completely delete Xcode, instead of keeping it on the user's Trash.")
+        var emptyTrash: Bool = false
+
         @OptionGroup
         var globalDirectory: GlobalDirectoryOption
 
@@ -359,7 +368,7 @@ struct Xcodes: ParsableCommand {
 
             let directory = getDirectory(possibleDirectory: globalDirectory.directory)
 
-            installer.uninstallXcode(version.joined(separator: " "), directory: directory)
+            installer.uninstallXcode(version.joined(separator: " "), directory: directory, emptyTrash: emptyTrash)
                 .done { Uninstall.exit() }
                 .catch { error in Uninstall.exit(withLegibleError: error) }