From e870ae825b48c4a43376bf4e01c72de3aa190093 Mon Sep 17 00:00:00 2001 From: Brandon Evans Date: Thu, 15 Oct 2020 22:15:57 -0600 Subject: [PATCH] Default to use aria2 for downloads if installed --- README.md | 2 + Sources/XcodesKit/Aria2CError.swift | 125 ++++++++++++++++++++++ Sources/XcodesKit/Environment.swift | 75 +++++++++++++ Sources/XcodesKit/Promise+.swift | 40 +++++++ Sources/XcodesKit/XcodeInstaller.swift | 89 +++++++++------ Sources/xcodes/main.swift | 19 +++- Tests/XcodesKitTests/XcodesKitTests.swift | 8 +- 7 files changed, 318 insertions(+), 40 deletions(-) create mode 100644 Sources/XcodesKit/Aria2CError.swift create mode 100644 Sources/XcodesKit/Promise+.swift diff --git a/README.md b/README.md index c669af2..a132b1e 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,8 @@ macOS User Password: Xcode 11.2.0 has been installed to /Applications/Xcode-11.2.0.app ``` +If you have [aria2](https://aria2.github.io) installed (it's available in Homebrew, `brew install aria2`), then xcodes will default to use it for downloads. It uses up to 16 connections to download Xcode 3-5x faster than URLSession. + ### Commands - `install `: Download and install a specific version of Xcode diff --git a/Sources/XcodesKit/Aria2CError.swift b/Sources/XcodesKit/Aria2CError.swift new file mode 100644 index 0000000..c652626 --- /dev/null +++ b/Sources/XcodesKit/Aria2CError.swift @@ -0,0 +1,125 @@ +import Foundation + +/// A LocalizedError that represents a non-zero exit code from running aria2c. +struct Aria2CError: LocalizedError { + var code: Code + + init?(exitStatus: Int32) { + guard let code = Code(rawValue: exitStatus) else { return nil } + self.code = code + } + + var errorDescription: String? { + "aria2c error: \(code.description)" + } + + // https://github.com/aria2/aria2/blob/master/src/error_code.h + enum Code: Int32, CustomStringConvertible { + case undefined = -1 + // Ignoring, not an error + // case finished = 0 + case unknownError = 1 + case timeOut + case resourceNotFound + case maxFileNotFound + case tooSlowDownloadSpeed + case networkProblem + case inProgress + case cannotResume + case notEnoughDiskSpace + case pieceLengthChanged + case duplicateDownload + case duplicateInfoHash + case fileAlreadyExists + case fileRenamingFailed + case fileOpenError + case fileCreateError + case fileIoError + case dirCreateError + case nameResolveError + case metalinkParseError + case ftpProtocolError + case httpProtocolError + case httpTooManyRedirects + case httpAuthFailed + case bencodeParseError + case bittorrentParseError + case magnetParseError + case optionError + case httpServiceUnavailable + case jsonParseError + case removed + case checksumError + + var description: String { + switch self { + case .undefined: + return "Undefined" + case .unknownError: + return "Unknown error" + case .timeOut: + return "Timed out" + case .resourceNotFound: + return "Resource not found" + case .maxFileNotFound: + return "Maximum number of file not found errors reached" + case .tooSlowDownloadSpeed: + return "Download speed too slow" + case .networkProblem: + return "Network problem" + case .inProgress: + return "Unfinished downloads in progress" + case .cannotResume: + return "Remote server did not support resume when resume was required to complete download" + case .notEnoughDiskSpace: + return "Not enough disk space available" + case .pieceLengthChanged: + return "Piece length was different from one in .aria2 control file" + case .duplicateDownload: + return "Duplicate download" + case .duplicateInfoHash: + return "Duplicate info hash torrent" + case .fileAlreadyExists: + return "File already exists" + case .fileRenamingFailed: + return "Renaming file failed" + case .fileOpenError: + return "Could not open existing file" + case .fileCreateError: + return "Could not create new file or truncate existing file" + case .fileIoError: + return "File I/O error" + case .dirCreateError: + return "Could not create directory" + case .nameResolveError: + return "Name resolution failed" + case .metalinkParseError: + return "Could not parse Metalink document" + case .ftpProtocolError: + return "FTP command failed" + case .httpProtocolError: + return "HTTP response header was bad or unexpected" + case .httpTooManyRedirects: + return "Too many redirects occurred" + case .httpAuthFailed: + return "HTTP authorization failed" + case .bencodeParseError: + return "Could not parse bencoded file (usually \".torrent\" file)" + case .bittorrentParseError: + return "\".torrent\" file was corrupted or missing information" + case .magnetParseError: + return "Magnet URI was bad" + case .optionError: + return "Bad/unrecognized option was given or unexpected option argument was given" + case .httpServiceUnavailable: + return "HTTP service unavailable" + case .jsonParseError: + return "Could not parse JSON-RPC request" + case .removed: + return "Reserved. Not used." + case .checksumError: + return "Checksum validation failed" + } + } + } +} diff --git a/Sources/XcodesKit/Environment.swift b/Sources/XcodesKit/Environment.swift index 7027128..691e4da 100644 --- a/Sources/XcodesKit/Environment.swift +++ b/Sources/XcodesKit/Environment.swift @@ -53,6 +53,81 @@ public struct Shell { public func xcodeSelectSwitch(password: String?, path: String) -> Promise { xcodeSelectSwitch(password, path) } + + public var downloadWithAria2: (Path, URL, Path, [HTTPCookie]) -> (Progress, Promise) = { aria2Path, url, destination, cookies in + let process = Process() + process.executableURL = aria2Path.url + process.arguments = [ + "--header=Cookie: \(cookies.map { "\($0.name)=\($0.value)" }.joined(separator: "; "))", + "--max-connection-per-server=16", + "--split=16", + "--summary-interval=1", + "--stop-with-process=\(ProcessInfo.processInfo.processIdentifier)", + "--dir=\(destination.parent.string)", + "--out=\(destination.basename())", + url.absoluteString, + ] + let stdOutPipe = Pipe() + process.standardOutput = stdOutPipe + let stdErrPipe = Pipe() + process.standardError = stdErrPipe + + var progress = Progress(totalUnitCount: 100) + + let observer = NotificationCenter.default.addObserver( + forName: .NSFileHandleDataAvailable, + object: nil, + queue: OperationQueue.main + ) { note in + guard + // This should always be the case for Notification.Name.NSFileHandleDataAvailable + let handle = note.object as? FileHandle, + handle === stdOutPipe.fileHandleForReading || handle === stdErrPipe.fileHandleForReading + else { return } + + defer { handle.waitForDataInBackgroundAndNotify() } + + let string = String(decoding: handle.availableData, as: UTF8.self) + let regex = try! NSRegularExpression(pattern: #"((?\d+)%\))"#) + let range = NSRange(location: 0, length: string.utf16.count) + + guard + let match = regex.firstMatch(in: string, options: [], range: range), + let matchRange = Range(match.range(withName: "percent"), in: string), + let percentCompleted = Int64(string[matchRange]) + else { return } + + progress.completedUnitCount = percentCompleted + } + + stdOutPipe.fileHandleForReading.waitForDataInBackgroundAndNotify() + stdErrPipe.fileHandleForReading.waitForDataInBackgroundAndNotify() + + do { + try process.run() + } catch { + return (progress, Promise(error: error)) + } + + let promise = Promise { seal in + DispatchQueue.global(qos: .default).async { + process.waitUntilExit() + + NotificationCenter.default.removeObserver(observer, name: .NSFileHandleDataAvailable, object: nil) + + guard process.terminationReason == .exit, process.terminationStatus == 0 else { + if let aria2cError = Aria2CError(exitStatus: process.terminationStatus) { + return seal.reject(aria2cError) + } else { + return seal.reject(Process.PMKError.execution(process: process, standardOutput: "", standardError: "")) + } + } + seal.fulfill(()) + } + } + + return (progress, promise) + } public var readLine: (String) -> String? = { prompt in print(prompt, terminator: "") diff --git a/Sources/XcodesKit/Promise+.swift b/Sources/XcodesKit/Promise+.swift new file mode 100644 index 0000000..9398cd7 --- /dev/null +++ b/Sources/XcodesKit/Promise+.swift @@ -0,0 +1,40 @@ +import Foundation +import PromiseKit + +/// Attempt and retry a task that fails with resume data up to `maximumRetryCount` times +func attemptResumableTask( + maximumRetryCount: Int = 3, + delayBeforeRetry: DispatchTimeInterval = .seconds(2), + _ body: @escaping (Data?) -> Promise +) -> Promise { + var attempts = 0 + func attempt(with resumeData: Data? = nil) -> Promise { + attempts += 1 + return body(resumeData).recover { error -> Promise in + guard + attempts < maximumRetryCount, + let resumeData = (error as NSError).userInfo[NSURLSessionDownloadTaskResumeData] as? Data + else { throw error } + + return after(delayBeforeRetry).then(on: nil) { attempt(with: resumeData) } + } + } + return attempt() +} + +/// Attempt and retry a task up to `maximumRetryCount` times +func attemptRetryableTask( + maximumRetryCount: Int = 3, + delayBeforeRetry: DispatchTimeInterval = .seconds(2), + _ body: @escaping () -> Promise +) -> Promise { + var attempts = 0 + func attempt() -> Promise { + attempts += 1 + return body().recover { error -> Promise in + guard attempts < maximumRetryCount else { throw error } + return after(delayBeforeRetry).then(on: nil) { attempt() } + } + } + return attempt() +} diff --git a/Sources/XcodesKit/XcodeInstaller.swift b/Sources/XcodesKit/XcodeInstaller.swift index 7d09662..f48fe0b 100644 --- a/Sources/XcodesKit/XcodeInstaller.swift +++ b/Sources/XcodesKit/XcodeInstaller.swift @@ -133,10 +133,15 @@ public final class XcodeInstaller { case latest case latestPrerelease } + + public enum Downloader { + case urlSession + case aria2(Path) + } - public func install(_ installationType: InstallationType) -> Promise { + public func install(_ installationType: InstallationType, downloader: Downloader) -> Promise { return firstly { () -> Promise in - return self.install(installationType, attemptNumber: 0) + return self.install(installationType, downloader: downloader, attemptNumber: 0) } .done { xcode in Current.logging.log("\nXcode \(xcode.version.descriptionWithoutBuildMetadata) has been installed to \(xcode.path.string)") @@ -144,9 +149,9 @@ public final class XcodeInstaller { } } - private func install(_ installationType: InstallationType, attemptNumber: Int) -> Promise { + private func install(_ installationType: InstallationType, downloader: Downloader, attemptNumber: Int) -> Promise { return firstly { () -> Promise<(Xcode, URL)> in - return self.getXcodeArchive(installationType) + return self.getXcodeArchive(installationType, downloader: downloader) } .then { xcode, url -> Promise in return self.installArchivedXcode(xcode, at: url) @@ -166,7 +171,7 @@ public final class XcodeInstaller { Current.logging.log(error.legibleLocalizedDescription) Current.logging.log("Removing damaged XIP and re-attempting installation.\n") try Current.files.removeItem(at: damagedXIPURL) - return self.install(installationType, attemptNumber: attemptNumber + 1) + return self.install(installationType, downloader: downloader, attemptNumber: attemptNumber + 1) } } default: @@ -175,7 +180,7 @@ public final class XcodeInstaller { } } - private func getXcodeArchive(_ installationType: InstallationType) -> Promise<(Xcode, URL)> { + private func getXcodeArchive(_ installationType: InstallationType, downloader: Downloader) -> Promise<(Xcode, URL)> { return firstly { () -> Promise<(Xcode, URL)> in switch installationType { case .latest: @@ -192,7 +197,7 @@ public final class XcodeInstaller { throw Error.versionAlreadyInstalled(installedXcode) } - return self.downloadXcode(version: latestNonPrereleaseXcode.version) + return self.downloadXcode(version: latestNonPrereleaseXcode.version, downloader: downloader) } case .latestPrerelease: Current.logging.log("Updating...") @@ -213,7 +218,7 @@ public final class XcodeInstaller { throw Error.versionAlreadyInstalled(installedXcode) } - return self.downloadXcode(version: latestPrereleaseXcode.version) + return self.downloadXcode(version: latestPrereleaseXcode.version, downloader: downloader) } case .url(let versionString, let path): guard let version = Version(xcodeVersion: versionString) ?? versionFromXcodeVersionFile() else { @@ -228,7 +233,7 @@ public final class XcodeInstaller { if let installedXcode = Current.files.installedXcodes().first(where: { $0.version.isEqualWithoutBuildMetadataIdentifiers(to: version) }) { throw Error.versionAlreadyInstalled(installedXcode) } - return self.downloadXcode(version: version) + return self.downloadXcode(version: version, downloader: downloader) } } } @@ -241,7 +246,7 @@ public final class XcodeInstaller { return version } - private func downloadXcode(version: Version) -> Promise<(Xcode, URL)> { + private func downloadXcode(version: Version, downloader: Downloader) -> Promise<(Xcode, URL)> { return firstly { () -> Promise in loginIfNeeded().map { version } } @@ -263,7 +268,7 @@ public final class XcodeInstaller { let formatter = NumberFormatter(numberStyle: .percent) var observation: NSKeyValueObservation? - let promise = self.downloadOrUseExistingArchive(for: xcode, progressChanged: { progress in + let promise = self.downloadOrUseExistingArchive(for: xcode, downloader: downloader, progressChanged: { progress in observation?.invalidate() observation = progress.observe(\.fractionCompleted) { progress, _ in // These escape codes move up a line and then clear to the end @@ -355,20 +360,55 @@ public final class XcodeInstaller { return nil } - public func downloadOrUseExistingArchive(for xcode: Xcode, progressChanged: @escaping (Progress) -> Void) -> Promise { + public func downloadOrUseExistingArchive(for xcode: Xcode, downloader: Downloader, progressChanged: @escaping (Progress) -> Void) -> Promise { // Check to see if the archive is in the expected path in case it was downloaded but failed to install let expectedArchivePath = Path.xcodesApplicationSupport/"Xcode-\(xcode.version).\(xcode.filename.suffix(fromLast: "."))" - if Current.files.fileExistsAtPath(expectedArchivePath.string) { + // aria2 downloads directly to the destination (instead of into /tmp first) so we need to make sure that the download isn't incomplete + let aria2DownloadMetadataPath = expectedArchivePath.parent/(expectedArchivePath.basename() + ".aria2") + var aria2DownloadIsIncomplete = false + if case .aria2 = downloader, aria2DownloadMetadataPath.exists { + aria2DownloadIsIncomplete = true + } + if Current.files.fileExistsAtPath(expectedArchivePath.string), aria2DownloadIsIncomplete == false { Current.logging.log("(1/6) Found existing archive that will be used for installation at \(expectedArchivePath).") return Promise.value(expectedArchivePath.url) } else { - return downloadXcode(xcode, progressChanged: progressChanged) + let destination = Path.xcodesApplicationSupport/"Xcode-\(xcode.version).\(xcode.filename.suffix(fromLast: "."))" + switch downloader { + case .aria2(let aria2Path): + return downloadXcodeWithAria2( + xcode, + to: destination, + aria2Path: aria2Path, + progressChanged: progressChanged + ) + case .urlSession: + return downloadXcodeWithURLSession( + xcode, + to: destination, + progressChanged: progressChanged + ) + } + } + } + + public func downloadXcodeWithAria2(_ xcode: Xcode, to destination: Path, aria2Path: Path, progressChanged: @escaping (Progress) -> Void) -> Promise { + let cookies = AppleAPI.Current.network.session.configuration.httpCookieStorage?.cookies(for: xcode.url) ?? [] + + return attemptRetryableTask(maximumRetryCount: 3) { + let (progress, promise) = Current.shell.downloadWithAria2( + aria2Path, + xcode.url, + destination, + cookies + ) + progressChanged(progress) + return promise.map { _ in destination.url } } } - public func downloadXcode(_ xcode: Xcode, progressChanged: @escaping (Progress) -> Void) -> Promise { - let destination = Path.xcodesApplicationSupport/"Xcode-\(xcode.version).\(xcode.filename.suffix(fromLast: "."))" + public func downloadXcodeWithURLSession(_ xcode: Xcode, to destination: Path, progressChanged: @escaping (Progress) -> Void) -> Promise { let resumeDataPath = Path.xcodesApplicationSupport/"Xcode-\(xcode.version).resumedata" let persistedResumeData = Current.files.contents(atPath: resumeDataPath.string) @@ -691,20 +731,3 @@ private extension XcodeInstaller { } } } - -/// Attempt and retry a task that fails with resume data up to `maximumRetryCount` times -private func attemptResumableTask(maximumRetryCount: Int = 3, delayBeforeRetry: DispatchTimeInterval = .seconds(2), _ body: @escaping (Data?) -> Promise) -> Promise { - var attempts = 0 - func attempt(with resumeData: Data? = nil) -> Promise { - attempts += 1 - return body(resumeData).recover { error -> Promise in - guard - attempts < maximumRetryCount, - let resumeData = (error as NSError).userInfo[NSURLSessionDownloadTaskResumeData] as? Data - else { throw error } - - return after(delayBeforeRetry).then(on: nil) { attempt(with: resumeData) } - } - } - return attempt() -} diff --git a/Sources/xcodes/main.swift b/Sources/xcodes/main.swift index 4f41486..d18cc52 100644 --- a/Sources/xcodes/main.swift +++ b/Sources/xcodes/main.swift @@ -95,9 +95,16 @@ app.add(subCommand: update) let urlFlag = Flag(longName: "url", type: String.self, description: "Local path to Xcode .xip") let latestFlag = Flag(longName: "latest", value: false, description: "Update and then install the latest non-prerelease version available.") let latestPrereleaseFlag = Flag(longName: "latest-prerelease", value: false, description: "Update and then install the latest prerelease version available, including GM seeds and GMs.") +let aria2 = Flag(longName: "aria2", type: String.self, description: "The path to an aria2 executable. Defaults to /usr/local/bin/aria2c.") +let noAria2 = Flag(longName: "no-aria2", value: false, description: "Don't use aria2 to download Xcode, even if its available.") let install = Command(usage: "install ", shortMessage: "Download and install a specific version of Xcode", - flags: [urlFlag, latestFlag, latestPrereleaseFlag], + longMessage: """ + Download and install a specific version of Xcode + + By default, xcodes will use a URLSession to download the specified version. If aria2 (https://aria2.github.io, available in Homebrew) is installed, either at /usr/local/bin/aria2c or at the path specified by the --aria2 flag, then it will be used instead. aria2 will use up to 16 connections to download Xcode 3-5x faster. If you have aria2 installed and would prefer to not use it, you can use the --no-aria2 flag. + """, + flags: [urlFlag, latestFlag, latestPrereleaseFlag, aria2, noAria2], example: """ xcodes install 10.2.1 xcodes install 11 Beta 7 @@ -117,8 +124,14 @@ let install = Command(usage: "install ", } else { installation = .version(versionString) } - - installer.install(installation) + + var downloader = XcodeInstaller.Downloader.urlSession + let aria2Path = flags.getString(name: "aria2").flatMap(Path.init) ?? Path.root.usr.local.bin/"aria2c" + if aria2Path.exists, flags.getBool(name: "no-aria2") != true { + downloader = .aria2(aria2Path) + } + + installer.install(installation, downloader: downloader) .catch { error in switch error { case Process.PMKError.execution(let process, let standardOutput, let standardError): diff --git a/Tests/XcodesKitTests/XcodesKitTests.swift b/Tests/XcodesKitTests/XcodesKitTests.swift index 9f9808a..5e4ea56 100644 --- a/Tests/XcodesKitTests/XcodesKitTests.swift +++ b/Tests/XcodesKitTests/XcodesKitTests.swift @@ -51,7 +51,7 @@ final class XcodesKitTests: XCTestCase { } let xcode = Xcode(version: Version("0.0.0")!, url: URL(string: "https://apple.com/xcode.xip")!, filename: "mock.xip", releaseDate: nil) - installer.downloadOrUseExistingArchive(for: xcode, progressChanged: { _ in }) + installer.downloadOrUseExistingArchive(for: xcode, downloader: .urlSession, progressChanged: { _ in }) .tap { result in guard case .fulfilled(let value) = result else { XCTFail("downloadOrUseExistingArchive rejected."); return } XCTAssertEqual(value, Path.applicationSupport.join("com.robotsandpencils.xcodes").join("Xcode-0.0.0.xip").url) @@ -69,7 +69,7 @@ final class XcodesKitTests: XCTestCase { } let xcode = Xcode(version: Version("0.0.0")!, url: URL(string: "https://apple.com/xcode.xip")!, filename: "mock.xip", releaseDate: nil) - installer.downloadOrUseExistingArchive(for: xcode, progressChanged: { _ in }) + installer.downloadOrUseExistingArchive(for: xcode, downloader: .urlSession, progressChanged: { _ in }) .tap { result in guard case .fulfilled(let value) = result else { XCTFail("downloadOrUseExistingArchive rejected."); return } XCTAssertEqual(value, Path.applicationSupport.join("com.robotsandpencils.xcodes").join("Xcode-0.0.0.xip").url) @@ -197,7 +197,7 @@ final class XcodesKitTests: XCTestCase { let expectation = self.expectation(description: "Finished") - installer.install(.version("0.0.0")) + installer.install(.version("0.0.0"), downloader: .urlSession) .ensure { let url = Bundle.module.url(forResource: "LogOutput-FullHappyPath", withExtension: "txt", subdirectory: "Fixtures")! XCTAssertEqual(log, try! String(contentsOf: url)) @@ -312,7 +312,7 @@ final class XcodesKitTests: XCTestCase { let expectation = self.expectation(description: "Finished") - installer.install(.version("0.0.0")) + installer.install(.version("0.0.0"), downloader: .urlSession) .ensure { let url = Bundle.module.url(forResource: "LogOutput-DamagedXIP", withExtension: "txt", subdirectory: "Fixtures")! let expectedText = try! String(contentsOf: url).replacingOccurrences(of: "/Users/brandon", with: Path.home.string)