From bf8f33976276098d493ae26becfee784f63f9c84 Mon Sep 17 00:00:00 2001 From: Brentley Jones Date: Wed, 6 Mar 2024 14:17:05 -0600 Subject: [PATCH 1/5] WIP: swift calculate_output_groups Signed-off-by: Brentley Jones Signed-off-by: Matt Pennig --- tools/BUILD | 34 +++++ tools/calculate_output_groups/Arguments.swift | 41 ++++++ tools/calculate_output_groups/BUILD | 64 +++++++++ .../BUILD.release.bazel | 1 + .../CalculateOutputGroups.swift | 31 +++++ .../OutputGroupsCalculator.swift | 126 ++++++++++++++++++ 6 files changed, 297 insertions(+) create mode 100644 tools/calculate_output_groups/Arguments.swift create mode 100644 tools/calculate_output_groups/BUILD create mode 100644 tools/calculate_output_groups/BUILD.release.bazel create mode 100644 tools/calculate_output_groups/CalculateOutputGroups.swift create mode 100644 tools/calculate_output_groups/OutputGroupsCalculator.swift diff --git a/tools/BUILD b/tools/BUILD index a0d546fde2..7c7632c011 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -4,6 +4,7 @@ load("//xcodeproj:defs.bzl", "xcodeproj", "xcschemes") load("//xcodeproj/internal:collections.bzl", "flatten", "uniq") _TOOLS = { + "calculate_output_groups": "//tools/calculate_output_groups", "files_and_groups": "//tools/generators/files_and_groups", "import_indexstores": "//tools/import_indexstores", "pbxnativetargets": "//tools/generators/pbxnativetargets", @@ -51,6 +52,38 @@ _XCSCHEME_DIAGNOSTICS = xcschemes.diagnostics( ) _XCSCHEMES = [ + xcschemes.scheme( + name = "calculate_output_groups", + profile = xcschemes.profile( + launch_target = xcschemes.launch_target( + _TOOLS["calculate_output_groups"], + ), + xcode_configuration = "Release", + ), + run = xcschemes.run( + args = [ + # colorDiagnostics + "NO", + # xcodeVersionActual + "1520", + # nonPreviewObjRoot + "/Users/brentley/Library/Developer/Xcode/DerivedData/tools-exxvdkcaoxdlhndlfnwxqvucohsr/Build/Intermediates.noindex", + # baseObjRoot + "/Users/brentley/Library/Developer/Xcode/DerivedData/tools-exxvdkcaoxdlhndlfnwxqvucohsr/Build/Intermediates.noindex", + # buildMarkerFile + "/Users/brentley/Library/Developer/Xcode/DerivedData/tools-exxvdkcaoxdlhndlfnwxqvucohsr/Build/Intermediates.noindex/build_marker", + # outputGroupPrefixes + "bc,bp,bi", + ], + build_targets = [ + _TOOLS["calculate_output_groups"], + ], + diagnostics = _XCSCHEME_DIAGNOSTICS, + launch_target = xcschemes.launch_target( + _TOOLS["calculate_output_groups"], + ), + ), + ), xcschemes.scheme( name = "files_and_groups", profile = xcschemes.profile( @@ -561,6 +594,7 @@ xcodeproj( filegroup( name = "release_files", srcs = [ + "//" + package_name() + "/calculate_output_groups:release_files", "//" + package_name() + "/extension_point_identifiers_parser:release_files", "//" + package_name() + "/generators:release_files", "//" + package_name() + "/import_indexstores:release_files", diff --git a/tools/calculate_output_groups/Arguments.swift b/tools/calculate_output_groups/Arguments.swift new file mode 100644 index 0000000000..34ec13ac1f --- /dev/null +++ b/tools/calculate_output_groups/Arguments.swift @@ -0,0 +1,41 @@ +import ArgumentParser +import Foundation + +extension OutputGroupsCalculator { + struct Arguments: ParsableArguments { + @Argument( + help: "Value of the 'XCODE_VERSION_ACTUAL' environment variable." + ) + var xcodeVersionActual: Int + + @Argument( + help: """ +Value of the 'OBJROOT' build setting when 'ENABLE_PREVIEWS = NO'. +""", + transform: { URL(fileURLWithPath: $0, isDirectory: true) } + ) + var nonPreviewObjRoot: URL + + @Argument( + help: """ +Value of 'nonPreviewObjRoot' when 'INDEX_ENABLE_BUILD_ARENA = NO'. +""", + transform: { URL(fileURLWithPath: $0, isDirectory: true) } + ) + var baseObjRoot: URL + + @Argument( + help: """ +Path to a file that has a ctime at or after the start of the build. +""", + transform: { URL(fileURLWithPath: $0, isDirectory: false) } + ) + var buildMarkerFile: URL + + @Argument( + help: "Comma seperated list of output group prefixes.", + transform: { $0.split(separator: ",").map(String.init) } + ) + var outputGroupPrefixes: [String] + } +} diff --git a/tools/calculate_output_groups/BUILD b/tools/calculate_output_groups/BUILD new file mode 100644 index 0000000000..d30a3f3be6 --- /dev/null +++ b/tools/calculate_output_groups/BUILD @@ -0,0 +1,64 @@ +load("@build_bazel_rules_apple//apple:apple.bzl", "apple_universal_binary") +load( + "@build_bazel_rules_apple//apple:macos.bzl", + "macos_command_line_application", +) +load( + "@build_bazel_rules_swift//swift:swift.bzl", + "swift_binary", + "swift_library", +) + +# This target exists to keep configurations the same between the generator +# and the tests, which makes the Xcode development experience better. If we used +# `swift_binary` or `apple_universal_binary` in `xcodeproj`, then the +# `macos_unit_test` transition (which is used to be able to set a minimum os +# version on the tests) will create slightly different configurations for our +# `swift_library`s. Maybe https://github.com/bazelbuild/bazel/issues/6526 will +# fix that for us. +macos_command_line_application( + name = "calculate_output_groups", + minimum_os_version = "12.0", + visibility = ["//visibility:public"], + deps = [":calculate_output_groups.library"], +) + +swift_library( + name = "calculate_output_groups.library", + srcs = glob(["*.swift"]), + module_name = "calculate_output_groups", + deps = [ + "//tools/lib/ToolCommon", + "@com_github_apple_swift_argument_parser//:ArgumentParser", + "@com_github_michaeleisel_zippyjson//:ZippyJSON", + ], +) + +swift_binary( + name = "calculate_output_groups_binary", + deps = [":calculate_output_groups.library"], +) + +apple_universal_binary( + name = "universal_calculate_output_groups", + binary = ":calculate_output_groups_binary", + forced_cpus = [ + "x86_64", + "arm64", + ], + minimum_os_version = "12.0", + platform_type = "macos", + visibility = ["//visibility:public"], +) + +# Release + +filegroup( + name = "release_files", + srcs = [ + "BUILD.release.bazel", + ":universal_calculate_output_groups", + ], + tags = ["manual"], + visibility = ["//:__subpackages__"], +) diff --git a/tools/calculate_output_groups/BUILD.release.bazel b/tools/calculate_output_groups/BUILD.release.bazel new file mode 100644 index 0000000000..a928619468 --- /dev/null +++ b/tools/calculate_output_groups/BUILD.release.bazel @@ -0,0 +1 @@ +exports_files(["universal_calculate_output_groups"]) diff --git a/tools/calculate_output_groups/CalculateOutputGroups.swift b/tools/calculate_output_groups/CalculateOutputGroups.swift new file mode 100644 index 0000000000..faa910d9aa --- /dev/null +++ b/tools/calculate_output_groups/CalculateOutputGroups.swift @@ -0,0 +1,31 @@ +import ArgumentParser +import Darwin +import ToolCommon + +@main +struct CalculateOutputGroups: AsyncParsableCommand { + @Argument( + help: "Value of the 'COLOR_DIAGNOSTICS' environment variable.", + transform: { $0 == "YES" } + ) + var colorDiagnostics: Bool + + @OptionGroup var arguments: OutputGroupsCalculator.Arguments + + func run() async throws { + let logger = DefaultLogger( + standardError: StderrOutputStream(), + standardOutput: StdoutOutputStream(), + colorize: colorDiagnostics + ) + + let calculator = OutputGroupsCalculator() + + do { + try await calculator.calculateOutputGroups(arguments: arguments) + } catch { + logger.logError(error.localizedDescription) + Darwin.exit(1) + } + } +} diff --git a/tools/calculate_output_groups/OutputGroupsCalculator.swift b/tools/calculate_output_groups/OutputGroupsCalculator.swift new file mode 100644 index 0000000000..04075ea007 --- /dev/null +++ b/tools/calculate_output_groups/OutputGroupsCalculator.swift @@ -0,0 +1,126 @@ +import Foundation +import ToolCommon +import ZippyJSON + +struct OutputGroupsCalculator { + func calculateOutputGroups(arguments: Arguments) async throws { + let pifCache = arguments.baseObjRoot + .appendingPathComponent("XCBuildData/PIFCache") + let projectCache = pifCache.appendingPathComponent("project") + let targetCache = pifCache.appendingPathComponent("target") + + let fileManager = FileManager.default + + guard fileManager.fileExists(atPath: projectCache.path) && + fileManager.fileExists(atPath: targetCache.path) + else { + throw UsageError(message: """ +error: PIFCache (\(pifCache)) doesn't exist. If you manually cleared Derived \ +Data, you need to close and re-open the project for the PIFCache to be created \ +again. Using the "Clean Build Folder" command instead (⇧ ⌘ K) won't trigger \ +this error. If this error still happens after re-opening the project, please \ +file a bug report here: \ +https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md +""") + } + + let projectURL = try Self.findProjectURL(in: projectCache) + let project = try Self.decodeProject(at: projectURL) + let targets = + try await Self.decodeTargets(project.targets, in: targetCache) + + dump(targets) + } + + static func findProjectURL(in projectCache: URL) throws -> URL { + let projectPIFsEnumerator = FileManager.default.enumerator( + at: projectCache, + includingPropertiesForKeys: [.contentModificationDateKey], + options: [ + .skipsHiddenFiles, + .skipsPackageDescendants, + .skipsSubdirectoryDescendants, + ] + )! + + var newestProjectPIF: URL? + var newestProjectPIFDate = Date.distantPast + for case let projectPIF as URL in projectPIFsEnumerator { + guard let resourceValues = try? projectPIF.resourceValues( + forKeys: [.contentModificationDateKey] + ), let modificationDate = resourceValues.contentModificationDate + else { + continue + } + + // TODO: The modification date is in the filename, should we use + // that instead? + if modificationDate > newestProjectPIFDate { + newestProjectPIF = projectPIF + newestProjectPIFDate = modificationDate + } + } + + guard let projectPIF = newestProjectPIF else { + throw UsageError(message: """ +error: Couldn't find a Project PIF at "\(projectCache)". Please file a bug \ +report here: https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md +""") + } + + return projectPIF + } + + static func decodeProject(at url: URL) throws -> ProjectPIF { + let decoder = ZippyJSONDecoder() + return try decoder.decode(ProjectPIF.self, from: Data(contentsOf: url)) + } + + static func decodeTargets( + _ targets: [String], + in targetCache: URL + ) async throws -> [TargetPIF] { + return try await withThrowingTaskGroup( + of: TargetPIF.self, + returning: [TargetPIF].self + ) { group in + for target in targets { + group.addTask { + let url = + targetCache.appendingPathComponent("\(target)-json") + let decoder = ZippyJSONDecoder() + return try decoder + .decode(TargetPIF.self, from: Data(contentsOf: url)) + } + } + + var targetPIFs: [TargetPIF] = [] + for try await target in group { + targetPIFs.append(target) + } + + return targetPIFs + } + } +} + +struct ProjectPIF: Decodable { + let targets: [String] +} + +struct TargetPIF: Decodable { + struct BuildConfiguration: Decodable { + let name: String + let buildSettings: [String: String] + } + + let guid: String + let buildConfigurations: [BuildConfiguration] +} + +struct Target { + let label: String + + // Maps Platform Name -> [Target ID] + let targetIds: [String: [String]] +} From c2f2f88b7b1b67d3e05a1df77538aa2d98953bdf Mon Sep 17 00:00:00 2001 From: Matt Pennig Date: Mon, 22 Apr 2024 15:39:54 -0500 Subject: [PATCH 2/5] WIP: swift calculate_output_groups Signed-off-by: Matt Pennig --- tools/calculate_output_groups/BUILD | 4 +- .../CalculateOutputGroups.swift | 1 + tools/calculate_output_groups/Errors.swift | 32 ++ tools/calculate_output_groups/Models.swift | 79 +++++ .../OutputGroupsCalculator.swift | 283 +++++++++++++----- .../URL+Extensions.swift | 30 ++ 6 files changed, 345 insertions(+), 84 deletions(-) create mode 100644 tools/calculate_output_groups/Errors.swift create mode 100644 tools/calculate_output_groups/Models.swift create mode 100644 tools/calculate_output_groups/URL+Extensions.swift diff --git a/tools/calculate_output_groups/BUILD b/tools/calculate_output_groups/BUILD index d30a3f3be6..1cf0f89bd7 100644 --- a/tools/calculate_output_groups/BUILD +++ b/tools/calculate_output_groups/BUILD @@ -18,7 +18,7 @@ load( # fix that for us. macos_command_line_application( name = "calculate_output_groups", - minimum_os_version = "12.0", + minimum_os_version = "13.0", visibility = ["//visibility:public"], deps = [":calculate_output_groups.library"], ) @@ -46,7 +46,7 @@ apple_universal_binary( "x86_64", "arm64", ], - minimum_os_version = "12.0", + minimum_os_version = "13.0", platform_type = "macos", visibility = ["//visibility:public"], ) diff --git a/tools/calculate_output_groups/CalculateOutputGroups.swift b/tools/calculate_output_groups/CalculateOutputGroups.swift index faa910d9aa..47f6cd179e 100644 --- a/tools/calculate_output_groups/CalculateOutputGroups.swift +++ b/tools/calculate_output_groups/CalculateOutputGroups.swift @@ -1,5 +1,6 @@ import ArgumentParser import Darwin +import Foundation import ToolCommon @main diff --git a/tools/calculate_output_groups/Errors.swift b/tools/calculate_output_groups/Errors.swift new file mode 100644 index 0000000000..43259eb900 --- /dev/null +++ b/tools/calculate_output_groups/Errors.swift @@ -0,0 +1,32 @@ +import ToolCommon + +extension UsageError { + static func buildMarker(_ path: String) -> Self { + .init(message: """ +error: Build marker (\(path)) doesn't exist. If you manually cleared Derived \ +Data, you need to close and re-open the project for the file to be created \ +again. Using the "Clean Build Folder" command instead (⇧ ⌘ K) won't trigger \ +this error. If this error still happens after re-opening the project, please \ +file a bug report here: \ +https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md +""") + } + + static func pifCache(_ path: String) -> Self { + .init(message: """ +error: PIFCache (\(path)) doesn't exist. If you manually cleared Derived \ +Data, you need to close and re-open the project for the PIFCache to be created \ +again. Using the "Clean Build Folder" command instead (⇧ ⌘ K) won't trigger \ +this error. If this error still happens after re-opening the project, please \ +file a bug report here: \ +https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md +""") + } + + static func buildRequest(_ path: String) -> Self { + .init(message: """ +error: Couldn't find a build-request.json file inside \(path)". Please file a bug \ +report here: https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md +""") + } +} diff --git a/tools/calculate_output_groups/Models.swift b/tools/calculate_output_groups/Models.swift new file mode 100644 index 0000000000..422d88266a --- /dev/null +++ b/tools/calculate_output_groups/Models.swift @@ -0,0 +1,79 @@ +enum PIF { + struct Project: Decodable { + let targets: [String] + } + + struct Target: Decodable { + struct BuildConfiguration: Decodable { + let name: String + let buildSettings: [String: String] + } + + let guid: String + let buildConfigurations: [BuildConfiguration] + } +} + +struct BuildRequest: Decodable { + let command: String = "build" // TODO: support other commands (e.g. "buildFiles") + let configurationName: String + let configuredTargets: [String] + let platform: String + + enum Root: CodingKey { + case configuredTargets + case parameters + + enum ConfiguredTargets: CodingKey { + case guid + } + enum Parameters: CodingKey { + case activeRunDestination + case configurationName + + enum ActiveRunDestination: CodingKey { + case platform + } + } + } + + init(from decoder: Decoder) throws { + let root = try decoder.container(keyedBy: Root.self) + let parameters = try root.nestedContainer(keyedBy: Root.Parameters.self, forKey: .parameters) + + // configurationName + self.configurationName = try parameters.decode(String.self, forKey: .configurationName) + + // configuredTargets + var configuredTargets = try root.nestedUnkeyedContainer(forKey: .configuredTargets) + var decodedTargets = [String]() + while !configuredTargets.isAtEnd { + let target = try configuredTargets.nestedContainer(keyedBy: Root.ConfiguredTargets.self) + decodedTargets.append(try target.decode(String.self, forKey: .guid)) + } + self.configuredTargets = decodedTargets + + // platform + let activeRunDestination = try parameters.nestedContainer(keyedBy: Root.Parameters.ActiveRunDestination.self, forKey: .activeRunDestination) + self.platform = try activeRunDestination.decode(String.self, forKey: .platform) + } +} + +enum Output { + typealias Map = [String: Target] + + struct Target: Codable { + struct Config: Codable { + struct Settings: Codable { + let base: [String] + var platforms: [String: Optional<[String]>] + } + + let build: Settings? + let buildFiles: Settings? + } + + let label: String + let configs: [String: Config] + } +} diff --git a/tools/calculate_output_groups/OutputGroupsCalculator.swift b/tools/calculate_output_groups/OutputGroupsCalculator.swift index 04075ea007..d3ce09ef5d 100644 --- a/tools/calculate_output_groups/OutputGroupsCalculator.swift +++ b/tools/calculate_output_groups/OutputGroupsCalculator.swift @@ -4,123 +4,242 @@ import ZippyJSON struct OutputGroupsCalculator { func calculateOutputGroups(arguments: Arguments) async throws { - let pifCache = arguments.baseObjRoot - .appendingPathComponent("XCBuildData/PIFCache") + let pifCache = arguments.baseObjRoot.appendingPathComponent("XCBuildData/PIFCache") let projectCache = pifCache.appendingPathComponent("project") let targetCache = pifCache.appendingPathComponent("target") - let fileManager = FileManager.default + guard let markerDate = arguments.buildMarkerFile.modificationDate else { + throw UsageError.buildMarker(arguments.buildMarkerFile.path) + } - guard fileManager.fileExists(atPath: projectCache.path) && - fileManager.fileExists(atPath: targetCache.path) + let fileManager = FileManager.default + guard + fileManager.fileExists(atPath: projectCache.path), + fileManager.fileExists(atPath: targetCache.path) else { - throw UsageError(message: """ -error: PIFCache (\(pifCache)) doesn't exist. If you manually cleared Derived \ -Data, you need to close and re-open the project for the PIFCache to be created \ -again. Using the "Clean Build Folder" command instead (⇧ ⌘ K) won't trigger \ -this error. If this error still happens after re-opening the project, please \ -file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md -""") + throw UsageError.pifCache(pifCache.path) + } + + async let buildRequest = loadBuildRequestFile( + inPath: arguments.baseObjRoot.appendingPathComponent("XCBuildData"), + since: markerDate + ) + async let targetMap = loadTargetMap( + fromBase: arguments.baseObjRoot, + projectCache: projectCache, + targetCache: targetCache + ) + + let output = try await outputGroups( + buildRequest: buildRequest, + targets: targetMap, + prefixes: arguments.outputGroupPrefixes + ) + print(output) + } + + private func loadBuildRequestFile(inPath path: URL, since: Date) async throws -> BuildRequest { + @Sendable func findBuildRequestURL() -> URL? { + path.newestDescendent(recursive: true, matching: { url in + guard + url.path.hasSuffix(".xcbuilddata/build-request.json"), + let date = url.modificationDate + else { return false } + return date >= since + }) + } + + if let url = findBuildRequestURL() { + return try url.decode(BuildRequest.self) + } + + // If the file was not immediately found, kick off a process to wait for the file to be created (or time out). + let findTask = Task { + while true { + try Task.checkCancellation() + try await Task.sleep(for: .seconds(1)) + if let buildRequestURL = findBuildRequestURL() { + return buildRequestURL + } + } + } + let timeoutTask = Task { + try await Task.sleep(for: .seconds(10)) + findTask.cancel() } - let projectURL = try Self.findProjectURL(in: projectCache) - let project = try Self.decodeProject(at: projectURL) - let targets = - try await Self.decodeTargets(project.targets, in: targetCache) + do { + let result = try await findTask.value + timeoutTask.cancel() + return try result.decode(BuildRequest.self) + } catch { + throw UsageError.buildRequest(path.path) + } + } - dump(targets) + private func loadTargetMap( + fromBase baseObjRoot: URL, + projectCache: URL, + targetCache: URL + ) async throws -> Output.Map { + let projectURL = try findProjectURL(in: projectCache) + let guidPayloadDir = baseObjRoot.appendingPathComponent("guid_payload") + try FileManager.default.createDirectory(at: guidPayloadDir, withIntermediateDirectories: true) + let guidPayloadFile = guidPayloadDir.appendingPathComponent(projectURL.lastPathComponent+"_v3.json") + let targets: Output.Map + do { + targets = try guidPayloadFile.decode(Output.Map.self) + } catch { + let project = try projectURL.decode(PIF.Project.self) + targets = try await decodeTargets(project.targets, in: targetCache) + let data = try JSONEncoder().encode(targets) + try data.write(to: guidPayloadFile) + } + return targets } - static func findProjectURL(in projectCache: URL) throws -> URL { - let projectPIFsEnumerator = FileManager.default.enumerator( - at: projectCache, - includingPropertiesForKeys: [.contentModificationDateKey], - options: [ - .skipsHiddenFiles, - .skipsPackageDescendants, - .skipsSubdirectoryDescendants, - ] - )! - - var newestProjectPIF: URL? - var newestProjectPIFDate = Date.distantPast - for case let projectPIF as URL in projectPIFsEnumerator { - guard let resourceValues = try? projectPIF.resourceValues( - forKeys: [.contentModificationDateKey] - ), let modificationDate = resourceValues.contentModificationDate - else { - continue + private func outputGroups( + buildRequest: BuildRequest, + targets: Output.Map, + prefixes: [String] + ) throws -> String { + var lines: [String] = [] + + for guid in buildRequest.configuredTargets { + guard + let target = targets[guid], + let config = target.configs[buildRequest.configurationName] + else { continue } + + var settings: Output.Target.Config.Settings? + switch buildRequest.command { + case "build": + settings = config.build + case "buildFiles": + settings = config.buildFiles + default: + break + } + guard let settings else { + throw PreconditionError(message: "Settings not found for target/command combination: \(guid) / \(buildRequest.command)") } - // TODO: The modification date is in the filename, should we use - // that instead? - if modificationDate > newestProjectPIFDate { - newestProjectPIF = projectPIF - newestProjectPIFDate = modificationDate + for platform in allPlatformsToSearch(buildRequest.platform) { + guard let platform = settings.platforms[platform] else { continue } + for prefix in prefixes { + for id in platform ?? settings.base { + lines.append("\(target.label)\n\(prefix) \(id)") + } + } } } - guard let projectPIF = newestProjectPIF else { - throw UsageError(message: """ -error: Couldn't find a Project PIF at "\(projectCache)". Please file a bug \ -report here: https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md -""") - } + return lines.joined(separator: "\n") + } - return projectPIF + // MARK: Helpers + + private func allPlatformsToSearch(_ platform: String) -> [String] { + if platform == "macosx" || platform.contains("simulator") { + return ["iphonesimulator", "appletvsimulator", "watchsimulator", "macosx"] + } else { + return ["iphoneos", "appletvos", "watchos", "macosx"] + } } - static func decodeProject(at url: URL) throws -> ProjectPIF { - let decoder = ZippyJSONDecoder() - return try decoder.decode(ProjectPIF.self, from: Data(contentsOf: url)) + private func findProjectURL(in projectCache: URL) throws -> URL { + guard let projectPIF = projectCache.newestDescendent() else { + throw UsageError.pifCache(projectCache.path) + } + return projectPIF } - static func decodeTargets( + private func decodeTargets( _ targets: [String], in targetCache: URL - ) async throws -> [TargetPIF] { - return try await withThrowingTaskGroup( - of: TargetPIF.self, - returning: [TargetPIF].self + ) async throws -> Output.Map { + try await withThrowingTaskGroup( + of: PIF.Target.self, + returning: Output.Map.self ) { group in + let decoder = ZippyJSONDecoder() for target in targets { group.addTask { - let url = - targetCache.appendingPathComponent("\(target)-json") - let decoder = ZippyJSONDecoder() - return try decoder - .decode(TargetPIF.self, from: Data(contentsOf: url)) + let url = targetCache.appendingPathComponent("\(target)-json") + return try decoder.decode(PIF.Target.self, from: Data(contentsOf: url)) } } - - var targetPIFs: [TargetPIF] = [] - for try await target in group { - targetPIFs.append(target) + return try await group.reduce(into: Output.Map()) { map, target in + map[target.guid] = target.output } - - return targetPIFs } } } -struct ProjectPIF: Decodable { - let targets: [String] +extension PIF.Target { + var output: Output.Target? { + guard let label = buildConfigurations.lazy.compactMap(\.label).first else { return nil } + return .init( + label: label, + configs: Dictionary(uniqueKeysWithValues: zip( + buildConfigurations.map(\.name), + buildConfigurations.map(\.output) + )) + ) + } } - -struct TargetPIF: Decodable { - struct BuildConfiguration: Decodable { - let name: String - let buildSettings: [String: String] +extension PIF.Target.BuildConfiguration { + var label: String? { + buildSettings["BAZEL_LABEL"] } - let guid: String - let buildConfigurations: [BuildConfiguration] -} + var output: Output.Target.Config { + var build: Output.Target.Config.Settings? + if let value = buildSettings["BAZEL_TARGET_ID"] { + build = .init(base: [value], platforms: [:]) + } + var buildFiles: Output.Target.Config.Settings? + if let value = buildSettings["BAZEL_COMPILE_TARGET_IDS"] { + buildFiles = .init(base: compileTargetIds(value), platforms: [:]) + } + if build != nil || buildFiles != nil { + for (key, value) in buildSettings { + if build != nil, key.starts(with: "BAZEL_TARGET_ID[sdk=") { + let platform = String(key.dropFirst(20).dropLast(2)) + if value == "$(BAZEL_TARGET_ID)" { + // This value indicates that the provided platform inherits from the base build setting. Store nil for later processing. + build?.platforms[platform] = nil + } else { + build?.platforms[platform] = [value] + } + } + if buildFiles != nil, key.starts(with: "BAZEL_COMPILE_TARGET_IDS[sdk=") { + let platform = String(key.dropFirst(29).dropLast(2)) + if value == "$(BAZEL_COMPILE_TARGET_IDS)" { + // This value indicates that the provided platform inherits from the base build setting. Store nil for later processing. + buildFiles?.platforms[platform] = nil + } else { + buildFiles?.platforms[platform] = compileTargetIds(value) + } + } + } + } -struct Target { - let label: String + return .init(build: build, buildFiles: buildFiles) + } - // Maps Platform Name -> [Target ID] - let targetIds: [String: [String]] + private func compileTargetIds(_ value: String) -> [String] { + var seenSpace = false + // value is a space-separated list of space-separated pairs. split into an array of pairs. + return value.split(whereSeparator: { + guard $0 == " " else { return false } + if seenSpace { + seenSpace = false + return true + } else { + seenSpace = true + } + return false + }).map(String.init) + } } diff --git a/tools/calculate_output_groups/URL+Extensions.swift b/tools/calculate_output_groups/URL+Extensions.swift new file mode 100644 index 0000000000..cd35229e52 --- /dev/null +++ b/tools/calculate_output_groups/URL+Extensions.swift @@ -0,0 +1,30 @@ +import Foundation +import ZippyJSON + +extension URL { + func decode(_ type: T.Type) throws -> T { + try ZippyJSONDecoder().decode(T.self, from: Data(contentsOf: self)) + } + + var modificationDate: Date? { + try? resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate + } + + func newestDescendent(recursive: Bool = false, matching: (URL)->Bool={ _ in true }) -> URL? { + let options: FileManager.DirectoryEnumerationOptions = recursive ? [] : [.skipsPackageDescendants, .skipsSubdirectoryDescendants] + let enumerator = FileManager.default.enumerator( + at: self, + includingPropertiesForKeys: [.contentModificationDateKey], + options: options.union(.skipsHiddenFiles) + )! + + return enumerator.compactMap({ $0 as? URL }).filter(matching).max { + guard + let first = try? $0.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate, + let second = try? $1.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate + else { return false } + + return first < second + } + } +} From 4221a8403363d1e6769d1181cc7d98383ff9b066 Mon Sep 17 00:00:00 2001 From: Matt Pennig Date: Tue, 23 Apr 2024 13:58:03 -0500 Subject: [PATCH 3/5] Convert calculate-output-groups to Swift Signed-off-by: Matt Pennig --- distribution/BUILD | 3 + .../fixtures/bwb.xcodeproj/project.pbxproj | 2 - .../fixtures/bwx.xcodeproj/project.pbxproj | 2 - .../fixtures/bwb.xcodeproj/project.pbxproj | 1 - .../CalculateOutputGroups.swift | 8 +- tools/calculate_output_groups/Errors.swift | 6 +- .../OutputGroupsCalculator.swift | 58 ++- .../AddBazelDependenciesTarget.swift | 3 - .../Generator/SetTargetConfigurations.swift | 2 +- .../CalculateSharedBuildSettings.swift | 2 +- tools/generators/pbxproj_prefix/README.md | 2 - .../BazelDependenciesBuildSettings.swift | 1 - .../BazelDependenciesBuildSettingsTests.swift | 1 - .../internal/bazel_integration_files/BUILD | 42 +- .../calculate_output_groups.py | 429 ------------------ .../generate_bazel_dependencies.sh | 3 +- .../templates/incremental_installer.sh | 2 +- .../internal/templates/legacy_installer.sh | 2 +- 18 files changed, 81 insertions(+), 488 deletions(-) delete mode 100755 xcodeproj/internal/bazel_integration_files/calculate_output_groups.py diff --git a/distribution/BUILD b/distribution/BUILD index ee4d7ed051..80da6cab88 100644 --- a/distribution/BUILD +++ b/distribution/BUILD @@ -59,6 +59,9 @@ pkg_tar( remap_paths = dicts.add( { "MODULE.release.bazel": "MODULE.bazel", + "tools/calculate_output_groups/BUILD.release.bazel": ( + "tools/calculate_output_groups/BUILD" + ), "tools/import_indexstores/BUILD.release.bazel": ( "tools/import_indexstores/BUILD" ), diff --git a/examples/integration/test/fixtures/bwb.xcodeproj/project.pbxproj b/examples/integration/test/fixtures/bwb.xcodeproj/project.pbxproj index 0e119b5d5e..cc3513714f 100644 --- a/examples/integration/test/fixtures/bwb.xcodeproj/project.pbxproj +++ b/examples/integration/test/fixtures/bwb.xcodeproj/project.pbxproj @@ -16274,7 +16274,6 @@ isa = XCBuildConfiguration; buildSettings = { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; @@ -18762,7 +18761,6 @@ isa = XCBuildConfiguration; buildSettings = { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; diff --git a/examples/integration/test/fixtures/bwx.xcodeproj/project.pbxproj b/examples/integration/test/fixtures/bwx.xcodeproj/project.pbxproj index 141f3c9485..ff5f60bdb5 100644 --- a/examples/integration/test/fixtures/bwx.xcodeproj/project.pbxproj +++ b/examples/integration/test/fixtures/bwx.xcodeproj/project.pbxproj @@ -22921,7 +22921,6 @@ isa = XCBuildConfiguration; buildSettings = { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; @@ -24815,7 +24814,6 @@ isa = XCBuildConfiguration; buildSettings = { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; diff --git a/examples/rules_ios/test/fixtures/bwb.xcodeproj/project.pbxproj b/examples/rules_ios/test/fixtures/bwb.xcodeproj/project.pbxproj index d8ce0c2ae0..b924aa31fe 100644 --- a/examples/rules_ios/test/fixtures/bwb.xcodeproj/project.pbxproj +++ b/examples/rules_ios/test/fixtures/bwb.xcodeproj/project.pbxproj @@ -4351,7 +4351,6 @@ isa = XCBuildConfiguration; buildSettings = { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; diff --git a/tools/calculate_output_groups/CalculateOutputGroups.swift b/tools/calculate_output_groups/CalculateOutputGroups.swift index 47f6cd179e..56fdfaecdd 100644 --- a/tools/calculate_output_groups/CalculateOutputGroups.swift +++ b/tools/calculate_output_groups/CalculateOutputGroups.swift @@ -14,16 +14,18 @@ struct CalculateOutputGroups: AsyncParsableCommand { @OptionGroup var arguments: OutputGroupsCalculator.Arguments func run() async throws { + var output = StdoutOutputStream() let logger = DefaultLogger( standardError: StderrOutputStream(), - standardOutput: StdoutOutputStream(), + standardOutput: output, colorize: colorDiagnostics ) - let calculator = OutputGroupsCalculator() + let calculator = OutputGroupsCalculator(logger: logger) do { - try await calculator.calculateOutputGroups(arguments: arguments) + let groups = try await calculator.calculateOutputGroups(arguments: arguments) + print(groups, to: &output) } catch { logger.logError(error.localizedDescription) Darwin.exit(1) diff --git a/tools/calculate_output_groups/Errors.swift b/tools/calculate_output_groups/Errors.swift index 43259eb900..dec5a45fd2 100644 --- a/tools/calculate_output_groups/Errors.swift +++ b/tools/calculate_output_groups/Errors.swift @@ -3,7 +3,7 @@ import ToolCommon extension UsageError { static func buildMarker(_ path: String) -> Self { .init(message: """ -error: Build marker (\(path)) doesn't exist. If you manually cleared Derived \ +Build marker (\(path)) doesn't exist. If you manually cleared Derived \ Data, you need to close and re-open the project for the file to be created \ again. Using the "Clean Build Folder" command instead (⇧ ⌘ K) won't trigger \ this error. If this error still happens after re-opening the project, please \ @@ -14,7 +14,7 @@ https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bu static func pifCache(_ path: String) -> Self { .init(message: """ -error: PIFCache (\(path)) doesn't exist. If you manually cleared Derived \ +PIFCache (\(path)) doesn't exist. If you manually cleared Derived \ Data, you need to close and re-open the project for the PIFCache to be created \ again. Using the "Clean Build Folder" command instead (⇧ ⌘ K) won't trigger \ this error. If this error still happens after re-opening the project, please \ @@ -25,7 +25,7 @@ https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bu static func buildRequest(_ path: String) -> Self { .init(message: """ -error: Couldn't find a build-request.json file inside \(path)". Please file a bug \ +Couldn't find latest build-request.json file after 30 seconds. Please file a bug \ report here: https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md """) } diff --git a/tools/calculate_output_groups/OutputGroupsCalculator.swift b/tools/calculate_output_groups/OutputGroupsCalculator.swift index d3ce09ef5d..dde901e08b 100644 --- a/tools/calculate_output_groups/OutputGroupsCalculator.swift +++ b/tools/calculate_output_groups/OutputGroupsCalculator.swift @@ -3,7 +3,9 @@ import ToolCommon import ZippyJSON struct OutputGroupsCalculator { - func calculateOutputGroups(arguments: Arguments) async throws { + let logger: Logger + + func calculateOutputGroups(arguments: Arguments) async throws -> String { let pifCache = arguments.baseObjRoot.appendingPathComponent("XCBuildData/PIFCache") let projectCache = pifCache.appendingPathComponent("project") let targetCache = pifCache.appendingPathComponent("target") @@ -19,7 +21,6 @@ struct OutputGroupsCalculator { else { throw UsageError.pifCache(pifCache.path) } - async let buildRequest = loadBuildRequestFile( inPath: arguments.baseObjRoot.appendingPathComponent("XCBuildData"), since: markerDate @@ -30,23 +31,31 @@ struct OutputGroupsCalculator { targetCache: targetCache ) - let output = try await outputGroups( + return try await outputGroups( buildRequest: buildRequest, targets: targetMap, prefixes: arguments.outputGroupPrefixes ) - print(output) } private func loadBuildRequestFile(inPath path: URL, since: Date) async throws -> BuildRequest { @Sendable func findBuildRequestURL() -> URL? { - path.newestDescendent(recursive: true, matching: { url in + guard let xcbuilddata = path.newestDescendent(matching: { url in guard - url.path.hasSuffix(".xcbuilddata/build-request.json"), + url.path.hasSuffix(".xcbuilddata"), let date = url.modificationDate else { return false } return date >= since - }) + }) else { + return nil + } + + let buildRequest = xcbuilddata.appendingPathComponent("build-request.json") + if FileManager.default.fileExists(atPath: buildRequest.path) { + return buildRequest + } else { + return nil + } } if let url = findBuildRequestURL() { @@ -54,22 +63,33 @@ struct OutputGroupsCalculator { } // If the file was not immediately found, kick off a process to wait for the file to be created (or time out). - let findTask = Task { - while true { - try Task.checkCancellation() - try await Task.sleep(for: .seconds(1)) - if let buildRequestURL = findBuildRequestURL() { - return buildRequestURL + do { + let findTask = Task { + logger.logWarning("The latest build-request.json file has not been updated yet. Waiting…") + while true { + try Task.checkCancellation() + try await Task.sleep(for: .seconds(1)) + if let buildRequestURL = findBuildRequestURL() { + return buildRequestURL + } } } - } - let timeoutTask = Task { - try await Task.sleep(for: .seconds(10)) - findTask.cancel() - } + let waitingTask = Task { + try await Task.sleep(for: .seconds(10)) + try Task.checkCancellation() + logger.logWarning(""" +The latest build-request.json file has still not been updated after 10 seconds. If this happens frequently, please file a bug report here: +https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md +""") + } + let timeoutTask = Task { + try await Task.sleep(for: .seconds(30)) + guard !Task.isCancelled else { return } + findTask.cancel() + } - do { let result = try await findTask.value + waitingTask.cancel() timeoutTask.cancel() return try result.decode(BuildRequest.self) } catch { diff --git a/tools/generators/legacy/src/Generator/AddBazelDependenciesTarget.swift b/tools/generators/legacy/src/Generator/AddBazelDependenciesTarget.swift index f9698f326d..071bb4816f 100644 --- a/tools/generators/legacy/src/Generator/AddBazelDependenciesTarget.swift +++ b/tools/generators/legacy/src/Generator/AddBazelDependenciesTarget.swift @@ -35,9 +35,6 @@ extension Generator { var buildSettings: BuildSettings = [ "BAZEL_PACKAGE_BIN_DIR": "rules_xcodeproj", - "CALCULATE_OUTPUT_GROUPS_SCRIPT": """ -$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py -""", "INDEXING_SUPPORTED_PLATFORMS__": """ $(INDEXING_SUPPORTED_PLATFORMS__NO) """, diff --git a/tools/generators/legacy/src/Generator/SetTargetConfigurations.swift b/tools/generators/legacy/src/Generator/SetTargetConfigurations.swift index 5809329ab1..f950d4af36 100644 --- a/tools/generators/legacy/src/Generator/SetTargetConfigurations.swift +++ b/tools/generators/legacy/src/Generator/SetTargetConfigurations.swift @@ -283,7 +283,7 @@ $(BAZEL_OUT)\#(linkParams.path.string.dropFirst(9)) buildSettings.set("TARGET_NAME", to: target.name) if !target.product.isResourceBundle { - // This is used in `calculate_output_groups.py`. We only want to set + // This is used in `calculate_output_groups`. We only want to set // it on buildable targets buildSettings.set("BAZEL_LABEL", to: target.label.description) } diff --git a/tools/generators/pbxnativetargets/src/Generator/CalculateSharedBuildSettings.swift b/tools/generators/pbxnativetargets/src/Generator/CalculateSharedBuildSettings.swift index 1a3506406c..4b38f814fc 100644 --- a/tools/generators/pbxnativetargets/src/Generator/CalculateSharedBuildSettings.swift +++ b/tools/generators/pbxnativetargets/src/Generator/CalculateSharedBuildSettings.swift @@ -114,7 +114,7 @@ extension Generator.CalculateSharedBuildSettings { } if productType != .resourceBundle { - // This is used in `calculate_output_groups.py`. We only want to set + // This is used in `calculate_output_groups`. We only want to set // it on buildable targets. buildSettings.append( .init( diff --git a/tools/generators/pbxproj_prefix/README.md b/tools/generators/pbxproj_prefix/README.md index 3c2624cf2c..056d6dfc0e 100644 --- a/tools/generators/pbxproj_prefix/README.md +++ b/tools/generators/pbxproj_prefix/README.md @@ -121,7 +121,6 @@ Here is an example output: isa = XCBuildConfiguration; buildSettings = { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; @@ -144,7 +143,6 @@ Here is an example output: isa = XCBuildConfiguration; buildSettings = { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; diff --git a/tools/generators/pbxproj_prefix/src/Generator/BazelDependenciesBuildSettings.swift b/tools/generators/pbxproj_prefix/src/Generator/BazelDependenciesBuildSettings.swift index eb2a9d1512..04bef6e842 100644 --- a/tools/generators/pbxproj_prefix/src/Generator/BazelDependenciesBuildSettings.swift +++ b/tools/generators/pbxproj_prefix/src/Generator/BazelDependenciesBuildSettings.swift @@ -24,7 +24,6 @@ extension Generator { return #""" { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; diff --git a/tools/generators/pbxproj_prefix/test/BazelDependenciesBuildSettingsTests.swift b/tools/generators/pbxproj_prefix/test/BazelDependenciesBuildSettingsTests.swift index 583349943b..374a565150 100644 --- a/tools/generators/pbxproj_prefix/test/BazelDependenciesBuildSettingsTests.swift +++ b/tools/generators/pbxproj_prefix/test/BazelDependenciesBuildSettingsTests.swift @@ -20,7 +20,6 @@ class BazelDependenciesBuildSettingsTests: XCTestCase { let expectedBuildSettings = #""" { BAZEL_PACKAGE_BIN_DIR = rules_xcodeproj; - CALCULATE_OUTPUT_GROUPS_SCRIPT = "$(BAZEL_INTEGRATION_DIR)/calculate_output_groups.py"; CC = ""; CXX = ""; INDEXING_SUPPORTED_PLATFORMS__ = "$(INDEXING_SUPPORTED_PLATFORMS__NO)"; diff --git a/xcodeproj/internal/bazel_integration_files/BUILD b/xcodeproj/internal/bazel_integration_files/BUILD index 287a22049d..c57850251f 100644 --- a/xcodeproj/internal/bazel_integration_files/BUILD +++ b/xcodeproj/internal/bazel_integration_files/BUILD @@ -1,10 +1,10 @@ _BASE_FILES = [ - "calculate_output_groups.py", "copy_dsyms.sh", "create_lldbinit.sh", "generate_bazel_dependencies.sh", - ":renamed_import_indexstores", "process_bazel_build_log.py", + ":renamed_calculate_output_groups", + ":renamed_import_indexstores", ] filegroup( @@ -92,19 +92,34 @@ echo '/*.framework/SwiftUIPreviewsFrameworks/***' >> "framework.exclude.rsynclis tags = ["manual"], ) -genrule( - name = "renamed_import_indexstores", - srcs = ["//tools/import_indexstores:universal_import_indexstores"], - outs = ["import_indexstores"], - # Make `import_indexstores` have the right name - cmd = """\ +rename_command = """\ readonly output="$@" if [[ $$(stat -f '%d' "$<") == $$(stat -f '%d' "$${output%/*}") ]]; then cp -c "$<" "$@" else cp "$<" "$@" fi -""", +""" + +genrule( + name = "renamed_calculate_output_groups", + srcs = ["//tools/calculate_output_groups:universal_calculate_output_groups"], + outs = ["calculate_output_groups"], + # Make `calculate_output_groups` have the right name + cmd = rename_command, + message = "Renaming calculate_output_groups", + tags = [ + "manual", + "no-sandbox", + ], +) + +genrule( + name = "renamed_import_indexstores", + srcs = ["//tools/import_indexstores:universal_import_indexstores"], + outs = ["import_indexstores"], + # Make `import_indexstores` have the right name + cmd = rename_command, message = "Renaming import_indexstores", tags = [ "manual", @@ -117,14 +132,7 @@ genrule( srcs = ["//tools/swiftc_stub:universal_swiftc_stub"], outs = ["swiftc"], # Make `swiftc_stub` have the right name - cmd = """\ -readonly output="$@" -if [[ $$(stat -f '%d' "$<") == $$(stat -f '%d' "$${output%/*}") ]]; then - cp -c "$<" "$@" -else - cp "$<" "$@" -fi -""", + cmd = rename_command, message = "Renaming swiftc_stub", tags = [ "manual", diff --git a/xcodeproj/internal/bazel_integration_files/calculate_output_groups.py b/xcodeproj/internal/bazel_integration_files/calculate_output_groups.py deleted file mode 100755 index 2146c803d8..0000000000 --- a/xcodeproj/internal/bazel_integration_files/calculate_output_groups.py +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/python3 - -import datetime -import glob -import json -import os -import sys -import time -import traceback - -# Ordered the same as we order platforms in the generated Xcode project, except -# macOS is last -_DEVICE_PLATFORMS = { - "iphoneos": None, - "appletvos": None, - "watchos": None, - "macosx": None, -} -_SIMULATOR_PLATFORMS = { - "iphonesimulator": None, - "appletvsimulator": None, - "watchsimulator": None, - "macosx": None, -} - - -def _wait_for_value(calculate_value, value_name): - wait_counter = 0 - while True: - value = calculate_value() - if value: - break - if wait_counter == 0: - now = datetime.datetime.now().strftime('%H:%M:%S') - print( - f"note: ({now}) {value_name} not updated yet, waiting...", - file = sys.stderr, - flush = True, - ) - if wait_counter == 10: - now = datetime.datetime.now().strftime('%H:%M:%S') - print( - f"""\ -warning: ({now}) {value_name} still not updated after 10 seconds. If happens \ -frequently, or the cache is never created, please file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md""", - file = sys.stderr, - flush = True, - ) - time.sleep(1) - wait_counter += 1 - if wait_counter > 0: - now = datetime.datetime.now().strftime('%H:%M:%S') - print( - f"""\ -note: ({now}) {value_name} updated after {wait_counter} seconds.""", - file = sys.stderr, - flush = True, - ) - return value - - -def _get_build_request( - xcode_version, - objroot, - build_request_min_ctime): - if xcode_version < 1430: - # Before Xcode 14.3 - def wait_for_description(): - build_description_cache = max( - glob.iglob( - f"{objroot}/XCBuildData/BuildDescriptionCacheIndex-*", - ), - key = os.path.getctime, - ) - if (os.path.getctime(build_description_cache) >= - build_request_min_ctime): - return build_description_cache - return None - - build_description_cache = ( - _wait_for_value(wait_for_description, "BuildDescriptionCacheIndex") - ) - with open(build_description_cache, 'rb') as f: - f.seek(-32, os.SEEK_END) - build_request_id = f.read().decode('ASCII') - - build_request_file = ( - f"{objroot}/XCBuildData/{build_request_id}-buildRequest.json" - ) - def wait_for_build_request_file(): - if os.path.exists(build_request_file): - with open(build_request_file, encoding = "utf-8") as f: - # Parse the build-request.json file - try: - return json.load(f) - except Exception as error: - print( - f"""\ -error: Failed to parse '{build_request_file}': -{type(error).__name__}: {error}. - -Please file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md""", - file = sys.stderr, - ) - exit(1) - return None - - return _wait_for_value( - wait_for_build_request_file, - f"\"{build_request_file}\"", - ) - - def wait_for_build_request(): - xcbuilddata = max( - glob.iglob(f"{objroot}/XCBuildData/*.xcbuilddata"), - key = os.path.getctime, - ) - if os.path.getctime(xcbuilddata) >= build_request_min_ctime: - build_request_file = f"{xcbuilddata}/build-request.json" - if os.path.exists(build_request_file): - with open(build_request_file, encoding = "utf-8") as f: - # Parse the build-request.json file - try: - return json.load(f) - except Exception as error: - print( - f"""\ -error: Failed to parse '{build_request_file}': -{type(error).__name__}: {error}. - -Please file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md""", - file = sys.stderr, - ) - exit(1) - return None - - return _wait_for_value( - wait_for_build_request, - "newest 'buildRequest.json' file", - ) - - -def _calculate_label_and_target_ids( - build_request, - guid_labels, - guid_target_ids): - # Xcode gets "stuck" in the `buildFiles` or `build` command for - # top-level targets, so we can't reliably change commands here. Leaving - # the code in place in case this is fixed in the future, or we want to - # do something similar in an XCBBuildService proxy. - # - # command = ( - # build_request.get("_buildCommand2", {}).get("command", "build") - # ) - command = "build" - parameters = build_request["parameters"] - platform = ( - parameters["activeRunDestination"]["platform"] - ) - configuration_name = parameters["configurationName"] - - labels_and_target_ids = [] - for target in build_request["configuredTargets"]: - label = guid_labels.get(target["guid"]) - if not label: - # `BazelDependency` and the like - continue - full_target_target_ids = guid_target_ids[target["guid"]] - target_target_ids = ( - full_target_target_ids.get(command) or - # Will only be `null` if `command == "buildFiles"` and there - # isn't a different compile target id - full_target_target_ids["build"] - ) - target_ids = _select_target_ids( - target_target_ids[configuration_name], - platform, - ) - for target_id in target_ids: - labels_and_target_ids.append((label, target_id)) - - if not labels_and_target_ids: - print( - """\ -error: Failed to determine labels and targets. Note, currently `.xcworkspace`s \ -aren't supported. Please make sure you are opening the generated \ -`.xcodeproj` file bundle directly. If you are, try using the "Clean Build \ -Folder" command instead (⇧ ⌘ K). If you still get this error after that, then \ -please file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md""", - file = sys.stderr, - ) - exit(1) - - return labels_and_target_ids - - -def _calculate_guid_labels_and_target_ids(base_objroot): - pif_cache = f"{base_objroot}/XCBuildData/PIFCache" - project_cache = f"{pif_cache}/project" - target_cache = f"{pif_cache}/target" - - # The PIF cache will only be created before the `SetSessionUserInfo` - # command, which normally happens when a project is opened. If Derived Data - # is cleared while the project is open - if not (os.path.exists(project_cache) and os.path.exists(target_cache)): - print( - f"""\ -error: PIFCache ({pif_cache}) doesn't exist. If you manually cleared Derived \ -Data, you need to close and re-open the project for the PIFCache to be created \ -again. Using the "Clean Build Folder" command instead (⇧ ⌘ K) won't trigger \ -this error. If this error still happens after re-opening the project, please \ -file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md""", - file = sys.stderr, - ) - sys.exit(1) - - project_pif = max( - glob.iglob(f"{project_cache}/*"), - key = os.path.getctime, - ) - - guid_payload_parent = f"{base_objroot}/guid_payload" - guid_payload_path = f"""\ -{guid_payload_parent}/{os.path.basename(project_pif)}_v2.json""" - - if os.path.exists(guid_payload_path): - with open(guid_payload_path, encoding = "utf-8") as f: - payload = json.load(f) - return payload["labels"], payload["targetIds"] - - with open(project_pif, encoding = "utf-8") as f: - project_pif = json.load(f) - - targets = project_pif["targets"] - - guid_labels = {} - guid_target_ids = {} - for target_name in targets: - target_file = f"{target_cache}/{target_name}-json" - with open(target_file, encoding = "utf-8") as f: - target_pif = json.load(f) - - label = None - build_target_ids = {} - compile_target_ids = {} - for configuration in target_pif["buildConfigurations"]: - config_build_target_ids = {"key": "BAZEL_TARGET_ID"} - config_compile_target_ids = {"key": "BAZEL_COMPILE_TARGET_IDS"} - for key, value in configuration["buildSettings"].items(): - if key.startswith("BAZEL_TARGET_ID"): - # This uses a list in the case where the value isn't meant to - # be inherited. Otherwise, `$(BAZEL_TARGET_ID)` is used later - # in this processing to pull from an inherited setting. - if value == "$(BAZEL_TARGET_ID)": - target_ids = value - else: - # This is only a single value but the later parsing of - # these target ids assumes a list. - target_ids = [value] - - config_build_target_ids[_platform_from_build_key(key)] = ( - target_ids - ) - elif key.startswith("BAZEL_COMPILE_TARGET_IDS"): - # This uses a list in the case where the value isn't meant to - # be inherited. Otherwise, `$(BAZEL_COMPILE_TARGET_IDS)` is - # used later in this processing to pull from an inherited - # setting. - if value == "$(BAZEL_COMPILE_TARGET_IDS)": - target_ids = "$(BAZEL_COMPILE_TARGET_IDS)" - else: - # Target identifiers contain a space but are space separated. - # Split on all spaces then rejoin across the identifier pairs. - target_ids = value.split(" ") - target_ids = [ - " ".join(target_ids[i:i+2]) - for i in range(0, len(target_ids), 2) - ] - - config_compile_target_ids[_platform_from_compile_key(key)] = ( - target_ids - ) - elif key == "BAZEL_LABEL": - label = value - configuration_name = configuration["name"] - build_target_ids[configuration_name] = config_build_target_ids - compile_target_ids[configuration_name] = config_compile_target_ids - - if not label: - # `BazelDependency` and the like - continue - - target_ids = { - "build": build_target_ids, - } - if len(compile_target_ids) > 1: - target_ids["buildFiles"] = compile_target_ids - - guid = target_pif["guid"] - guid_labels[guid] = label - guid_target_ids[guid] = target_ids - - os.makedirs(guid_payload_parent, exist_ok = True) - with open(guid_payload_path, "w", encoding = "utf-8") as f: - payload = { - "labels": guid_labels, - "targetIds": guid_target_ids, - } - json.dump(payload, f) - - return guid_labels, guid_target_ids - - -def _platform_from_build_key(key): - if key.startswith("BAZEL_TARGET_ID[sdk="): - return key[20:-2] - return "" - - -def _platform_from_compile_key(key): - if key.startswith("BAZEL_COMPILE_TARGET_IDS[sdk="): - return key[29:-2] - return "" - - -def _select_target_ids(target_ids, platform): - key = target_ids["key"] - - platforms = {platform: None} - - # We need to try other similar platforms (i.e. other simulator platforms if - # `platform`` is for a simulator). This is to support schemes with targets - # of multiple platforms in them. Because `dict` is insertion ordered, - # `platform` will be checked first. - platforms.update(_similar_platforms(platform)) - - for platform in platforms: - platform_target_ids = target_ids.get(platform) - if platform_target_ids: - if platform_target_ids == f"$({key})": - return target_ids[""] - return platform_target_ids - return target_ids[""] - - -def _similar_platforms(platform): - if platform == "macosx" or "simulator" in platform: - return _SIMULATOR_PLATFORMS - return _DEVICE_PLATFORMS - - -def _main( - xcode_version, - objroot, - base_objroot, - marker_file, - prefixes_str): - if not os.path.exists(marker_file): - return - - build_request_min_ctime = os.path.getctime(marker_file) - - try: - xcode_version = int(xcode_version) - except ValueError: - print( - """ -warning: xcode_version was not an integer. Please file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md -""", - file = sys.stderr, - ) - xcode_version = 9999 - - prefixes = prefixes_str.split(",") - - try: - build_request = _get_build_request( - xcode_version, - objroot, - build_request_min_ctime, - ) - guid_labels, guid_target_ids = ( - _calculate_guid_labels_and_target_ids(base_objroot) - ) - except Exception: - print( - f"""\ -error: Failed to calculate labels and target ids from PIFCache: -{traceback.format_exc()} -Please file a bug report here: \ -https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bug.md""", - file = sys.stderr, - ) - sys.exit(1) - - labels_and_target_ids = _calculate_label_and_target_ids( - build_request, - guid_labels, - guid_target_ids, - ) - - print("\n".join( - [ - f"{label}\n{prefix} {id}" - for label, id in labels_and_target_ids - for prefix in prefixes - ], - )) - - -if __name__ == "__main__": - _main( - # XCODE_VERSION_ACTUAL - sys.argv[1], - # non_preview_objroot - sys.argv[2], - # base_objroot - sys.argv[3], - # build_marker_file - sys.argv[4], - # output_group_prefixes - sys.argv[5], - ) diff --git a/xcodeproj/internal/bazel_integration_files/generate_bazel_dependencies.sh b/xcodeproj/internal/bazel_integration_files/generate_bazel_dependencies.sh index d4238ade23..6d630d7d00 100755 --- a/xcodeproj/internal/bazel_integration_files/generate_bazel_dependencies.sh +++ b/xcodeproj/internal/bazel_integration_files/generate_bazel_dependencies.sh @@ -49,7 +49,8 @@ else # writes to it can happen during indexing, which breaks the off-by-one-by-design # nature of it IFS=$'\n' read -r -d '' -a labels_and_output_groups < \ - <( "$CALCULATE_OUTPUT_GROUPS_SCRIPT" \ + <( "$BAZEL_INTEGRATION_DIR/calculate_output_groups" \ + "$COLOR_DIAGNOSTICS" \ "$XCODE_VERSION_ACTUAL" \ "$non_preview_objroot" \ "$base_objroot" \ diff --git a/xcodeproj/internal/templates/incremental_installer.sh b/xcodeproj/internal/templates/incremental_installer.sh index 5fc7e20b99..6a5c10f617 100644 --- a/xcodeproj/internal/templates/incremental_installer.sh +++ b/xcodeproj/internal/templates/incremental_installer.sh @@ -122,7 +122,7 @@ find "$dest/rules_xcodeproj/bazel" \ -type f \( -name "*.sh" -o -name "*.py" -o -name "ld" -o -name "libtool" \) \ -print0 | xargs -0 chmod u+x find "$dest/rules_xcodeproj/bazel" \ - -type f ! \( -name "swiftc" -o -name "ld" -o -name "libtool" -o -name "import_indexstores" -o -name "*.sh" -o -name "*.py" \) \ + -type f ! \( -name "swiftc" -o -name "ld" -o -name "libtool" -o -name "import_indexstores" -o -name "calculate_output_groups" -o -name "*.sh" -o -name "*.py" \) \ -print0 | xargs -0 chmod -x # Copy over `project.xcworkspace/contents.xcworkspacedata` if needed diff --git a/xcodeproj/internal/templates/legacy_installer.sh b/xcodeproj/internal/templates/legacy_installer.sh index e458acc435..b604380023 100644 --- a/xcodeproj/internal/templates/legacy_installer.sh +++ b/xcodeproj/internal/templates/legacy_installer.sh @@ -177,7 +177,7 @@ find "$dest/rules_xcodeproj/bazel" \ -type f \( -name "*.sh" -o -name "*.py" \) \ -print0 | xargs -0 chmod u+x find "$dest/rules_xcodeproj/bazel" \ - -type f ! \( -name "swiftc" -o -name "ld" -o -name "libtool" -o -name "import_indexstores" -o -name "*.sh" -o -name "*.py" \) \ + -type f ! \( -name "swiftc" -o -name "ld" -o -name "libtool" -o -name "import_indexstores" -o -name "calculate_output_groups" -o -name "*.sh" -o -name "*.py" \) \ -print0 | xargs -0 chmod -x # Copy over project.xcworkspace/contents.xcworkspacedata if needed From c325c1839234db13aa644b841e44d42fd57985f5 Mon Sep 17 00:00:00 2001 From: Matt Pennig Date: Mon, 5 Aug 2024 12:25:56 -0500 Subject: [PATCH 4/5] bugfix --- tools/BUILD | 2 +- tools/calculate_output_groups/Models.swift | 2 +- .../OutputGroupsCalculator.swift | 20 +++++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tools/BUILD b/tools/BUILD index 7c7632c011..d9f80fbc0c 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -65,7 +65,7 @@ _XCSCHEMES = [ # colorDiagnostics "NO", # xcodeVersionActual - "1520", + "1540", # nonPreviewObjRoot "/Users/brentley/Library/Developer/Xcode/DerivedData/tools-exxvdkcaoxdlhndlfnwxqvucohsr/Build/Intermediates.noindex", # baseObjRoot diff --git a/tools/calculate_output_groups/Models.swift b/tools/calculate_output_groups/Models.swift index 422d88266a..c2c55622c5 100644 --- a/tools/calculate_output_groups/Models.swift +++ b/tools/calculate_output_groups/Models.swift @@ -66,7 +66,7 @@ enum Output { struct Config: Codable { struct Settings: Codable { let base: [String] - var platforms: [String: Optional<[String]>] + var platforms: [String: [String]?] } let build: Settings? diff --git a/tools/calculate_output_groups/OutputGroupsCalculator.swift b/tools/calculate_output_groups/OutputGroupsCalculator.swift index dde901e08b..6b8aa40a5a 100644 --- a/tools/calculate_output_groups/OutputGroupsCalculator.swift +++ b/tools/calculate_output_groups/OutputGroupsCalculator.swift @@ -144,12 +144,16 @@ https://github.com/MobileNativeFoundation/rules_xcodeproj/issues/new?template=bu throw PreconditionError(message: "Settings not found for target/command combination: \(guid) / \(buildRequest.command)") } + var ids = settings.base for platform in allPlatformsToSearch(buildRequest.platform) { guard let platform = settings.platforms[platform] else { continue } - for prefix in prefixes { - for id in platform ?? settings.base { - lines.append("\(target.label)\n\(prefix) \(id)") - } + // An explicit nil value in platforms indicates inheritence from base + ids = platform ?? settings.base + break + } + for prefix in prefixes { + for id in ids { + lines.append("\(target.label)\n\(prefix) \(id)") } } } @@ -228,18 +232,18 @@ extension PIF.Target.BuildConfiguration { let platform = String(key.dropFirst(20).dropLast(2)) if value == "$(BAZEL_TARGET_ID)" { // This value indicates that the provided platform inherits from the base build setting. Store nil for later processing. - build?.platforms[platform] = nil + build?.platforms[platform] = Optional<[String]>.none } else { - build?.platforms[platform] = [value] + build?.platforms[platform] = .some([value]) } } if buildFiles != nil, key.starts(with: "BAZEL_COMPILE_TARGET_IDS[sdk=") { let platform = String(key.dropFirst(29).dropLast(2)) if value == "$(BAZEL_COMPILE_TARGET_IDS)" { // This value indicates that the provided platform inherits from the base build setting. Store nil for later processing. - buildFiles?.platforms[platform] = nil + buildFiles?.platforms[platform] = Optional<[String]>.none } else { - buildFiles?.platforms[platform] = compileTargetIds(value) + buildFiles?.platforms[platform] = .some(compileTargetIds(value)) } } } From ff13b61052448b9da33a851eb8b84b4f93b135fc Mon Sep 17 00:00:00 2001 From: Matt Pennig Date: Mon, 5 Aug 2024 12:26:09 -0500 Subject: [PATCH 5/5] swift-issue-reporting renamed --- xcodeproj/repositories.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcodeproj/repositories.bzl b/xcodeproj/repositories.bzl index 029ea994bd..35d1fd7c7b 100644 --- a/xcodeproj/repositories.bzl +++ b/xcodeproj/repositories.bzl @@ -390,8 +390,8 @@ swift_library( visibility = ["//visibility:public"], ) """, - sha256 = "97169124feb98b409f5b890bd95bb147c2fba0dba3098f9bf24c539270ee9601", - strip_prefix = "xctest-dynamic-overlay-0.2.1", + sha256 = "1ebde9c9403d5befb6956556e26f9308000722f7da9e87fed2e770d3918d647c", + strip_prefix = "swift-issue-reporting-0.2.1", url = "https://github.com/pointfreeco/xctest-dynamic-overlay/archive/refs/tags/0.2.1.tar.gz", ignore_version_differences = ignore_version_differences, )