Skip to content

Commit

Permalink
Allow remapping indexstores to multiple target names
Browse files Browse the repository at this point in the history 
This isn’t exercised yet. Incremental generation will use it.

Signed-off-by: Brentley Jones <[email protected]>
  • Loading branch information
@brentleyjones
brentleyjones committed Oct 11, 2023
1 parent 386feed commit 41a7147
Show file tree
Hide file tree
Showing 10 changed files with 245 additions and 189 deletions.
3 changes: 3 additions & 0 deletions tools/import_indexstores/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ swift_library(
name = "import_indexstores.library",
srcs = glob(["*.swift"]),
module_name = "import_indexstores",
deps = [
"//tools/lib/ToolCommon",
],
)

swift_binary(
Expand Down
29 changes: 0 additions & 29 deletions tools/import_indexstores/Errors.swift
View file

This file was deleted.

45 changes: 45 additions & 0 deletions tools/import_indexstores/Import.swift
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import Foundation

extension ImportIndex {
static func `import`(
_ indexstores: Set<String>,
into indexDataStoreDir: URL,
arch: String,
developerDir: String,
indexImport: String,
objectFilePrefix: String,
projectDirPrefix: String,
srcRoot: String,
targetNameOverride: String?,
xcodeExecutionRoot: String,
xcodeOutputBase: String
) throws {
let filelistContent = indexstores
.map { projectDirPrefix + $0 + "\n" }
.joined()

let filelist = try TemporaryFile()
try filelistContent
.write(to: filelist.url, atomically: true, encoding: .utf8)

let remaps = remapArgs(
arch: arch,
developerDir: developerDir,
objectFilePrefix: objectFilePrefix,
srcRoot: srcRoot,
targetNameOverride: targetNameOverride,
xcodeExecutionRoot: xcodeExecutionRoot,
xcodeOutputBase: xcodeOutputBase
)

try runSubProcess(
indexImport,
remaps + [
"-undo-rules_swift-renames",
"-incremental",
"@\(filelist.url.path)",
indexDataStoreDir.path,
]
)
}
}
264 changes: 136 additions & 128 deletions tools/import_indexstores/ImportIndexstores.swift
View file
Original file line number Diff line number Diff line change
@@ -1,145 +1,165 @@
import Foundation
import ToolCommon

@main
struct ImportIndex {
static func main() async throws {
let args = CommandLine.arguments
let pidFile =
do {
let args = CommandLine.arguments
let pidFile =
try getEnvironmentVariable("OBJROOT") + "/import_indexstores.pid"

guard args.count > 1 else {
throw PreconditionError(
message: "Not enough arguments, expected path to execution root"
)
}
let buildExecutionRoot = args[1]
guard args.count > 1 else {
throw PreconditionError(message: """
Not enough arguments, expected path to execution root
"""
)
}
let buildExecutionRoot = args[1]

// Exit early if no indexstore filelists were provided
if args.count == 2 {
return
}
// Exit early if no indexstore filelists were provided
if args.count == 2 {
return
}

// MARK: pidFile
// MARK: pidFile

// Kill any previously running import
if FileManager.default.fileExists(atPath: pidFile) {
let pid = try String(contentsOfFile: pidFile)
// Kill any previously running import
if FileManager.default.fileExists(atPath: pidFile) {
let pid = try String(contentsOfFile: pidFile)

try runSubProcess("/bin/kill", [pid])
while true {
if try runSubProcess("/bin/kill", ["-0", pid]) != 0 {
break
try runSubProcess("/bin/kill", [pid])
while true {
if try runSubProcess("/bin/kill", ["-0", pid]) != 0 {
break
}
sleep(1)
}
sleep(1)
}
}

// Set pid to allow cleanup later
try String(ProcessInfo.processInfo.processIdentifier)
.write(toFile: pidFile, atomically: true, encoding: .utf8)
defer {
try? FileManager.default.removeItem(atPath: pidFile)
}

// MARK: filelist

let projectDirPrefix = try getEnvironmentVariable("PROJECT_DIR") + "/"

// Merge all filelists into a single file
var indexStores: Set<String> = []
for filePath in args.dropFirst(2) {
let url = URL(fileURLWithPath: filePath)
for try await indexStore in url.lines {
indexStores.insert(indexStore)
// Set pid to allow cleanup later
try String(ProcessInfo.processInfo.processIdentifier)
.write(toFile: pidFile, atomically: true, encoding: .utf8)
defer {
try? FileManager.default.removeItem(atPath: pidFile)
}
}

// Exit early if no indexstores were provided
guard !indexStores.isEmpty else {
return
}
// MARK: filelist

let projectDirPrefix =
try getEnvironmentVariable("PROJECT_DIR") + "/"

// Merge all filelists into a single file
var indexstores: [String: Set<String>] = [:]
for filePath in args.dropFirst(2) {
let url = URL(fileURLWithPath: filePath)

var iterator = url.allLines.makeAsyncIterator()
while let indexstore = try await iterator.next() {
guard let targetNameOverride = try await iterator.next()
else {
throw PreconditionError(message: """
indexstore filelists must contain pairs of <indexstore> <target-name-override> \
lines
"""
)
}

indexstores[targetNameOverride, default: []]
.insert(indexstore)
}
}

let filelistContent = indexStores
.map { projectDirPrefix + $0 + "\n" }
.joined()
let filelist = try TemporaryFile()
try filelistContent
.write(to: filelist.url, atomically: true, encoding: .utf8)
// Exit early if no indexstores were provided
guard !indexstores.isEmpty else {
return
}

// MARK: Remaps
let indexDataStoreDir = URL(
fileURLWithPath:
try getEnvironmentVariable("INDEX_DATA_STORE_DIR")
)
let recordsDir =
indexDataStoreDir.appendingPathComponent("v5/records")

// We remove any `/private` prefix from the current execution_root,
// since it's removed in the Project navigator
let xcodeExecutionRoot: String
if buildExecutionRoot.hasPrefix("/private") {
xcodeExecutionRoot = String(buildExecutionRoot.dropFirst(8))
} else {
xcodeExecutionRoot = buildExecutionRoot
}
try FileManager.default.createDirectory(
at: recordsDir,
withIntermediateDirectories: true
)

let projectTempDir = try getEnvironmentVariable("PROJECT_TEMP_DIR")
// We remove any `/private` prefix from the current execution_root,
// since it's removed in the Project navigator
let xcodeExecutionRoot: String
if buildExecutionRoot.hasPrefix("/private") {
xcodeExecutionRoot = String(buildExecutionRoot.dropFirst(8))
} else {
xcodeExecutionRoot = buildExecutionRoot
}

let objectFilePrefix: String
if try getEnvironmentVariable("ACTION") == "indexbuild" {
// Remove `Index.noindex/` part of path
objectFilePrefix = projectTempDir.replacingOccurrences(
of: "/Index.noindex/Build/Intermediates.noindex/",
with: "/Build/Intermediates.noindex/"
)
} else {
// Remove SwiftUI Previews part of path
objectFilePrefix = try projectTempDir.replacingRegex(
matching: #"""
let projectTempDir = try getEnvironmentVariable("PROJECT_TEMP_DIR")

let objectFilePrefix: String
if try getEnvironmentVariable("ACTION") == "indexbuild" {
// Remove `Index.noindex/` part of path
objectFilePrefix = projectTempDir.replacingOccurrences(
of: "/Index.noindex/Build/Intermediates.noindex/",
with: "/Build/Intermediates.noindex/"
)
} else {
// Remove SwiftUI Previews part of path
objectFilePrefix = try projectTempDir.replacingRegex(
matching: #"""
Intermediates\.noindex/Previews/[^/]*/Intermediates\.noindex
"""#,
with: "Intermediates.noindex"
)
}

let xcodeOutputBase = xcodeExecutionRoot
.split(separator: "/")
.dropLast(2)
.joined(separator: "/")

let archs = try getEnvironmentVariable("ARCHS")
let arch = String(archs.split(separator: " ", maxSplits: 1).first!)

let remaps = remapArgs(
arch: arch,
developerDir: try getEnvironmentVariable("DEVELOPER_DIR"),
objectFilePrefix: objectFilePrefix,
srcRoot: try getEnvironmentVariable("SRCROOT"),
xcodeExecutionRoot: xcodeExecutionRoot,
xcodeOutputBase: xcodeOutputBase
)

// MARK: Import

let indexDataStoreDir = URL(
fileURLWithPath: try getEnvironmentVariable("INDEX_DATA_STORE_DIR")
)
let recordsDir = indexDataStoreDir.appendingPathComponent("v5/records")
with: "Intermediates.noindex"
)
}

try FileManager.default.createDirectory(
at: recordsDir,
withIntermediateDirectories: true
)
let xcodeOutputBase = xcodeExecutionRoot
.split(separator: "/")
.dropLast(2)
.joined(separator: "/")

let archs = try getEnvironmentVariable("ARCHS")
let arch = String(archs.split(separator: " ", maxSplits: 1).first!)

let developerDir = try getEnvironmentVariable("DEVELOPER_DIR")
let indexImport = try getEnvironmentVariable("INDEX_IMPORT")
let srcRoot = try getEnvironmentVariable("SRCROOT")

try await withThrowingTaskGroup(of: Void.self) { group in
for (targetNameOverride, indexstores) in indexstores {
group.addTask {
try Self.import(
indexstores,
into: indexDataStoreDir,
arch: arch,
developerDir: developerDir,
indexImport: indexImport,
objectFilePrefix: objectFilePrefix,
projectDirPrefix: projectDirPrefix,
srcRoot: srcRoot,
targetNameOverride:
targetNameOverride.isEmpty ?
nil : targetNameOverride,
xcodeExecutionRoot: xcodeExecutionRoot,
xcodeOutputBase: xcodeOutputBase
)
}
}

try runSubProcess(
try getEnvironmentVariable("INDEX_IMPORT"),
remaps + [
"-undo-rules_swift-renames",
"-incremental",
"@\(filelist.url.path)",
indexDataStoreDir.path,
]
)
try await group.waitForAll()
}

// Unit files are created fresh, but record files are copied from
// `bazel-out/`, which are read-only. We need to adjust their
// permissions.
// TODO: do this in `index-import`
try setWritePermissions(in: recordsDir)
// Unit files are created fresh, but record files are copied from
// `bazel-out/`, which are read-only. We need to adjust their
// permissions.
// TODO: do this in `index-import`
try setWritePermissions(in: recordsDir)
} catch {
fputs(error.localizedDescription, stderr)
Darwin.exit(1)
}
}
}

Expand Down Expand Up @@ -167,18 +187,6 @@ Environment variable "\#(key)" is set to an empty string
return value
}

@discardableResult private func runSubProcess(
_ executable: String,
_ args: [String]
) throws -> Int32 {
let task = Process()
task.launchPath = executable
task.arguments = args
try task.run()
task.waitUntilExit()
return task.terminationStatus
}

private func setWritePermissions(in url: URL) throws {
let enumerator = FileManager.default.enumerator(
at: url,
Expand Down
Loading

0 comments on commit 41a7147

Please sign in to comment.