Skip to content

Commit

Permalink
SDK imports metadata (swiftlang#75)
Browse files Browse the repository at this point in the history 
This provides new metadata during builds:
- Use the new `-sdk_imports` linker option when appropriate and provide a post-processed version of it as a resource in each bundle.
- For each codeless bundle, "stamp" its Info.plist with any clients, this allows finding the library associated with a particular codeless bundle.

Both behaviors are enabled by the `ENABLE_SDK_IMPORTS` build setting which defaults to the inverse of `ASSETCATALOG_COMPILER_SKIP_APP_STORE_DEPLOYMENT`.

Co-authored-by: Owen Voorhees <[email protected]>
  • Loading branch information
@neonichu @owenv
neonichu and owenv authored Feb 7, 2025
1 parent 7da0a63 commit da67fc7
Show file tree
Hide file tree
Showing 40 changed files with 613 additions and 104 deletions.
13 changes: 9 additions & 4 deletions Sources/SWBCore/PlannedTaskAction.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,29 +74,32 @@ public struct InfoPlistProcessorTaskActionContext: PlatformBuildContext, Seriali
public var sdk: SDK?
public var sdkVariant: SDKVariant?
public var cleanupRequiredArchitectures: [String]
public var clientLibrariesForCodelessBundle: [String]

public init(scope: MacroEvaluationScope, productType: ProductTypeSpec?, platform: Platform?, sdk: SDK?, sdkVariant: SDKVariant?, cleanupRequiredArchitectures: [String]) {
public init(scope: MacroEvaluationScope, productType: ProductTypeSpec?, platform: Platform?, sdk: SDK?, sdkVariant: SDKVariant?, cleanupRequiredArchitectures: [String], clientLibrariesForCodelessBundle: [String] = []) {
self.scope = scope
self.productType = productType
self.platform = platform
self.sdk = sdk
self.sdkVariant = sdkVariant
self.cleanupRequiredArchitectures = cleanupRequiredArchitectures
self.clientLibrariesForCodelessBundle = clientLibrariesForCodelessBundle
}

public var targetBuildVersionPlatforms: Set<BuildVersion.Platform>? {
targetBuildVersionPlatforms(in: scope)
}

public func serialize<T: Serializer>(to serializer: T) {
serializer.beginAggregate(7)
serializer.beginAggregate(8)
serializer.serialize(scope)
serializer.serialize(platform?.identifier)
serializer.serialize(sdk?.canonicalName)
serializer.serialize(sdkVariant?.name)
serializer.serialize(productType?.identifier)
serializer.serialize(productType?.domain)
serializer.serialize(cleanupRequiredArchitectures)
serializer.serialize(clientLibrariesForCodelessBundle)
serializer.endAggregate()
}

Expand All @@ -105,7 +108,7 @@ public struct InfoPlistProcessorTaskActionContext: PlatformBuildContext, Seriali
// Get the platform registry to use to look up the platform from the deserializer's delegate.
guard let delegate = deserializer.delegate as? (any InfoPlistProcessorTaskActionContextDeserializerDelegate) else { throw DeserializerError.invalidDelegate("delegate must be a BuildDescriptionDeserializerDelegate") }

try deserializer.beginAggregate(7)
try deserializer.beginAggregate(8)
let scope: MacroEvaluationScope = try deserializer.deserialize()

let platformIdentifier: String? = try deserializer.deserialize()
Expand Down Expand Up @@ -158,8 +161,9 @@ public struct InfoPlistProcessorTaskActionContext: PlatformBuildContext, Seriali
}()

let cleanupRequiredArchitectures: [String] = try deserializer.deserialize()
let clientLibrariesForCodelessBundle: [String] = try deserializer.deserialize()

self = InfoPlistProcessorTaskActionContext(scope: scope, productType: productType, platform: platform, sdk: sdk, sdkVariant: sdkVariant, cleanupRequiredArchitectures: cleanupRequiredArchitectures)
self = InfoPlistProcessorTaskActionContext(scope: scope, productType: productType, platform: platform, sdk: sdk, sdkVariant: sdkVariant, cleanupRequiredArchitectures: cleanupRequiredArchitectures, clientLibrariesForCodelessBundle: clientLibrariesForCodelessBundle)
}
}

Expand Down Expand Up @@ -333,6 +337,7 @@ public protocol TaskActionCreationDelegate
func createValidateDevelopmentAssetsTaskAction() -> any PlannedTaskAction
func createSignatureCollectionTaskAction() -> any PlannedTaskAction
func createClangModuleVerifierInputGeneratorTaskAction() -> any PlannedTaskAction
func createProcessSDKImportsTaskAction() -> any PlannedTaskAction
}

extension TaskActionCreationDelegate {
Expand Down
4 changes: 4 additions & 0 deletions Sources/SWBCore/Settings/BuiltinMacros.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,7 @@ public final class BuiltinMacros {
/// be migrated. This is merely a stop-gap measure until we can turn on the debug
/// dylib for macOS completely.
public static let ENABLE_PREVIEWS_DYLIB_OVERRIDE = BuiltinMacros.declareBooleanMacro("ENABLE_PREVIEWS_DYLIB_OVERRIDE")
public static let ENABLE_SDK_IMPORTS = BuiltinMacros.declareBooleanMacro("ENABLE_SDK_IMPORTS")
public static let ENABLE_SIGNATURE_AGGREGATION = BuiltinMacros.declareBooleanMacro("ENABLE_SIGNATURE_AGGREGATION")
public static let DISABLE_TASK_SANDBOXING = BuiltinMacros.declareBooleanMacro("DISABLE_TASK_SANDBOXING")
public static let ENABLE_USER_SCRIPT_SANDBOXING = BuiltinMacros.declareBooleanMacro("ENABLE_USER_SCRIPT_SANDBOXING")
Expand Down Expand Up @@ -798,6 +799,7 @@ public final class BuiltinMacros {
public static let LD_LTO_OBJECT_FILE = BuiltinMacros.declarePathMacro("LD_LTO_OBJECT_FILE")
public static let LD_NO_PIE = BuiltinMacros.declareBooleanMacro("LD_NO_PIE")
public static let LD_RUNPATH_SEARCH_PATHS = BuiltinMacros.declareStringListMacro("LD_RUNPATH_SEARCH_PATHS")
public static let LD_SDK_IMPORTS_FILE = BuiltinMacros.declarePathMacro("LD_SDK_IMPORTS_FILE")
public static let LD_WARN_UNUSED_DYLIBS = BuiltinMacros.declareBooleanMacro("LD_WARN_UNUSED_DYLIBS")
public static let LEX = BuiltinMacros.declarePathMacro("LEX")
public static let LEXFLAGS = BuiltinMacros.declareStringListMacro("LEXFLAGS")
Expand Down Expand Up @@ -1656,6 +1658,7 @@ public final class BuiltinMacros {
ENABLE_DEBUG_DYLIB_OVERRIDE,
ENFORCE_VALID_ARCHS,
ENABLE_PREVIEWS_DYLIB_OVERRIDE,
ENABLE_SDK_IMPORTS,
ENABLE_SIGNATURE_AGGREGATION,
ENABLE_TESTABILITY,
ENABLE_TESTING_SEARCH_PATHS,
Expand Down Expand Up @@ -1843,6 +1846,7 @@ public final class BuiltinMacros {
LD_LTO_OBJECT_FILE,
LD_NO_PIE,
LD_RUNPATH_SEARCH_PATHS,
LD_SDK_IMPORTS_FILE,
LD_WARN_UNUSED_DYLIBS,
LEGACY_DEVELOPER_DIR,
LEX,
Expand Down
7 changes: 7 additions & 0 deletions Sources/SWBCore/Specs/CoreBuildSystem.xcspec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4462,6 +4462,13 @@ When this setting is enabled:
DisplayName = "Sticker Sharing Level";
Description = "When `GENERATE_INFOPLIST_FILE` is enabled, sets the value of the NSStickerSharingLevel key in the `Info.plist` file to the value of this build setting.";
},

// Support for producing SDK imports metadata
{
Name = "ENABLE_SDK_IMPORTS";
Type = Boolean;
DefaultValue = "$(ASSETCATALOG_COMPILER_SKIP_APP_STORE_DEPLOYMENT:not)";
},
);
},
)
1 change: 1 addition & 0 deletions Sources/SWBCore/Specs/RegisterSpecs.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ public struct BuiltinSpecsExtension: SpecificationsExtension {
CreateAssetPackManifestToolSpec.self,
CreateBuildDirectorySpec.self,
MergeInfoPlistSpec.self,
ProcessSDKImportsSpec.self,
ProcessXCFrameworkLibrarySpec.self,
RegisterExecutionPolicyExceptionToolSpec.self,
SwiftHeaderToolSpec.self,
Expand Down
4 changes: 2 additions & 2 deletions Sources/SWBCore/Specs/Tools/InfoPlistTool.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public final class InfoPlistToolSpec : GenericCommandLineToolSpec, SpecIdentifie
fatalError("unexpected direct invocation")
}

public func constructInfoPlistTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, generatedPkgInfoFile: Path? = nil, additionalContentFilePaths: [Path] = [], requiredArch: String? = nil, appPrivacyContentFiles: [Path] = []) async {
public func constructInfoPlistTasks(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, generatedPkgInfoFile: Path? = nil, additionalContentFilePaths: [Path] = [], requiredArch: String? = nil, appPrivacyContentFiles: [Path] = [], clientLibrariesForCodelessBundle: [String] = []) async {
let input = cbc.input
let inputPath = input.absolutePath
let outputPath = cbc.output
Expand Down Expand Up @@ -108,7 +108,7 @@ public final class InfoPlistToolSpec : GenericCommandLineToolSpec, SpecIdentifie

commandLine += ["-o", outputPath.str]

let context = InfoPlistProcessorTaskActionContext(scope: cbc.scope, productType: cbc.producer.productType, platform: cbc.producer.platform, sdk: cbc.producer.sdk, sdkVariant: cbc.producer.sdkVariant, cleanupRequiredArchitectures: cleanupArchs.sorted())
let context = InfoPlistProcessorTaskActionContext(scope: cbc.scope, productType: cbc.producer.productType, platform: cbc.producer.platform, sdk: cbc.producer.sdk, sdkVariant: cbc.producer.sdkVariant, cleanupRequiredArchitectures: cleanupArchs.sorted(), clientLibrariesForCodelessBundle: clientLibrariesForCodelessBundle)
let inputs = [inputPath] + effectiveAdditionalContentFilePaths + appPrivacyContentFiles
let serializer = MsgPackSerializer()
serializer.serialize(context)
Expand Down
144 changes: 81 additions & 63 deletions Sources/SWBCore/Specs/Tools/LinkerTools.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,17 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
// Generate the command line.
var commandLine = commandLineFromTemplate(cbc, delegate, optionContext: optionContext, specialArgs: specialArgs, lookup: lookup).map(\.asString)

// Add flags to emit SDK imports info.
let sdkImportsInfoFile = cbc.scope.evaluate(BuiltinMacros.LD_SDK_IMPORTS_FILE)
let supportsSDKImportsFeature = (try? optionContext?.toolVersion >= .init("1164")) == true
let usesLDClassic = commandLine.contains("-r") || commandLine.contains("-ld_classic") || cbc.scope.evaluate(BuiltinMacros.CURRENT_ARCH) == "armv7k"
if !usesLDClassic, supportsSDKImportsFeature, !sdkImportsInfoFile.isEmpty, cbc.scope.evaluate(BuiltinMacros.ENABLE_SDK_IMPORTS), cbc.producer.isApplePlatform {
commandLine.insert(contentsOf: ["-Xlinker", "-sdk_imports", "-Xlinker", sdkImportsInfoFile.str, "-Xlinker", "-sdk_imports_each_object"], at: commandLine.count - 2) // This preserves the assumption that the last argument is the linker output which a few tests make.
outputs.append(delegate.createNode(sdkImportsInfoFile))

await cbc.producer.processSDKImportsSpec.createTasks(CommandBuildContext(producer: cbc.producer, scope: cbc.scope, inputs: []), delegate, ldSDKImportsPath: sdkImportsInfoFile)
}

// Select the driver to use based on the input file types, replacing the value computed by commandLineFromTemplate().
let usedCXX = usedTools.values.contains(where: { $0.contains(where: { $0.languageDialect?.isPlusPlus ?? false }) })
commandLine[0] = resolveExecutablePath(cbc, computeLinkerPath(cbc, usedCXX: usedCXX)).str
Expand Down Expand Up @@ -957,7 +968,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
}

// Args without parameters (-Xlinker-prefixed, e.g. -Xlinker)
for arg in ["-bitcode_verify", "-export_dynamic"] {
for arg in ["-bitcode_verify", "-export_dynamic", "-sdk_imports_each_object"] {
while let index = commandLine.firstIndex(of: arg) {
guard index > 0, commandLine[index - 1] == argPrefix else { break }
commandLine.removeSubrange(index - 1 ... index)
Expand All @@ -976,7 +987,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
}

// Args with a parameter (-Xlinker-prefixed, e.g. -Xlinker arg -Xlinker param)
for arg in ["-object_path_lto", "-add_ast_path", "-dependency_info", "-map", "-order_file", "-bitcode_symbol_map", "-final_output", "-allowable_client"] {
for arg in ["-object_path_lto", "-add_ast_path", "-dependency_info", "-map", "-order_file", "-bitcode_symbol_map", "-final_output", "-allowable_client", "-sdk_imports"] {
while let index = commandLine.firstIndex(of: arg) {
guard index > 0,
index + 2 < commandLine.count,
Expand Down Expand Up @@ -1228,67 +1239,7 @@ public final class LdLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
return nil
}

do {
do {
let commandLine = [toolPath.str, "-version_details"]
return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, commandLine, { executionResult in
let gnuLD = [
#/GNU ld version (?<version>[\d.]+)-.*/#,
#/GNU ld \(GNU Binutils.*\) (?<version>[\d.]+)/#,
]
if let match = try gnuLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .gnuld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

let goLD = [
#/GNU gold version (?<version>[\d.]+)-.*/#,
#/GNU gold \(GNU Binutils.*\) (?<version>[\d.]+)/#,
]
if let match = try goLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .gold, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

struct LDVersionDetails: Decodable {
let version: Version
let architectures: Set<String>
}

let details: LDVersionDetails
do {
details = try JSONDecoder().decode(LDVersionDetails.self, from: executionResult.stdout)
} catch {
throw CommandLineOutputJSONParsingError(commandLine: commandLine, data: executionResult.stdout)
}

return DiscoveredLdLinkerToolSpecInfo(linker: .ld64, toolPath: toolPath, toolVersion: details.version, architectures: details.architectures)
})
} catch let e as CommandLineOutputJSONParsingError {
let vCommandLine = [toolPath.str, "-v"]
return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, vCommandLine, { executionResult in
let lld = [
#/LLD (?<version>[\d.]+) .*/#,
]
if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

let versionCommandLine = [toolPath.str, "--version"]
return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, versionCommandLine, { executionResult in
let lld = [
#/LLD (?<version>[\d.]+) .*/#,
]
if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

throw e
})
})
}
} catch {
delegate.error(error)
return nil
}
return await discoveredLinkerToolsInfo(producer, delegate, at: toolPath)
}
}

Expand Down Expand Up @@ -1582,6 +1533,73 @@ public final class LibtoolLinkerSpec : GenericLinkerSpec, SpecIdentifierType {
}
}

/// Consults the global cache of discovered info for the linker at `toolPath` and returns it, creating it if necessary.
///
/// This is global and public because it is used by `SWBTaskExecution` and `CoreBasedTests`, which is the basis of many of our tests (so caching this info across tests is desirable).
public func discoveredLinkerToolsInfo(_ producer: any CommandProducer, _ delegate: any CoreClientTargetDiagnosticProducingDelegate, at toolPath: Path) async -> (any DiscoveredCommandLineToolSpecInfo)? {
do {
do {
let commandLine = [toolPath.str, "-version_details"]
return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, commandLine, { executionResult in
let gnuLD = [
#/GNU ld version (?<version>[\d.]+)-.*/#,
#/GNU ld \(GNU Binutils.*\) (?<version>[\d.]+)/#,
]
if let match = try gnuLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .gnuld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

let goLD = [
#/GNU gold version (?<version>[\d.]+)-.*/#,
#/GNU gold \(GNU Binutils.*\) (?<version>[\d.]+)/#,
]
if let match = try goLD.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .gold, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

struct LDVersionDetails: Decodable {
let version: Version
let architectures: Set<String>
}

let details: LDVersionDetails
do {
details = try JSONDecoder().decode(LDVersionDetails.self, from: executionResult.stdout)
} catch {
throw CommandLineOutputJSONParsingError(commandLine: commandLine, data: executionResult.stdout)
}

return DiscoveredLdLinkerToolSpecInfo(linker: .ld64, toolPath: toolPath, toolVersion: details.version, architectures: details.architectures)
})
} catch let e as CommandLineOutputJSONParsingError {
let vCommandLine = [toolPath.str, "-v"]
return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, vCommandLine, { executionResult in
let lld = [
#/LLD (?<version>[\d.]+) .*/#,
]
if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

let versionCommandLine = [toolPath.str, "--version"]
return try await producer.discoveredCommandLineToolSpecInfo(delegate, nil, versionCommandLine, { executionResult in
let lld = [
#/LLD (?<version>[\d.]+) .*/#,
]
if let match = try lld.compactMap({ try $0.firstMatch(in: String(decoding: executionResult.stdout, as: UTF8.self)) }).first {
return DiscoveredLdLinkerToolSpecInfo(linker: .lld, toolPath: toolPath, toolVersion: try Version(String(match.output.version)), architectures: Set())
}

throw e
})
})
}
} catch {
delegate.error(error)
return nil
}
}

/// Enumerates a linker command line, calling the provided `handle` closure for each logical argument, providing the argument name as `arg` and a `value` function which returns the value of that argument at `offset` positions distance from the `arg` value based on how many arguments it takes.
///
/// The purpose of this function is to automatically abstract away handling of the `-Xlinker` and `-Wl` syntax used by compiler drivers to forward arguments to the linker which they themselves do not understand.
Expand Down
Loading

0 comments on commit da67fc7

Please sign in to comment.