Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid FoundationNetworking and libcurl on non-Darwin platforms #681

Draft
wants to merge 13 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions Package.resolved
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,15 @@
"revision" : "53e5cb9b18222f66cb8d6fb684d7383e705e0936"
}
},
{
"identity" : "swift-http-types",
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-http-types.git",
"state" : {
"revision" : "68deedb6b98837564cf0231fa1df48de35881993",
"version" : "0.2.1"
}
},
{
"identity" : "swift-lmdb",
"kind" : "remoteSourceControl",
Expand Down
6 changes: 5 additions & 1 deletion Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ let swiftSettings: [SwiftSetting] = [
let package = Package(
name: "SwiftDocC",
platforms: [
.macOS(.v10_15),
.macOS(.v11),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bumped to .v11 per a discussion with @franklinsch, this allows us to use throwing FileHandle.readToEnd() unlike the unsafe non-throwing FileHandle.readDataToEndOfFile that crashes the test suite on Linux.

.iOS(.v13)
],
products: [
Expand All @@ -45,6 +45,7 @@ let package = Package(
.product(name: "SymbolKit", package: "swift-docc-symbolkit"),
.product(name: "CLMDB", package: "swift-lmdb"),
.product(name: "Crypto", package: "swift-crypto"),
.product(name: "HTTPTypes", package: "swift-http-types"),
],
swiftSettings: swiftSettings
),
Expand Down Expand Up @@ -90,6 +91,7 @@ let package = Package(
.target(
name: "SwiftDocCTestUtilities",
dependencies: [
"SwiftDocC",
MaxDesiatov marked this conversation as resolved.
Show resolved Hide resolved
.product(name: "SymbolKit", package: "swift-docc-symbolkit"),
],
swiftSettings: swiftSettings
Expand Down Expand Up @@ -139,6 +141,7 @@ if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
.package(url: "https://github.com/apple/swift-docc-symbolkit", branch: "main"),
.package(url: "https://github.com/apple/swift-crypto.git", from: "2.5.0"),
.package(url: "https://github.com/apple/swift-docc-plugin", from: "1.2.0"),
.package(url: "https://github.com/apple/swift-http-types.git", .upToNextMinor(from: "0.2.1")),
]
} else {
// Building in the Swift.org CI system, so rely on local versions of dependencies.
Expand All @@ -149,5 +152,6 @@ if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
.package(path: "../swift-argument-parser"),
.package(path: "../swift-docc-symbolkit"),
.package(path: "../swift-crypto"),
.package(path: "../swift-http-types"),
]
}
6 changes: 4 additions & 2 deletions Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex+Ext.swift
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,10 @@ public class FileSystemRenderNodeProvider: RenderNodeProvider {
// we need to process JSON files only
if file.url.pathExtension.lowercased() == "json" {
do {
let data = try Data(contentsOf: file.url)
renderNode = try RenderNode.decode(fromJSON: data)
let fileHandle = try FileHandle(forReadingFrom: file.url)
if let data = try fileHandle.readToEnd() {
renderNode = try RenderNode.decode(fromJSON: data)
}
} catch {
let diagnostic = Diagnostic(source: file.url,
severity: .warning,
Expand Down
17 changes: 10 additions & 7 deletions Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex.swift
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,8 @@ public class NavigatorIndex {

/// A specific error to describe issues when processing a `NavigatorIndex`.
public enum Error: Swift.Error, DescribedError {

/// Missing bundle identifier.
case missingBundleIndentifier
case missingBundleIdentifier
MaxDesiatov marked this conversation as resolved.
Show resolved Hide resolved

/// A RenderNode has no title and won't be indexed.
case missingTitle(description: String)
Expand All @@ -72,8 +71,8 @@ public class NavigatorIndex {
case navigatorIndexIsNil

public var errorDescription: String {
switch self {
case .missingBundleIndentifier:
switch self {
case .missingBundleIdentifier:
return "A navigator index requires a bundle identifier, which is missing."
case .missingTitle:
return "The page has no valid title available."
Expand Down Expand Up @@ -169,13 +168,17 @@ public class NavigatorIndex {

let information = try environment.openDatabase(named: "information", flags: [])

let data = try Data(contentsOf: url.appendingPathComponent("availability.index", isDirectory: false))
let availabilityIndexFileURL = url.appendingPathComponent("availability.index", isDirectory: false)
let availabilityIndexFileHandle = try FileHandle(forReadingFrom: availabilityIndexFileURL)
guard let data = try availabilityIndexFileHandle.readToEnd() else {
throw FileSystemError.noDataReadFromFile(path: availabilityIndexFileURL.path)
}
let plistDecoder = PropertyListDecoder()
let availabilityIndex = try plistDecoder.decode(AvailabilityIndex.self, from: data)
let bundleIdentifier = bundleIdentifier ?? information.get(type: String.self, forKey: NavigatorIndex.bundleKey) ?? NavigatorIndex.UnknownBundleIdentifier

guard bundleIdentifier != NavigatorIndex.UnknownBundleIdentifier else {
throw Error.missingBundleIndentifier
throw Error.missingBundleIdentifier
}

// Use `.fnv1` by default if no path hasher is set for compatibility reasons.
Expand Down Expand Up @@ -282,7 +285,7 @@ public class NavigatorIndex {
self.availabilityIndex = AvailabilityIndex()

guard self.bundleIdentifier != NavigatorIndex.UnknownBundleIdentifier else {
throw Error.missingBundleIndentifier
throw Error.missingBundleIdentifier
}
}

Expand Down
26 changes: 21 additions & 5 deletions Sources/SwiftDocC/Indexing/Navigator/NavigatorTree.swift
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,20 @@ public class NavigatorTree {
- presentationIdentifier: Defines if nodes should have a presentation identifier useful in presentation contexts.
- broadcast: The callback to update get updates of the current process.
*/
public func read(from url: URL, bundleIdentifier: String? = nil, interfaceLanguages: Set<InterfaceLanguage>, timeout: TimeInterval, delay: TimeInterval = 0.01, queue: DispatchQueue, presentationIdentifier: String? = nil, broadcast: BroadcastCallback?) throws {
let data = try Data(contentsOf: url)
public func read(
from url: URL,
bundleIdentifier: String? = nil,
interfaceLanguages: Set<InterfaceLanguage>,
timeout: TimeInterval,
delay: TimeInterval = 0.01,
queue: DispatchQueue,
presentationIdentifier: String? = nil,
broadcast: BroadcastCallback?
) throws {
let fileHandle = try FileHandle(forReadingFrom: url)
guard let data = try fileHandle.readToEnd() else {
throw Error.cannotOpenFile(path: url.path)
}
let readingCursor = ReadingCursor(data: data)
self.readingCursor = readingCursor

Expand Down Expand Up @@ -312,9 +324,13 @@ public class NavigatorTree {
presentationIdentifier: String? = nil,
onNodeRead: ((NavigatorTree.Node) -> Void)? = nil
) throws -> NavigatorTree {
let fileUrl = URL(fileURLWithPath: path)
let data = try Data(contentsOf: fileUrl)

guard
let fileHandle = FileHandle(forReadingAtPath: path),
let data = try fileHandle.readToEnd()
else {
throw Error.cannotOpenFile(path: path)
}

var map = [UInt32: Node]()
var index: UInt32 = 0
var cursor = 0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,10 @@ enum SVGIDExtractor {
/// Returns nil if any errors are encountered or if an `id` attribute is
/// not found in the given SVG.
static func extractID(from svg: URL) -> String? {
guard let data = try? Data(contentsOf: svg) else {
guard
let fileHandle = try? FileHandle(forReadingFrom: svg),
let data = try? fileHandle.readToEnd()
else {
return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ public struct LocalFileSystemDataProvider: DocumentationWorkspaceDataProvider, F

public func contentsOfURL(_ url: URL) throws -> Data {
precondition(url.isFileURL, "Unexpected non-file url '\(url)'.")
return try Data(contentsOf: url)
let fileHandle = try FileHandle(forReadingFrom: url)
guard let data = try fileHandle.readToEnd() else {
throw FileSystemError.noDataReadFromFile(path: url.path)
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we have to do this instead of Data(contentsOf:) everywhere in the future then I'm concerned that either we or someone new to the repo would forget and break things.

Is there anyway that we can shim Data(contentsOf:) on non-Darwin platforms to do this or any way that we could mark Data(contentsOf:) as unavailable so that we don't accidentally break things in the future?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not aware of a way to do that. There's no way to prevent someone from adding import FoundationNetworking in the future and shadowing the locally declared shim. The right way to do this is for swift-corelibs-foundation to deprecate this initializer, but that's up to its maintainers and whether they want to do the same thing on Darwin so that these different platform implementations don't diverge even more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another thing that comes to mind is just checking for FoundationNetworking imports in ./bin/test and raising an error if that's found. That will mean if anyone adds Data(contentsOf:) that will fail to build on Linux and Windows unless they add import FoundationNetworking, which in turn will make ./bin/test.

Is ./bin/test what you run on CI?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Applied the latter suggestion in f2b5908.

Copy link
Contributor

@d-ronnqvist d-ronnqvist Aug 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is ./bin/test what you run on CI?

Yes


return data
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,12 @@ public struct PrebuiltLocalFileSystemDataProvider: DocumentationWorkspaceDataPro

public func contentsOfURL(_ url: URL) throws -> Data {
precondition(url.isFileURL, "Unexpected non-file url '\(url)'.")
return try Data(contentsOf: url)
let fileHandle = try FileHandle(forReadingFrom: url)
guard let data = try fileHandle.readToEnd() else {
throw FileSystemError.noDataReadFromFile(path: url.path)
}

return data
}
}

53 changes: 45 additions & 8 deletions Sources/SwiftDocC/Servers/DocumentationSchemeHandler.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,17 @@
*/

import Foundation

#if canImport(WebKit)
import WebKit
import HTTPTypes

@available(*, deprecated, renamed: "DocumentationSchemeHandler")
public typealias TopicReferenceSchemeHandler = DocumentationSchemeHandler
public class DocumentationSchemeHandler: NSObject {

public typealias FallbackResponseHandler = (URLRequest) -> (URLResponse, Data)?

enum Error: Swift.Error {
case noURLProvided
}

public typealias FallbackResponseHandler = (HTTPRequest) -> (HTTPTypes.HTTPResponse, Data)?

// The schema to support the documentation.
public static let scheme = "doc"
public static var fullScheme: String {
Expand Down Expand Up @@ -71,7 +72,7 @@ public class DocumentationSchemeHandler: NSObject {
}

/// Returns a response to a given request.
public func response(to request: URLRequest) -> (URLResponse, Data?) {
public func response(to request: HTTPRequest) -> (HTTPTypes.HTTPResponse, Data?) {
var (response, data) = fileServer.response(to: request)
if data == nil, let fallbackHandler = fallbackHandler,
let (fallbackResponse, fallbackData) = fallbackHandler(request) {
Expand All @@ -82,11 +83,47 @@ public class DocumentationSchemeHandler: NSObject {
}
}

#if canImport(WebKit)
import WebKit

// MARK: WKURLSchemeHandler protocol
extension DocumentationSchemeHandler: WKURLSchemeHandler {

public func webView(_ webView: WKWebView, start urlSchemeTask: WKURLSchemeTask) {
let (response, data) = self.response(to: urlSchemeTask.request)
let request = urlSchemeTask.request

// Render authority pseudo-header in accordance with https://www.rfc-editor.org/rfc/rfc3986.html#section-3.2
let authority: String
guard let url = request.url else {
urlSchemeTask.didFailWithError(Error.noURLProvided)
return
}

let userAuthority: String
if let user = url.user {
userAuthority = "\(user)@"
} else {
userAuthority = ""
}

let portAuthority: String
if let port = url.port {
portAuthority = ":\(port)"
} else {
portAuthority = ""
}

authority = "\(userAuthority)\(url.host ?? "")\(portAuthority)"
let httpRequest = HTTPRequest(method: .get, scheme: request.url?.scheme, authority: authority, path: request.url?.path)

let (httpResponse, data) = self.response(to: httpRequest)

let response = URLResponse(
url: url,
mimeType: httpResponse.headerFields[.contentType],
expectedContentLength: httpResponse.headerFields[.contentLength].flatMap(Int.init) ?? -1,
textEncodingName: httpResponse.headerFields[.contentEncoding]
)
urlSchemeTask.didReceive(response)
if let data = data {
urlSchemeTask.didReceive(data)
Expand Down
48 changes: 26 additions & 22 deletions Sources/SwiftDocC/Servers/FileServer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@
*/

import Foundation
import HTTPTypes
import SymbolKit
#if canImport(FoundationNetworking)
import FoundationNetworking
#endif

fileprivate let slashCharSet = CharacterSet(charactersIn: "/")

Expand Down Expand Up @@ -56,49 +54,49 @@ public class FileServer {
/**
Returns the data for a given URL.
*/
public func data(for url: URL) -> Data? {
public func data(for path: String) -> Data? {
MaxDesiatov marked this conversation as resolved.
Show resolved Hide resolved
let providerKey = providers.keys.sorted { (l, r) -> Bool in
l.count > r.count
}.filter { (path) -> Bool in
return url.path.trimmingCharacters(in: slashCharSet).hasPrefix(path)
}.filter { (providerPath) -> Bool in
return path.trimmingCharacters(in: slashCharSet).hasPrefix(providerPath)
}.first ?? "" //in case missing an exact match, get the root one
guard let provider = providers[providerKey] else {
fatalError("A provider has not been passed to a FileServer.")
}
return provider.data(for: url.path.trimmingCharacters(in: slashCharSet).removingPrefix(providerKey))
return provider.data(for: path.trimmingCharacters(in: slashCharSet).removingPrefix(providerKey))
}

/**
Returns a tuple with a response and the given data.
- Parameter request: The request coming from a web client.
- Returns: The response and data which are going to be served to the client.
*/
public func response(to request: URLRequest) -> (URLResponse, Data?) {
guard let url = request.url else {
return (HTTPURLResponse(url: baseURL, statusCode: 400, httpVersion: "HTTP/1.1", headerFields: nil)!, nil)
public func response(to request: HTTPRequest) -> (HTTPTypes.HTTPResponse, Data?) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a source breaking change. Is there anywhere to maintain both declarations to ease this transition?

Copy link
Contributor Author

@MaxDesiatov MaxDesiatov Aug 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can guard the old function that uses URLRequest and URLResponse on #if canImport(Darwin) if that's ok? Otherwise its presence on non-Darwin platforms adds a dependency on FoundationNetworking with curl and the rest of the system networking/crypto libraries, which is what this PR was meant to remove in the first place.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure. I'm concerned that someone could have an existing setup on Linux with curl and the rest of the system networking libraries and that their setup would break when they update to a new DocC version.

We try to provide a transition period for breaking changes. If that's not possible then so be it (although we would try to inform people about it ahead of time in the Forums). But it there's a way to stage this, then I think that it would be less disruptive that way.

Would it be possible to use a compile time define to exclude FoundationNetworking? That way we could have a 3 stage transition (first opt-in to exclude, then exclude by default, and finally remove it completely)

guard let path = request.path as NSString? else {
return (.init(status: 400), nil)
}
var data: Data? = nil
let response: URLResponse
let response: HTTPTypes.HTTPResponse

let mimeType: String

// We need to make sure that the path extension is for an actual file and not a symbol name which is a false positive
// like: "'...(_:)-6u3ic", that would be recognized as filename with the extension "(_:)-6u3ic". (rdar://71856738)
if url.pathExtension.isAlphanumeric && !url.lastPathComponent.isSwiftEntity {
data = self.data(for: url)
mimeType = FileServer.mimeType(for: url.pathExtension)
if path.pathExtension.isAlphanumeric && !path.lastPathComponent.isSwiftEntity {
data = self.data(for: path as String)
mimeType = FileServer.mimeType(for: path.pathExtension)
} else { // request is for a path, we need to fake a redirect here
if url.pathComponents.isEmpty {
xlog("Tried to load an invalid URL: \(url.absoluteString).\nFalling back to serve index.html.")
if path.pathComponents.isEmpty {
xlog("Tried to load an invalid path: \(path).\nFalling back to serve index.html.")
}
mimeType = "text/html"
data = self.data(for: baseURL.appendingPathComponent("/index.html"))
data = self.data(for: "/index.html")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain why this isn't a change in behavior? What's needless about this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only path component of url that data(for:) took as argument was used. The rest of the components are ignored by DocC server implementations.

Additionally, HTTPRequest unlike URLRequest doesn't store full URLs, only paths. The authority and scheme components can be assembled from scheme and authority properties on HTTPRequest , but those aren't used anywhere in the DocC codebase, other than a test in this diff below where we have to construct a new URLRequest to pass it to WKWebView on Darwin platforms.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But if request.path is "/something" then this used to pass "/something/index.html" as an argument but now it only passes "index.html". Isn't that returning data for the wrong path?

Copy link
Contributor Author

@MaxDesiatov MaxDesiatov Aug 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If request.path is "/something" then that is handled by the if branch above, no this else branch. The latter branch didn't use request.path previously, that logic is unchanged, as verified by the test suite.

}

if let data = data {
response = URLResponse(url: url, mimeType: mimeType, expectedContentLength: data.count, textEncodingName: nil)
response = .init(status: .ok, headerFields: [.contentType: mimeType, .contentLength: "\(data.count)"])
} else {
response = URLResponse(url: url, mimeType: nil, expectedContentLength: 0, textEncodingName: nil)
response = .init(status: .ok, headerFields: [.contentType: "application/octet-stream"])
}

return (response, data)
Expand Down Expand Up @@ -175,7 +173,8 @@ public class FileSystemServerProvider: FileServerProvider {

public func data(for path: String) -> Data? {
let finalURL = directoryURL.appendingPathComponent(path)
return try? Data(contentsOf: finalURL)
let fileHandle = try? FileHandle(forReadingFrom: finalURL)
return try? fileHandle?.readToEnd()
}

}
Expand Down Expand Up @@ -230,7 +229,12 @@ public class MemoryFileServerProvider: FileServerProvider {

for file in enumerator {
guard let file = file as? String else { fatalError("Enumerator returned an unexpected type.") }
guard let data = try? Data(contentsOf: URL(fileURLWithPath: path).appendingPathComponent(file)) else { continue }
let fileURL = URL(fileURLWithPath: path).appendingPathComponent(file)
guard
let fileHandle = try? FileHandle(forReadingFrom: fileURL),
let data = try? fileHandle.readToEnd()
else { continue }

if recursive == false && file.contains("/") { continue } // skip if subfolder and recursive is disabled
addFile(path: "/\(trimmedSubPath)/\(file)", data: data)
}
Expand Down
Loading