Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for WASILibc #2671

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Add support for WASILibc #2671

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
70ff245
Add support for WASILibc
MaxDesiatov Mar 4, 2024
66e3f0e
Clean up formatting
MaxDesiatov Mar 4, 2024
9170712
Clean up formatting
MaxDesiatov Mar 4, 2024
1958d91
Fix `NIOEmbedded` for WASI
MaxDesiatov Mar 4, 2024
4cfee57
`NIOHTTP1` should depend on `NIOPosix` only on Posix-ish platforms
MaxDesiatov Mar 4, 2024
e5ccda5
Clean up naming in `Package.swift`
MaxDesiatov Mar 4, 2024
9fe470a
Bump to later commit of swift-atomics
MaxDesiatov Mar 4, 2024
85f140e
Address PR feedback
MaxDesiatov Mar 8, 2024
4ee91c6
Reduce the diff
MaxDesiatov Mar 8, 2024
fdd622f
Merge branch 'main' into maxd/wasi-support
MaxDesiatov Mar 8, 2024
b1b5d7f
spaces >> tabs
MaxDesiatov Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 11 additions & 2 deletions Package.swift
View file
Open in desktop
Expand Up @@ -15,6 +15,7 @@

import PackageDescription


MaxDesiatov marked this conversation as resolved.
Show resolved Hide resolved
MaxDesiatov marked this conversation as resolved.
Show resolved Hide resolved
let swiftAtomics: PackageDescription.Target.Dependency = .product(name: "Atomics", package: "swift-atomics")
let swiftCollections: PackageDescription.Target.Dependency = .product(name: "DequeModule", package: "swift-collections")
let swiftSystem: PackageDescription.Target.Dependency = .product(
Expand All @@ -23,6 +24,9 @@ let swiftSystem: PackageDescription.Target.Dependency = .product(
condition: .when(platforms: [.macOS, .iOS, .tvOS, .watchOS, .linux, .android])
)

// These platforms require a depdency on `NIOPosix` from `NIOHTTP1` to maintain backward
// compatibility with previous NIO versions.
let historicalNIOPosixDependencyRequired: [Platform] = [.macOS, .iOS, .tvOS, .watchOS, .linux, .android]

let package = Package(
name: "swift-nio",
Expand All @@ -42,7 +46,7 @@ let package = Package(
.library(name: "_NIOFileSystemFoundationCompat", targets: ["NIOFileSystemFoundationCompat"]),
],
dependencies: [
.package(url: "https://github.com/apple/swift-atomics.git", from: "1.1.0"),
.package(url: "https://github.com/apple/swift-atomics.git", revision: "1f2007ef4165432f59f28615c473eb79a844a2af"),
MaxDesiatov marked this conversation as resolved.
Show resolved Hide resolved
.package(url: "https://github.com/apple/swift-collections.git", from: "1.0.2"),
.package(url: "https://github.com/apple/swift-system.git", from: "1.2.0"),
.package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"),
Expand All @@ -58,6 +62,7 @@ let package = Package(
"CNIODarwin",
"CNIOLinux",
"CNIOWindows",
"CNIOWASI",
"_NIODataStructures",
swiftCollections,
swiftAtomics,
Expand Down Expand Up @@ -142,6 +147,10 @@ let package = Package(
name: "CNIOWindows",
dependencies: []
),
.target(
name: "CNIOWASI",
dependencies: []
),
.target(
name: "NIOConcurrencyHelpers",
dependencies: [
Expand All @@ -151,7 +160,7 @@ let package = Package(
.target(
name: "NIOHTTP1",
dependencies: [
"NIO",
.target(name: "NIO", condition: .when(platforms: historicalNIOPosixDependencyRequired)),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This condition should probably be persisted across almost any library target that depends on NIO which also depends on NIOCore.

_NIOConcurrency, NIOFoundationCompat, NIOWebSocket, NIOTLS appear to be the other places.

"NIOCore",
"NIOConcurrencyHelpers",
"CNIOLLHTTP",
Expand Down
2 changes: 1 addition & 1 deletion Sources/CNIOSHA1/c_nio_sha1.c
View file
Open in desktop
Expand Up @@ -54,7 +54,7 @@
#endif
#ifdef __ANDROID__
#include <sys/endian.h>
#elif defined(__linux__) || defined(__APPLE__)
#elif defined(__linux__) || defined(__APPLE__) || defined(__wasm32__)
#include <sys/types.h>
#endif

Expand Down
18 changes: 18 additions & 0 deletions Sources/CNIOWASI/include/CNIOWASI.h
View file
Open in desktop
@@ -0,0 +1,18 @@
#pragma once

#if __wasi__

#include <fcntl.h>
#include <time.h>

static inline void CNIOWASI_gettime(struct timespec *tv) {
// ClangImporter doesn't support `CLOCK_MONOTONIC` declaration in WASILibc, thus we have to define a bridge manually
clock_gettime(CLOCK_MONOTONIC, tv);
}

static inline int CNIOWASI_O_CREAT() {
// ClangImporter doesn't support `O_CREATE` declaration in WASILibc, thus we have to define a bridge manually
return O_CREAT;
}

#endif
10 changes: 6 additions & 4 deletions Sources/NIOConcurrencyHelpers/NIOLock.swift
View file
Open in desktop
Expand Up @@ -21,6 +21,8 @@ import WinSDK
import Glibc
#elseif canImport(Musl)
import Musl
#elseif canImport(WASILibc)
import WASILibc
#else
#error("The concurrency NIOLock module was unable to identify your C library.")
#endif
Expand All @@ -43,7 +45,7 @@ extension LockOperations {

#if os(Windows)
InitializeSRWLock(mutex)
#else
#elseif !os(WASI)
var attr = pthread_mutexattr_t()
pthread_mutexattr_init(&attr)
debugOnly {
Expand All @@ -61,7 +63,7 @@ extension LockOperations {

#if os(Windows)
// SRWLOCK does not need to be free'd
#else
#elseif !os(WASI)
let err = pthread_mutex_destroy(mutex)
precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)")
#endif
Expand All @@ -73,7 +75,7 @@ extension LockOperations {

#if os(Windows)
AcquireSRWLockExclusive(mutex)
#else
#elseif !os(WASI)
let err = pthread_mutex_lock(mutex)
precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)")
#endif
Expand All @@ -85,7 +87,7 @@ extension LockOperations {

#if os(Windows)
ReleaseSRWLockExclusive(mutex)
#else
#elseif !os(WASI)
let err = pthread_mutex_unlock(mutex)
precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)")
#endif
Expand Down
2 changes: 2 additions & 0 deletions Sources/NIOConcurrencyHelpers/atomics.swift
View file
Open in desktop
Expand Up @@ -30,6 +30,8 @@ fileprivate func sys_sched_yield() {
import Glibc
#elseif canImport(Musl)
import Musl
#elseif canImport(WASILibc)
import WASILibc
#else
#error("The concurrency atomics module was unable to identify your C library.")
#endif
Expand Down
24 changes: 15 additions & 9 deletions Sources/NIOConcurrencyHelpers/lock.swift
View file
Open in desktop
Expand Up @@ -21,6 +21,8 @@ import WinSDK
import Glibc
#elseif canImport(Musl)
import Musl
#elseif canImport(WASILibc)
import WASILibc
#else
#error("The concurrency lock module was unable to identify your C library.")
#endif
Expand All @@ -45,7 +47,7 @@ public final class Lock {
public init() {
#if os(Windows)
InitializeSRWLock(self.mutex)
#else
#elseif !os(WASI)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we can merge a PR that silently disables these primitives on WASI without being extremely confident that we'll never need them. Is there any way we can replace this with a feature-detection on threading?

var attr = pthread_mutexattr_t()
pthread_mutexattr_init(&attr)
debugOnly {
Expand All @@ -60,7 +62,7 @@ public final class Lock {
deinit {
#if os(Windows)
// SRWLOCK does not need to be free'd
#else
#elseif !os(WASI)
let err = pthread_mutex_destroy(self.mutex)
precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)")
#endif
Expand All @@ -74,7 +76,7 @@ public final class Lock {
public func lock() {
#if os(Windows)
AcquireSRWLockExclusive(self.mutex)
#else
#elseif !os(WASI)
let err = pthread_mutex_lock(self.mutex)
precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)")
#endif
Expand All @@ -87,7 +89,7 @@ public final class Lock {
public func unlock() {
#if os(Windows)
ReleaseSRWLockExclusive(self.mutex)
#else
#elseif !os(WASI)
let err = pthread_mutex_unlock(self.mutex)
precondition(err == 0, "\(#function) failed in pthread_mutex with error \(err)")
#endif
Expand Down Expand Up @@ -127,7 +129,7 @@ public final class ConditionLock<T: Equatable> {
#if os(Windows)
private let cond: UnsafeMutablePointer<CONDITION_VARIABLE> =
UnsafeMutablePointer.allocate(capacity: 1)
#else
#elseif !os(WASI)
private let cond: UnsafeMutablePointer<pthread_cond_t> =
UnsafeMutablePointer.allocate(capacity: 1)
#endif
Expand All @@ -140,7 +142,7 @@ public final class ConditionLock<T: Equatable> {
self.mutex = NIOLock()
#if os(Windows)
InitializeConditionVariable(self.cond)
#else
#elseif !os(WASI)
let err = pthread_cond_init(self.cond, nil)
precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)")
#endif
Expand All @@ -149,11 +151,13 @@ public final class ConditionLock<T: Equatable> {
deinit {
#if os(Windows)
// condition variables do not need to be explicitly destroyed
#else
#elseif !os(WASI)
let err = pthread_cond_destroy(self.cond)
precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)")
#endif
#if !os(WASI)
self.cond.deallocate()
#endif
}

/// Acquire the lock, regardless of the value of the state variable.
Expand Down Expand Up @@ -193,7 +197,7 @@ public final class ConditionLock<T: Equatable> {
#if os(Windows)
let result = SleepConditionVariableSRW(self.cond, mutex, INFINITE, 0)
precondition(result, "\(#function) failed in SleepConditionVariableSRW with error \(GetLastError())")
#else
#elseif !os(WASI)
let err = pthread_cond_wait(self.cond, mutex)
precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)")
#endif
Expand Down Expand Up @@ -235,6 +239,8 @@ public final class ConditionLock<T: Equatable> {
// NOTE: this may be a spurious wakeup, adjust the timeout accordingly
dwMilliseconds = dwMilliseconds - (timeGetTime() - dwWaitStart)
}
#elseif os(WASI)
return true
#else
let nsecPerSec: Int64 = 1000000000
self.lock()
Expand Down Expand Up @@ -277,7 +283,7 @@ public final class ConditionLock<T: Equatable> {
self.unlock()
#if os(Windows)
WakeAllConditionVariable(self.cond)
#else
#elseif !os(WASI)
let err = pthread_cond_broadcast(self.cond)
precondition(err == 0, "\(#function) failed in pthread_cond with error \(err)")
#endif
Expand Down
2 changes: 2 additions & 0 deletions Sources/NIOCore/AsyncAwaitSupport.swift
View file
Open in desktop
Expand Up @@ -33,6 +33,7 @@ extension EventLoopFuture {
}
}

#if !os(WASI)
extension EventLoopGroup {
/// Shuts down the event loop gracefully.
@available(macOS 10.15, iOS 13, tvOS 13, watchOS 6, *)
Expand All @@ -49,6 +50,7 @@ extension EventLoopGroup {
}
}
}
#endif

extension EventLoopPromise {
/// Complete a future with the result (or error) of the `async` function `body`.
Expand Down
13 changes: 11 additions & 2 deletions Sources/NIOCore/BSDSocketAPI.swift
View file
Open in desktop
Expand Up @@ -78,6 +78,11 @@ private let sysInet_pton: @convention(c) (CInt, UnsafePointer<CChar>?, UnsafeMut
#elseif canImport(Darwin)
import Darwin

private let sysInet_ntop: @convention(c) (CInt, UnsafeRawPointer?, UnsafeMutablePointer<CChar>?, socklen_t) -> UnsafePointer<CChar>? = inet_ntop
private let sysInet_pton: @convention(c) (CInt, UnsafePointer<CChar>?, UnsafeMutableRawPointer?) -> CInt = inet_pton
#elseif canImport(WASILibc)
import WASILibc

private let sysInet_ntop: @convention(c) (CInt, UnsafeRawPointer?, UnsafeMutablePointer<CChar>?, socklen_t) -> UnsafePointer<CChar>? = inet_ntop
private let sysInet_pton: @convention(c) (CInt, UnsafePointer<CChar>?, UnsafeMutableRawPointer?) -> CInt = inet_pton
#else
Expand Down Expand Up @@ -199,12 +204,14 @@ extension NIOBSDSocket.ProtocolFamily {
public static let inet6: NIOBSDSocket.ProtocolFamily =
NIOBSDSocket.ProtocolFamily(rawValue: PF_INET6)

#if !os(WASI)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My general preference here would be that the APIs here are present, they just end up failing when actually used in WASI. That would mean hardcoding the syscall value on WASI, or deliberately using a spoiler value. That note applies to the rest of this file.

/// UNIX local to the host.
public static let unix: NIOBSDSocket.ProtocolFamily =
NIOBSDSocket.ProtocolFamily(rawValue: PF_UNIX)
#endif
}

#if !os(Windows)
#if !os(Windows) && !os(WASI)
extension NIOBSDSocket.ProtocolFamily {
/// UNIX local to the host, alias for `PF_UNIX` (`.unix`)
public static let local: NIOBSDSocket.ProtocolFamily =
Expand Down Expand Up @@ -370,6 +377,7 @@ extension NIOBSDSocket.Option {
public static let mptcp_info = NIOBSDSocket.Option(rawValue: 1)
}

#if !os(WASI)
// Socket Options
extension NIOBSDSocket.Option {
/// Get the error status and clear.
Expand All @@ -396,8 +404,9 @@ extension NIOBSDSocket.Option {
public static let so_reuseaddr: NIOBSDSocket.Option =
NIOBSDSocket.Option(rawValue: SO_REUSEADDR)
}
#endif

#if !os(Windows)
#if !os(Windows) && !os(WASI)
extension NIOBSDSocket.Option {
/// Indicate when to generate timestamps.
public static let so_timestamp: NIOBSDSocket.Option =
Expand Down
11 changes: 9 additions & 2 deletions Sources/NIOCore/ByteBuffer-aux.swift
View file
Open in desktop
Expand Up @@ -12,7 +12,9 @@
//
//===----------------------------------------------------------------------===//

#if canImport(Dispatch)
import Dispatch
#endif
import _NIOBase64

extension ByteBuffer {
Expand Down Expand Up @@ -267,7 +269,8 @@ extension ByteBuffer {
return self.setString(String(substring), at: index)
}
}


#if canImport(Dispatch)
// MARK: DispatchData APIs
/// Write `dispatchData` into this `ByteBuffer`, moving the writer index forward appropriately.
///
Expand Down Expand Up @@ -332,7 +335,7 @@ extension ByteBuffer {
self._moveReaderIndex(forwardBy: length)
return result
}

#endif

// MARK: Other APIs

Expand Down Expand Up @@ -660,6 +663,7 @@ extension ByteBuffer {
self = ByteBufferAllocator().buffer(buffer: buffer)
}

#if canImport(Dispatch)
/// Create a fresh `ByteBuffer` containing the bytes contained in the given `DispatchData`.
///
/// This will allocate a new `ByteBuffer` with enough space to fit the bytes of the `DispatchData` and potentially
Expand All @@ -674,6 +678,7 @@ extension ByteBuffer {
public init(dispatchData: DispatchData) {
self = ByteBufferAllocator().buffer(dispatchData: dispatchData)
}
#endif
}

extension ByteBuffer: Codable {
Expand Down Expand Up @@ -785,6 +790,7 @@ extension ByteBufferAllocator {
return newBuffer
}

#if canImport(Dispatch)
/// Create a fresh `ByteBuffer` containing the bytes contained in the given `DispatchData`.
///
/// This will allocate a new `ByteBuffer` with enough space to fit the bytes of the `DispatchData` and potentially
Expand All @@ -797,6 +803,7 @@ extension ByteBufferAllocator {
buffer.writeDispatchData(dispatchData)
return buffer
}
#endif
}


Expand Down
4 changes: 4 additions & 0 deletions Sources/NIOCore/ByteBuffer-conversions.swift
View file
Open in desktop
Expand Up @@ -12,7 +12,9 @@
//
//===----------------------------------------------------------------------===//

#if canImport(Dispatch)
import Dispatch
#endif

extension Array where Element == UInt8 {

Expand Down Expand Up @@ -48,6 +50,7 @@ extension String {
}
}

#if canImport(Dispatch)
extension DispatchData {

/// Creates a `DispatchData` from a given `ByteBuffer`. The entire readable portion of the buffer will be read.
Expand All @@ -59,3 +62,4 @@ extension DispatchData {
}

}
#endif
2 changes: 2 additions & 0 deletions Sources/NIOCore/ByteBuffer-core.swift
View file
Open in desktop
Expand Up @@ -20,6 +20,8 @@ import Darwin
import Glibc
#elseif canImport(Musl)
import Musl
#elseif canImport(WASILibc)
import WASILibc
#else
#error("The Byte Buffer module was unable to identify your C library.")
#endif
Expand Down