Test swiftly release artifacts in a plain Linux environment

.github/workflows/pull_request.yml

@@ -39,13 +39,81 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Build Artifact
-        run: swift run build-swiftly-release --skip "999.0.0"
+        run: swift run build-swiftly-release --skip --tests "999.0.0"
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         with:
+          name: swiftly-release-x86_64
           path: .build/release/swiftly-*.tar.gz
           if-no-files-found: error
           retention-days: 1
+      - name: Upload Tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+          path: .build/debug/test-swiftly.tar.gz
+          if-no-files-found: error
+          retention-days: 1
+
+  releasetest:
+    name: Test Release
+    needs: releasebuildcheck
+    runs-on: ubuntu-latest
+    container:
+      image: "ubuntu:22.04"
+    steps:
+      - name: Prepare System
+        run: apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install ca-certificates gpg tzdata
+      - name: Download Release
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-release-x86_64
+      - name: Download Tests
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+      - name: Extract and Run Workflow Tests
+        run: cp swiftly-*.tar.gz /root/swiftly.tar.gz && cp test-swiftly.tar.gz /root && cd /root && tar zxf test-swiftly.tar.gz && SWIFTLY_SYSTEM_MUTATING=1 ./swiftlyPackageTests.xctest SwiftlyTests.E2ETests/testAutomatedWorkflow
+
+  releasetestzsh:
+    name: Test Release ZSH
+    needs: releasebuildcheck
+    runs-on: ubuntu-latest
+    container:
+      image: "ubuntu:24.04"
+    steps:
+      - name: Prepare System
+        run: apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install ca-certificates gpg tzdata zsh && chsh -s /bin/zsh && echo '. /root/.zprofile' > /root/.zshenv
+      - name: Download Release
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-release-x86_64
+      - name: Download Tests
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+      - name: Extract and Run Workflow Tests
+        run: cp swiftly-*.tar.gz /root/swiftly.tar.gz && cp test-swiftly.tar.gz /root && cd /root && tar zxf test-swiftly.tar.gz && SWIFTLY_SYSTEM_MUTATING=1 ./swiftlyPackageTests.xctest SwiftlyTests.E2ETests/testAutomatedWorkflow
+
+  releasetestfish:
+    name: Test Release FISH
+    needs: releasebuildcheck
+    runs-on: ubuntu-latest
+    container:
+      image: "ubuntu:24.04"
+    steps:
+      - name: Prepare System
+        run: apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install ca-certificates gpg tzdata fish && chsh -s /bin/fish
+      - name: Download Release
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-release-x86_64
+      - name: Download Tests
+        uses: actions/download-artifact@v4
+        with:
+          name: swiftly-tests-x86_64
+      - name: Extract and Run Workflow Tests
+        run: cp swiftly-*.tar.gz /root/swiftly.tar.gz && cp test-swiftly.tar.gz /root && cd /root && tar zxf test-swiftly.tar.gz && SWIFTLY_SYSTEM_MUTATING=1 ./swiftlyPackageTests.xctest SwiftlyTests.E2ETests/testAutomatedWorkflow
 
   formatcheck:
     name: Format Check

.swift-version

@@ -1 +1 @@
-6.0
+6.0.2

Tests/SwiftlyTests/E2ETests.swift

@@ -63,4 +63,46 @@ final class E2ETests: SwiftlyTests {
             try await validateInstalledToolchains([installedToolchain], description: "install latest")
         }
     }
+
+    func testAutomatedWorkflow() async throws {
+        try XCTSkipIf(ProcessInfo.processInfo.environment["SWIFTLY_SYSTEM_MUTATING"] == nil, "Not running test since it mutates the system and SWIFTLY_SYSTEM_MUTATING environment variable is not set. This test should be run in a throw away environment, such as a container.")
+
+        print("Extracting swiftly release")
+#if os(Linux)
+        try Swiftly.currentPlatform.runProgram("tar", "-zxvf", "swiftly.tar.gz", quiet: false)
+#elseif os(macOS)
+        try Swiftly.currentPlatform.runProgram("installer", "-pkg", "swiftly.pkg", "-target", "CurrentUserHomeDirectory", quiet: false)
+#endif
+
+        print("Running 'swiftly init --assume-yes --verbose' to install swiftly and the latest toolchain")
+
+#if os(Linux)
+        let extractedSwiftly = "./swiftly"
+#elseif os(macOS)
+        let extractedSwiftly = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("usr/local/bin/swiftly").path
+#endif
+
+        try Swiftly.currentPlatform.runProgram(extractedSwiftly, "init", "--assume-yes", quiet: false)
+
+        let shell = try await Swiftly.currentPlatform.getShell()
+
+        var env = ProcessInfo.processInfo.environment
+
+        // Setting this environment helps to ensure that the profile gets sourced with bash, even if it is not in an interactive shell
+        if shell.hasSuffix("bash") {
+            env["BASH_ENV"] = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".profile").path
+        } else if shell.hasSuffix("zsh") {
+            env["ZDOTDIR"] = FileManager.default.homeDirectoryForCurrentUser.path
+        } else if shell.hasSuffix("fish") {
+            env["XDG_CONFIG_HOME"] = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".config").path
+        }
+
+        try Swiftly.currentPlatform.runProgram(shell, "-l", "-c", "swiftly install --assume-yes latest --post-install-file=./post-install.sh", env: env)
+
+        if FileManager.default.fileExists(atPath: "./post-install.sh") {
+            try Swiftly.currentPlatform.runProgram(shell, "./post-install.sh")
+        }
+
+        try Swiftly.currentPlatform.runProgram(shell, "-l", "-c", "swift --version", quiet: false, env: env)
+    }
 }

Tools/build-swiftly-release/BuildSwiftlyRelease.swift

@@ -40,6 +40,10 @@ public func runProgramEnv(_ args: String..., quiet: Bool = false, env: [String:
 }
 
 public func runProgram(_ args: String..., quiet: Bool = false) throws {
+    try runProgram(args, quiet: quiet)
+}
+
+public func runProgram(_ args: [String], quiet: Bool = false) throws {
     let process = Process()
     process.executableURL = URL(fileURLWithPath: "/usr/bin/env")
     process.arguments = args
@@ -190,6 +194,9 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
     var identifier: String = "org.swift.swiftly"
 #endif
 
+    @Flag(help: "Produce a swiftly-test.tar.gz that has a standalone test suite to test the released bundle.")
+    var tests: Bool = false
+
     @Argument(help: "Version of swiftly to build the release.")
     var version: String
 
@@ -380,11 +387,33 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
         try runProgram(tar, "--directory=\(releaseDir)", "-czf", releaseArchive, "swiftly", "LICENSE.txt")
 
         print(releaseArchive)
+
+        if self.tests {
+            try runProgram(swift, "build", "--build-tests", "--pkg-config-path=\(pkgConfigPath)/lib/pkgconfig")
+
+            // Copy swift standard libraries to the build directory to bundle them with the test executable
+            let swiftLibEnum = FileManager.default.enumerator(atPath: "/usr/lib/swift")
+            var soFiles: [String] = []
+
+            while let file = swiftLibEnum?.nextObject() as? String {
+                if file.hasSuffix(".so") {
+                    let baseName = file.split(separator: "/").last!
+                    try? FileManager.default.removeItem(atPath: cwd + "/.build/debug/" + baseName)
+                    try FileManager.default.copyItem(atPath: "/usr/lib/swift/" + file, toPath: cwd + "/.build/debug/" + baseName)
+                    soFiles.append(String(baseName))
+                }
+            }
+
+            try runProgram([tar, "--directory=\(cwd + "/.build/debug")", "-czf", "\(cwd + "/.build/debug/test-swiftly.tar.gz")", "swiftlyPackageTests.xctest"] + soFiles)
+
+            print(".build/debug/test-swiftly.tar.gz")
+        }
     }
 
     func buildMacOSRelease(cert: String?, identifier: String) async throws {
         // Check system requirements
         let git = try await self.assertTool("git", message: "Please install git with either `xcode-select --install` or `brew install git`")
+        let tar = try await self.assertTool("tar", message: "Tar not found")
 
         let swift = try await checkSwiftRequirement()
 
@@ -439,5 +468,15 @@ struct BuildSwiftlyRelease: AsyncParsableCommand {
                 ".build/release/swiftly-\(self.version).pkg"
             )
         }
+
+        print(".build/release/swiftly-\(self.version).pkg")
+
+        if self.tests {
+            try runProgram(swift, "build", "--build-tests")
+
+            try runProgram([tar, "--directory=.build/debug", "-czf", ".build/debug/test-swiftly.tar.gz", "swiftlyPackageTests.xctest"])
+
+            print(".build/debug/test-swiftly.tar.gz")
+        }
     }
 }