[FAI-15921] Add e2e testings for Windows system in CI (#131)

Co-authored-by: Chris Wu <[email protected]>

Author: jeniii

.github/workflows/ci.yaml

@@ -63,6 +63,7 @@ jobs:
 
   test:
     name: Test NodeJS Cli
+    needs: [build]
     strategy:
       matrix:
         os: 
@@ -72,8 +73,7 @@ jobs:
           - macos-latest # macos arm64
           - windows-latest # windows x64
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 20
-    needs: [build]
+    timeout-minutes: 60
     steps:
       - name: Check out
         uses: actions/checkout@v4
@@ -110,22 +110,15 @@ jobs:
       # Run the binary: make sure the binary executes fine on the system/arch
       - name: Run binary
         working-directory: airbyte-local-cli-nodejs/out/pkg
-        run: |
-          ./airbyte-local --help
+        run: ./airbyte-local --help
 
+      # We use Shellspec to run the tests in Bash
       - name: Install ShellSpec
-        if: matrix.os == 'ubuntu-latest'
         shell: bash
-        run: |
-          curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes
+        run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes
 
-      ## 
-      # Run ShellSpec tests
-      #  - linux: run on x64 but not arm64 as we don't have arm64 aibtye images
-      #  - macos: doesn't have docker by default in the github runner. Too much work to configure and Macos syntax should be relatively the same as linux
-      #  - windows: TODO
-      ##
-      - name: Run ShellSpec tests
+      # Run ALL e2e tests on Linux x64
+      - name: Run Bash tests (Linux x64)
         if: matrix.os == 'ubuntu-latest'
         working-directory: airbyte-local-cli-nodejs
         env:
@@ -135,6 +128,60 @@ jobs:
           cp -rf ./test/resources ./test/exec/
           shellspec --chdir ./test/exec
 
+      # Run partial e2e tests (excluding docker tests)
+      #  - linux arm64: without pushing arm64 airbye connector images, the tests will fail
+      - name: Run Bash tests (Linux arm64)
+        if: matrix.os == 'ubuntu-arm64'
+        working-directory: airbyte-local-cli-nodejs
+        run: |
+          cp ./out/pkg/airbyte-local ./test/exec/airbyte-local
+          cp -rf ./test/resources ./test/exec/
+          shellspec --chdir ./test/exec --tag argvParsing,main
+
+      # Run partial e2e tests (excluding any tests related to docker check)
+      #  - macos: doesn't have docker by default in the github runner. Too much work to configure and Macos syntax should be relatively the same as linux
+      - name: Run Bash tests (MacOS)
+        if: runner.os == 'MacOS'
+        working-directory: airbyte-local-cli-nodejs
+        run: |
+          cp ./out/pkg/airbyte-local ./test/exec/airbyte-local
+          cp -rf ./test/resources ./test/exec/
+          export PATH="$HOME/.local/bin:$PATH"
+          which shellspec
+          shellspec --chdir ./test/exec --tag argvParsing
+    
+      # Run partial e2e tests on Windows bash (excluding docker tests)
+      # - windows: requires to config the shellspec in PATH. Separete to its own step to run
+      - name: Run Bash tests (Windows)
+        if: matrix.os == 'windows-latest'
+        working-directory: airbyte-local-cli-nodejs
+        shell: bash
+        run: |
+          cp ./out/pkg/airbyte-local ./test/exec/airbyte-local
+          cp -rf ./test/resources ./test/exec/
+
+          # shellspec is installed in $HOME/.local/bin
+          export PATH="$HOME/.local/bin:$PATH"
+          which shellspec
+          shellspec --chdir ./test/exec --tag argvParsing,main
+
+      # Run all e2e tests on Windows powershell
+      # We use Pester to run e2e tests on Windows powershell. Pester is pre-installed on the Windows runner
+      - name: Run Powershell tests (Windows)
+        if: matrix.os == 'windows-latest'
+        working-directory: airbyte-local-cli-nodejs
+        shell: pwsh
+        env:
+          FAROS_API_KEY: ${{ secrets.FAROS_API_KEY }}
+        run: |
+          # cp ./out/pkg/airbyte-local ./test/airbyte-local
+          Copy-Item -Path ./out/pkg/airbyte-local.exe -Destination ./test/airbyte-local.exe
+          # cd ./test
+          Set-Location -Path ./test
+
+          # Run Pester
+          Invoke-Pester -Output Detailed ./pester/index.Tests.ps1
+
   tag:
     name: Tag
     needs: [test]

airbyte-local-cli-nodejs/DEVELOPER.md

@@ -1,4 +1,6 @@
-# WIP: Airbyte CLI Rewrite
+# Airbyte Local CLI
+
+This is the manaual for developers
 
 ```sh
 nvm use           # Use Node v22
@@ -12,7 +14,7 @@ npm run pkg       # Packaging with yao-pkg and generate an executable in out/pkg
 
 ### Node Js Testing
 
-Please make sure you have a valid DEV Faros api key set in `FARO_API_KEY` environment variable and you have your docker on, otherwise some integration tests would fail.
+Please make sure you have a valid DEV Faros api key set in `FAROS_API_KEY` environment variable and you have your docker running, otherwise some integration tests would fail. All tests are running against DEV so you need to create the api key in dev instead of prod.
 
 ```sh
 # Run typescript unit and integration tests
@@ -26,8 +28,13 @@ npm test
 
 ### CLI Executable Testing
 
-For shell testing, we use shellspec. Please install shellspec in your terminal to test it locally.
-Same as above, you will need to set in `FARO_API_KEY` environment variable and have your docker on.
+It should be easy to run Bash testing locally, while we rely Github action to do Powershell testing.
+If you want to add new e2e testing, please make sure you add to both Bash and Powershell if possible.
+
+#### Bash Testing
+
+For Bash shell testing, we use shellspec. Please install shellspec in your terminal to test it locally.
+Same as above, you will need to set DEV `FAROS_API_KEY` environment variable and have your docker running.
 
 ```sh
 # Install shellspec on MacOS
@@ -40,4 +47,30 @@ export FARO_API_KEY="<valid_DEV_faros_api_key>"
 npm run shellspec
 ```
 
+Tests are tagged with 3 different tags `argvParsing`, `main`, `docker`.
+This is mostly for testing in different System/Arch in Github CI. We do different level of testing on different System/Arch depending on how the Github runner's capability, e.g. Github Mac runner doesn't have docker installed.
+
+- `argvParsing`: testing arguments parsing.
+- `main`: testing the main logic in the CLI that is after checking Docker is installed.
+- `docker`: testing anything that will spin up a docker container.
+
 Note: If you want to add new file to test, please remember to add `_spec.sh` postfix to your filename. Otherwise, shellspec won't pick your tests up.
+
+#### Powershell Testing on Windows
+
+We use Pester to run Powershell testing.
+It's now enabled on github action and the test are defined in `./tests/pester/index.Tests.ps1`.
+
+Since Github Windows runner ([Windows Server 2022](https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md)) only support running Windows images, we pushed 3 testing Windows images for Airbyte connnector.
+They are all tagged with `windows-v0.14.11-rc0` as they are based on `v0.14.11`.
+
+- `farosai/airbyte-example-source:windows-v0.14.11-rc0`
+- `farosai/airbyte-faros-graphql-source:windows-v0.14.11-rc0`
+- `farosai/airbyte-faros-destination:windows-v0.14.11-rc0`
+
+Please note that we do not support running Windows images on Windows in the CLI. We only support Linux images on Windows system. These Windows images are for testing purpose only.
+If you want to push or update the images, you can refer to branch [`jg/windows-test-images`](https://github.com/faros-ai/airbyte-connectors/tree/jg/windows-test-images) in repo `faros-ai/airbyte-connectors`. There are Dockerfiles and steps in Github CI to build and push the test images.
+
+## Packaging
+
+We package the NodeJs project to executables by using [yao-pkg](https://github.com/yao-pkg/pkg). We picked it as the popular `pkg` tool from Vercel is archived and `yao-pkg` seems to be the next most maintainable option.

airbyte-local-cli-nodejs/README.md

@@ -44,8 +44,8 @@ Here is the steps of downloading the CLI on MacOS. Linux should have very simila
 
 ```sh
 # Download from the CLI
-# Please swap out `v0.0.1-beta.6` to the version you want
-wget -O airbyte-local.zip https://github.com/faros-ai/airbyte-local-cli/releases/download/v0.0.1-beta.6/airbyte-local-macos-arm64.zip | unzip -o airbyte-local.zip
+# Please swap out `v0.0.5` to the version you want
+wget -O airbyte-local.zip https://github.com/faros-ai/airbyte-local-cli/releases/download/v0.0.5/airbyte-local-macos-arm64.zip | unzip -o airbyte-local.zip
 
 # run `--help` or `--version` to check if the CLI is installed correctly
 ./airbyte-local # this shows the help manual
@@ -57,8 +57,8 @@ wget -O airbyte-local.zip https://github.com/faros-ai/airbyte-local-cli/releases
 
 ```ps1
 # Download from the CLI
-# Please swap out `v0.0.1-beta.6` to the version you want
-Invoke-WebRequest -Uri "https://github.com/faros-ai/airbyte-local-cli/releases/download/v0.0.1-beta.6/airbyte-local-win-x64.zip" -OutFile "airbyte-local-win-x64.zip"
+# Please swap out `v0.0.5` to the version you want
+Invoke-WebRequest -Uri "https://github.com/faros-ai/airbyte-local-cli/releases/download/v0.0.5/airbyte-local-win-x64.zip" -OutFile "airbyte-local-win-x64.zip"
 Expand-Archive -Path "airbyte-local-win-x64.zip" -DestinationPath . -Force
 
 # run `--help` or `--version` to check if the CLI is installed correctly

airbyte-local-cli-nodejs/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@faros-ai/airbyte-local-cli-nodejs",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "Airbyte local cli node js version",
   "private": true,
   "packageManager": "^[email protected]",