Merge pull request #612 from Mossaka/renable-bench

feat(bench): Re-enable `cargo bench`

.github/workflows/benchmarks.yml

@@ -0,0 +1,51 @@
+name: Run Benchmarks
+
+on:
+ schedule:
+ - cron: '0 0 * * *' # Runs daily at midnight
+ pull_request:
+
+jobs:
+ benchmark:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: fetch submodules
+ run: git submodule update --init --recursive
+ - uses: actions-rust-lang/setup-rust-toolchain@v1
+ with:
+ toolchain: stable
+ target: wasm32-wasi
+ - name: Setup build env
+ run: |
+ os=$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')
+ ./scripts/setup-$os.sh
+ shell: bash
+ - name: Run Benchmarks
+ run: |
+ set -o pipefail
+ cargo bench -p containerd-shim-benchmarks -- --output-format bencher | tee output.txt
+ # Download previous benchmark result from cache (if exists)
+ - name: Cache benchmark data
+ uses: actions/cache@v4
+ with:
+ path: ./cache
+ key: ${{ runner.os }}-benchmark
+ - name: Store benchmark result
+ uses: benchmark-action/[email protected]
+ with:
+ name: Criterion.rs Benchmark
+ tool: 'cargo'
+ output-file-path: output.txt
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ # my experimental local benchmarking seems to have a 20% margin of error. 
+ # So I set the alert threshold to 130% of the previous benchmark result.
+ # If the current benchmark result is more than 130% of the previous benchmark result, it will fail.
+ alert-threshold: '130%'
+ fail-on-alert: true
+ alert-comment-cc-users: '@runwasi-committers'
+ # Enable Job Summary
+ summary-always: true
+ # Where the previous data file is stored
+ external-data-json-path: ./cache/benchmark-data.json

Cargo.toml

@@ -6,7 +6,8 @@ members = [
  "crates/oci-tar-builder",
  "crates/containerd-shim-wasmedge",
  "crates/containerd-shim-wasmtime",
- "crates/containerd-shim-wasmer"
+ "crates/containerd-shim-wasmer",
+ "benches/containerd-shim-benchmarks",
 ]
 resolver = "2"
 
@@ -46,5 +47,9 @@ windows-sys = "0.52"
 serial_test = "2"
 tracing = "0.1"
 
+# wasmtime
+wasmtime = { version = "21.0.1", features = ["async"] }
+wasmtime-wasi = { version = "21.0.1" }
+
 [profile.release]
 panic = "abort"

Makefile

@@ -315,6 +315,10 @@ test/k3s-oci-%: dist/img-oci.tar bin/k3s dist-%
 .PHONY: test/k3s/clean
 test/k3s/clean: bin/k3s/clean;
 
+.PHONY: bench
+bench:
+ $(CARGO) bench -p containerd-shim-benchmarks
+
 .PHONY: clean
 clean:
  -rm -rf dist

benches/containerd-shim-benchmarks/Cargo.toml

@@ -6,18 +6,18 @@ edition.workspace = true
 [dependencies]
 anyhow = { workspace = true }
 chrono = { workspace = true }
-containerd-shim-wasm = { path = "../../crates/containerd-shim-wasm" }
-containerd-shim-wasmedge = { path = "../../crates/containerd-shim-wasmedge" }
+containerd-shim-wasm = { path = "../../crates/containerd-shim-wasm", features = ["testing"] }
 containerd-shim-wasmtime = { path = "../../crates/containerd-shim-wasmtime" }
 libc = { workspace = true }
 oci-spec = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tempfile = "3.8"
+wasmtime = { workspace = true }
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
 
 [[bench]]
-name = "webassembly-benchmarks"
+name = "wasmtime-benchmarks"
 harness = false