Yandu/use overlay only

.github/workflows/test-integration.yml

@@ -0,0 +1,31 @@
+name: Test Integration
+
+on:
+  push:
+  workflow_call:
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup development environment
+        uses: ./.github/actions/setup-env
+
+      - name: Start background services
+        run: dev &
+
+      - name: Install js modules for HardLink tests
+        run: make install-js
+
+      - name: Build Go binaries
+        run: make build
+
+      - name: Run Go tests
+        run: make test-integration

Makefile

@@ -24,16 +24,17 @@ PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto')
 MIGRATE_DIR := ./migrations
 SERVICE := $(PROJECT).server
 BENCH_PROFILE ?= ""
+KUBE_CONTEXT ?= orbstack
 
 .PHONY: migrate migrate-create clean build lint release
-.PHONY: test test-one test-fuzz test-js lint-js install-js build-js
+.PHONY: test test-one test-fuzz test-js test-integration lint-js install-js build-js
 .PHONY: reset-db setup-local build-cache-version server server-profile cached
 .PHONY: client-update client-large-update client-get client-rebuild client-rebuild-with-cache
 .PHONY: client-getcache client-gc-contents client-gc-project client-gc-random-projects
 .PHONY: cachedclient-probe cachedclient-populate cachedclient-stats
 .PHONY: health upload-container-image upload-prerelease-container-image run-container gen-docs
 .PHONY: load-test-new load-test-update load-test-update-large load-test-get load-test-get-compress
-
+.PHONY: k8s k8s/start k8s/stop k8s/delete k8s/reset k8s/deploy
 migrate:
 	migrate -database $(DB_URI)?sslmode=disable -path $(MIGRATE_DIR) up
 
@@ -106,6 +107,11 @@ else
 	cd test && go test -run $(name)
 endif
 
+test-integration: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
+test-integration: export RUN_WITH_SUDO = true
+test-integration: migrate
+	cd test && go test -tags integration
+
 bench: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests
 bench: migrate
 	cd test && go test -bench . -run=^# $(BENCH_PROFILE)
@@ -283,3 +289,26 @@ lint-js: js/node_modules js/src/pb
 install-js: js/node_modules
 
 build-js: js/dist
+
+k8s:
+	which orb >/dev/null 2>&1; if [ $$? -ne 0 ]; then echo "orb not found"; exit 1; fi
+k8s/start: k8s	
+	orb start k8s
+
+k8s/stop: k8s
+	orb stop k8s
+
+k8s/delete: k8s
+	orb delete k8s
+
+k8s/reset: k8s/stop k8s/delete k8s/start
+
+k8s/deploy: k8s k8s/start
+	kubectl --context=$(KUBE_CONTEXT) create namespace dateilager-local || true
+	kubectl --context=$(KUBE_CONTEXT) apply -f test/k8s-local/cached-csi.yaml -n dateilager-local
+	kubectl --context=$(KUBE_CONTEXT) apply -f test/k8s-local/cached-daemon.yaml -n dateilager-local
+
+k8s/reset_namespace: k8s k8s/start
+	kubectl --context=$(KUBE_CONTEXT) delete ds dateilager-csi-cached -n dateilager-local --force --grace-period=0 || true
+	kubectl --context=$(KUBE_CONTEXT) delete pod busybox-csi -n dateilager-local --force --grace-period=0 || true
+	kubectl --context=$(KUBE_CONTEXT) delete namespace dateilager-local || true

flake.nix

@@ -81,6 +81,7 @@
               flake.packages.glibcLocales
               flake.packages.google-cloud-sdk
               git
+              k3d
               protobuf
               protoc-gen-go
               protoc-gen-go-grpc

go.mod

@@ -1,8 +1,11 @@
 module github.com/gadget-inc/dateilager
 
-go 1.22
+go 1.23.0
+
+toolchain go1.23.6
 
 require (
+	github.com/charlievieth/fastwalk v1.0.9
 	github.com/container-storage-interface/spec v1.9.0
 	github.com/dgraph-io/ristretto v0.1.1
 	github.com/gadget-inc/fsdiff v0.4.4
@@ -16,67 +19,66 @@ require (
 	github.com/minio/sha256-simd v1.0.0
 	github.com/o1egl/paseto v1.0.0
 	github.com/spf13/cobra v1.6.0
-	github.com/stretchr/testify v1.8.3
+	github.com/stretchr/testify v1.10.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0
 	go.opentelemetry.io/otel v1.16.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0
 	go.opentelemetry.io/otel/sdk v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0
 	go.uber.org/zap v1.23.0
-	golang.org/x/oauth2 v0.15.0
-	golang.org/x/sync v0.5.0
-	golang.org/x/sys v0.19.0
+	golang.org/x/oauth2 v0.23.0
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.26.0
 	google.golang.org/grpc v1.59.0
-	google.golang.org/protobuf v1.33.0
+	google.golang.org/protobuf v1.35.1
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
 	github.com/aead/chacha20poly1305 v0.0.0-20201124145622-1a5aba2a8b29 // indirect
 	github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/charlievieth/fastwalk v1.0.9 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang/glog v1.1.2 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
-	github.com/google/uuid v1.3.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/klauspost/cpuid/v2 v2.1.2 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/onsi/gomega v1.33.1 // indirect
+	github.com/onsi/ginkgo/v2 v2.21.0 // indirect
+	github.com/onsi/gomega v1.35.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )