diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml new file mode 100644 index 0000000..7ded20d --- /dev/null +++ b/.github/workflows/test-integration.yml @@ -0,0 +1,31 @@ +name: Test Integration + +on: + push: + workflow_call: + +defaults: + run: + shell: bash -l {0} + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup development environment + uses: ./.github/actions/setup-env + + - name: Start background services + run: dev & + + - name: Install js modules for HardLink tests + run: make install-js + + - name: Build Go binaries + run: make build + + - name: Run Go tests + run: make test-integration \ No newline at end of file diff --git a/Makefile b/Makefile index 1c37699..6efbaa0 100644 --- a/Makefile +++ b/Makefile @@ -24,16 +24,17 @@ PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto') MIGRATE_DIR := ./migrations SERVICE := $(PROJECT).server BENCH_PROFILE ?= "" +KUBE_CONTEXT ?= orbstack .PHONY: migrate migrate-create clean build lint release -.PHONY: test test-one test-fuzz test-js lint-js install-js build-js +.PHONY: test test-one test-fuzz test-js test-integration lint-js install-js build-js .PHONY: reset-db setup-local build-cache-version server server-profile cached .PHONY: client-update client-large-update client-get client-rebuild client-rebuild-with-cache .PHONY: client-getcache client-gc-contents client-gc-project client-gc-random-projects .PHONY: cachedclient-probe cachedclient-populate cachedclient-stats .PHONY: health upload-container-image upload-prerelease-container-image run-container gen-docs .PHONY: load-test-new load-test-update load-test-update-large load-test-get load-test-get-compress - +.PHONY: k8s k8s/start k8s/stop k8s/delete k8s/reset k8s/deploy migrate: migrate -database $(DB_URI)?sslmode=disable -path $(MIGRATE_DIR) up @@ -106,6 +107,11 @@ else cd test && go test -run $(name) endif +test-integration: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests +test-integration: export RUN_WITH_SUDO = true +test-integration: migrate + cd test && go test -tags integration + bench: export DB_URI = postgres://$(DB_USER):$(DB_PASS)@$(DB_HOST):5432/dl_tests bench: migrate cd test && go test -bench . -run=^# $(BENCH_PROFILE) @@ -283,3 +289,26 @@ lint-js: js/node_modules js/src/pb install-js: js/node_modules build-js: js/dist + +k8s: + which orb >/dev/null 2>&1; if [ $$? -ne 0 ]; then echo "orb not found"; exit 1; fi +k8s/start: k8s + orb start k8s + +k8s/stop: k8s + orb stop k8s + +k8s/delete: k8s + orb delete k8s + +k8s/reset: k8s/stop k8s/delete k8s/start + +k8s/deploy: k8s k8s/start + kubectl --context=$(KUBE_CONTEXT) create namespace dateilager-local || true + kubectl --context=$(KUBE_CONTEXT) apply -f test/k8s-local/cached-csi.yaml -n dateilager-local + kubectl --context=$(KUBE_CONTEXT) apply -f test/k8s-local/cached-daemon.yaml -n dateilager-local + +k8s/reset_namespace: k8s k8s/start + kubectl --context=$(KUBE_CONTEXT) delete ds dateilager-csi-cached -n dateilager-local --force --grace-period=0 || true + kubectl --context=$(KUBE_CONTEXT) delete pod busybox-csi -n dateilager-local --force --grace-period=0 || true + kubectl --context=$(KUBE_CONTEXT) delete namespace dateilager-local || true diff --git a/flake.nix b/flake.nix index 1125981..af2847e 100644 --- a/flake.nix +++ b/flake.nix @@ -81,6 +81,7 @@ flake.packages.glibcLocales flake.packages.google-cloud-sdk git + k3d protobuf protoc-gen-go protoc-gen-go-grpc diff --git a/go.mod b/go.mod index ef54613..c91cd8f 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,11 @@ module github.com/gadget-inc/dateilager -go 1.22 +go 1.23.0 + +toolchain go1.23.6 require ( + github.com/charlievieth/fastwalk v1.0.9 github.com/container-storage-interface/spec v1.9.0 github.com/dgraph-io/ristretto v0.1.1 github.com/gadget-inc/fsdiff v0.4.4 @@ -16,7 +19,7 @@ require ( github.com/minio/sha256-simd v1.0.0 github.com/o1egl/paseto v1.0.0 github.com/spf13/cobra v1.6.0 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.10.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 @@ -24,59 +27,58 @@ require ( go.opentelemetry.io/otel/sdk v1.16.0 go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/zap v1.23.0 - golang.org/x/oauth2 v0.15.0 - golang.org/x/sync v0.5.0 - golang.org/x/sys v0.19.0 + golang.org/x/oauth2 v0.23.0 + golang.org/x/sync v0.8.0 + golang.org/x/sys v0.26.0 google.golang.org/grpc v1.59.0 - google.golang.org/protobuf v1.33.0 + google.golang.org/protobuf v1.35.1 ) require ( - cloud.google.com/go/compute v1.23.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/aead/chacha20poly1305 v0.0.0-20201124145622-1a5aba2a8b29 // indirect github.com/aead/poly1305 v0.0.0-20180717145839-3fee0db0b635 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/charlievieth/fastwalk v1.0.9 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/glog v1.1.2 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/klauspost/cpuid/v2 v2.1.2 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect - github.com/onsi/gomega v1.33.1 // indirect + github.com/onsi/ginkgo/v2 v2.21.0 // indirect + github.com/onsi/gomega v1.35.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.10.0 // indirect + go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/text v0.19.0 // indirect google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d0e7547..bfd4e94 100644 --- a/go.sum +++ b/go.sum @@ -19,10 +19,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -76,8 +74,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -105,8 +104,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -172,12 +171,12 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= @@ -213,8 +212,8 @@ github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -232,24 +231,27 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= -github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -265,8 +267,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -299,8 +301,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -313,8 +315,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -375,16 +377,16 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -394,8 +396,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -434,8 +436,8 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -443,8 +445,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -491,8 +493,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -519,8 +521,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -590,8 +590,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -603,8 +603,9 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/api/cached.go b/pkg/api/cached.go index 73dc51e..44e00cf 100644 --- a/pkg/api/cached.go +++ b/pkg/api/cached.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "os" + "os/exec" "path" "path/filepath" "syscall" @@ -26,7 +27,10 @@ import ( ) const ( - DriverName = "com.gadget.dateilager.cached" + DriverName = "com.gadget.dateilager.cached" + CACHE_PATH_SUFFIX = "dl_cache" + UPPER_DIR = "upper" + WORK_DIR = "work" ) type Cached struct { @@ -58,17 +62,29 @@ func (c *Cached) PopulateDiskCache(ctx context.Context, req *pb.PopulateDiskCach return &pb.PopulateDiskCacheResponse{Version: version}, nil } +func (c *Cached) GetCachePath() string { + return filepath.Join(c.StagingPath, CACHE_PATH_SUFFIX) +} + // Fetch the cache into the staging dir func (c *Cached) Prepare(ctx context.Context) error { start := time.Now() - version, count, err := c.Client.GetCache(ctx, c.StagingPath) + version, count, err := c.Client.GetCache(ctx, c.GetCachePath()) if err != nil { return err } + // Once we've prepared the cache make it read-only for + // everyone except the user running the daemon + err = os.Chmod(c.GetCachePath(), 0755) + if err != nil { + return fmt.Errorf("failed to change permissions of cache path %s: %v", c.GetCachePath(), err) + } + c.currentVersion = version + logger.Info(ctx, c.GetCachePath()) logger.Info(ctx, "downloaded golden copy", key.DurationMS.Field(time.Since(start)), key.Version.Field(version), key.Count.Field(int64(count))) return nil } @@ -140,39 +156,81 @@ func (c *Cached) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu } targetPath := req.GetTargetPath() - volumeID := req.GetVolumeId() - volumeAttributes := req.GetVolumeContext() + // The parent of the target path and can store CSI metadata, it's the name given to the volume mount in the pod spec + volumePath := path.Join(targetPath, "..") + var version int64 - var cachePath string - var targetPermissions os.FileMode + // Perform an overlay mount + upperdir := path.Join(volumePath, UPPER_DIR) + err := os.MkdirAll(upperdir, 0777) + if err != nil { + return nil, fmt.Errorf("failed to create overlay upper directory %s: %v", upperdir, err) + } - if suffix, exists := volumeAttributes["placeCacheAtPath"]; exists { - // running in suffix mode, desired outcome: - // - the mount point is writable by the pod - // - the cache is mounted at the suffix, and is not writable - cachePath = path.Join(targetPath, suffix) - targetPermissions = 0777 - } else { - // running in unsuffixed mode, desired outcome: - // - the mount point *is* the cache, and is not writable by the pod - cachePath = targetPath - targetPermissions = 0755 + upperInfo, err := os.Stat(upperdir) + if err != nil { + return nil, fmt.Errorf("failed to stat overlay upper directory %s: %v", upperdir, err) + } + if upperInfo.Mode()&os.ModePerm != 0777 { + err = os.Chmod(upperdir, 0777) + if err != nil { + return nil, fmt.Errorf("failed to change permissions of overlay upper directory %s: %v", upperdir, err) + } } - if err := os.MkdirAll(targetPath, targetPermissions); err != nil { - return nil, fmt.Errorf("failed to create target directory %s: %s", targetPath, err) + workdir := path.Join(volumePath, WORK_DIR) + err = os.MkdirAll(workdir, 0777) + if err != nil { + return nil, fmt.Errorf("failed to create overlay work directory %s: %v", workdir, err) } - if err := os.Chmod(targetPath, targetPermissions); err != nil { - return nil, fmt.Errorf("failed to change ownership of target directory %s: %s", targetPath, err) + workInfo, err := os.Stat(workdir) + if err != nil { + return nil, fmt.Errorf("failed to stat overlay work directory %s: %v", workdir, err) + } + if workInfo.Mode()&os.ModePerm != 0777 { + err = os.Chmod(workdir, 0777) + if err != nil { + return nil, fmt.Errorf("failed to change permissions of overlay work directory %s: %v", workdir, err) + } } - version, err := c.writeCache(cachePath) + // Create the cache directory, we can make it writable by the pod + err = os.MkdirAll(targetPath, 0777) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create target path directory %s: %v", targetPath, err) + } + + mountArgs := []string{ + "-t", + "overlay", + "overlay", + "-n", + "--options", + fmt.Sprintf("redirect_dir=on,lowerdir=%s,upperdir=%s,workdir=%s", c.StagingPath, upperdir, workdir), + targetPath, } - logger.Info(ctx, "volume published", key.VolumeID.Field(volumeID), key.TargetPath.Field(targetPath), key.Version.Field(version)) + err = execCommand("mount", mountArgs...) + if err != nil { + return nil, fmt.Errorf("failed to mount overlay: %s", err) + } + + cachePath := path.Join(targetPath, CACHE_PATH_SUFFIX) + info, err := os.Stat(cachePath) + if err != nil { + return nil, fmt.Errorf("failed to stat cache path %s, this path should exist in the overlay mount at %s: %v", cachePath, targetPath, err) + } + + if info.Mode()&os.ModePerm != 0755 { + err = os.Chmod(cachePath, 0755) + if err != nil { + return nil, fmt.Errorf("failed to change permissions of cache path %s: %v", cachePath, err) + } + } + + version = c.currentVersion + logger.Info(ctx, "mounted overlay", key.TargetPath.Field(targetPath), key.Version.Field(version)) return &csi.NodePublishVolumeResponse{}, nil } @@ -187,9 +245,19 @@ func (s *Cached) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublish } targetPath := req.GetTargetPath() + volumePath := path.Join(targetPath, "..") - // Clean up directory - if err := os.RemoveAll(targetPath); err != nil { + // Unmount the overlay + err := execCommand("umount", targetPath) + if err != nil { + return nil, fmt.Errorf("failed to unmount overlay at %s: %v", targetPath, err) + } + + // Clean up upper and work directories from the overlay + if err := os.RemoveAll(path.Join(volumePath, UPPER_DIR)); err != nil { + return nil, fmt.Errorf("failed to remove directory %s: %s", targetPath, err) + } + if err := os.RemoveAll(path.Join(volumePath, WORK_DIR)); err != nil { return nil, fmt.Errorf("failed to remove directory %s: %s", targetPath, err) } @@ -260,7 +328,7 @@ func (c *Cached) writeCache(destination string) (int64, error) { } } - err = files.HardlinkDir(c.StagingPath, destination) + err = files.HardlinkDir(c.GetCachePath(), destination) if err != nil { return -1, fmt.Errorf("failed to hardlink cache to destination %s: %v", destination, err) } @@ -287,3 +355,13 @@ func getFolderSize(path string) (int64, error) { }) return totalSize, err } + +func execCommand(cmdName string, args ...string) error { + if os.Getenv("RUN_WITH_SUDO") == "true" { + cmd := exec.Command("sudo", append([]string{cmdName}, args...)...) + return cmd.Run() + } + + cmd := exec.Command(cmdName, args...) + return cmd.Run() +} diff --git a/test/cached_csi_test.go b/test/cached_csi_test.go index 69d885a..d87b23e 100644 --- a/test/cached_csi_test.go +++ b/test/cached_csi_test.go @@ -1,3 +1,6 @@ +//go:build integration +// +build integration + package test import ( @@ -10,6 +13,8 @@ import ( "github.com/gadget-inc/dateilager/internal/auth" "github.com/gadget-inc/dateilager/internal/db" util "github.com/gadget-inc/dateilager/internal/testutil" + "github.com/gadget-inc/dateilager/pkg/api" + "github.com/gadget-inc/dateilager/pkg/cached" "github.com/kubernetes-csi/csi-test/pkg/sanity" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -76,7 +81,7 @@ func TestCachedCSIDriverMountsCache(t *testing.T) { }) require.NoError(t, err) - verifyDir(t, targetDir, -1, map[string]expectedFile{ + verifyDir(t, path.Join(targetDir, "dl_cache"), -1, map[string]expectedFile{ fmt.Sprintf("objects/%v/pack/a/1", aHash): {content: "pack/a/1 v1"}, fmt.Sprintf("objects/%v/pack/a/2", aHash): {content: "pack/a/2 v1"}, fmt.Sprintf("objects/%v/pack/b/1", bHash): {content: "pack/b/1 v1"}, @@ -84,16 +89,34 @@ func TestCachedCSIDriverMountsCache(t *testing.T) { "versions": {content: fmt.Sprintf("%v\n", version)}, }) + // Check to see that we have created the upper and work directories + require.DirExists(t, path.Join(tmpDir, api.UPPER_DIR)) + require.DirExists(t, path.Join(tmpDir, api.WORK_DIR)) + + upperInfo, err := os.Stat(path.Join(tmpDir, api.UPPER_DIR)) + require.NoError(t, err) + require.Equal(t, formatFileMode(os.FileMode(0777)), formatFileMode(upperInfo.Mode()&os.ModePerm)) + + workInfo, err := os.Stat(path.Join(tmpDir, api.WORK_DIR)) + require.NoError(t, err) + require.Equal(t, formatFileMode(os.FileMode(0777)), formatFileMode(workInfo.Mode()&os.ModePerm)) + fileInfo, err := os.Stat(targetDir) require.NoError(t, err) // the target dir should not be world writable -- only by the user the CSI driver is running as (which will be root) - require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(fileInfo.Mode()&os.ModePerm)) + require.Equal(t, formatFileMode(os.FileMode(0777)), formatFileMode(fileInfo.Mode()&os.ModePerm)) // files inside cache dir should also *not* be writable -- it's managed by the CSI and must remain pristine - cacheFileInfo, err := os.Stat(path.Join(targetDir, fmt.Sprintf("objects/%v/pack/a/1", aHash))) + cacheFileInfo, err := os.Stat(path.Join(targetDir, "dl_cache", fmt.Sprintf("objects/%v/pack/a/1", aHash))) require.NoError(t, err) require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(cacheFileInfo.Mode()&os.ModePerm)) + + _, err = cached.NodeUnpublishVolume(tc.Context(), &csi.NodeUnpublishVolumeRequest{ + VolumeId: "foobar", + TargetPath: targetDir, + }) + require.NoError(t, err) } func TestCachedCSIDriverMountsCacheAtSuffix(t *testing.T) { @@ -117,23 +140,37 @@ func TestCachedCSIDriverMountsCacheAtSuffix(t *testing.T) { require.NoError(t, err, "cached.Prepare must succeed") targetDir := path.Join(tmpDir, "vol-target") + + stagingDir := path.Join(tmpDir, "vol-staging-target") _, err = cached.NodePublishVolume(tc.Context(), &csi.NodePublishVolumeRequest{ VolumeId: "foobar", - StagingTargetPath: path.Join(tmpDir, "vol-staging-target"), + StagingTargetPath: stagingDir, TargetPath: targetDir, VolumeCapability: &csi.VolumeCapability{}, - VolumeContext: map[string]string{"placeCacheAtPath": "inner_mount"}, + VolumeContext: map[string]string{}, }) require.NoError(t, err) verifyDir(t, path.Join(tmpDir, "vol-target"), -1, map[string]expectedFile{ - fmt.Sprintf("inner_mount/objects/%v/pack/a/1", aHash): {content: "pack/a/1 v1"}, - fmt.Sprintf("inner_mount/objects/%v/pack/a/2", aHash): {content: "pack/a/2 v1"}, - fmt.Sprintf("inner_mount/objects/%v/pack/b/1", bHash): {content: "pack/b/1 v1"}, - fmt.Sprintf("inner_mount/objects/%v/pack/b/2", bHash): {content: "pack/b/2 v1"}, - "inner_mount/versions": {content: fmt.Sprintf("%v\n", version)}, + fmt.Sprintf("dl_cache/objects/%v/pack/a/1", aHash): {content: "pack/a/1 v1"}, + fmt.Sprintf("dl_cache/objects/%v/pack/a/2", aHash): {content: "pack/a/2 v1"}, + fmt.Sprintf("dl_cache/objects/%v/pack/b/1", bHash): {content: "pack/b/1 v1"}, + fmt.Sprintf("dl_cache/objects/%v/pack/b/2", bHash): {content: "pack/b/2 v1"}, + "dl_cache/versions": {content: fmt.Sprintf("%v\n", version)}, }) + // Check to see that we have created the upper and work directories + require.DirExists(t, path.Join(tmpDir, api.UPPER_DIR)) + require.DirExists(t, path.Join(tmpDir, api.WORK_DIR)) + + upperInfo, err := os.Stat(path.Join(tmpDir, api.UPPER_DIR)) + require.NoError(t, err) + require.Equal(t, formatFileMode(os.FileMode(0777)), formatFileMode(upperInfo.Mode()&os.ModePerm)) + + workInfo, err := os.Stat(path.Join(tmpDir, api.WORK_DIR)) + require.NoError(t, err) + require.Equal(t, formatFileMode(os.FileMode(0777)), formatFileMode(workInfo.Mode()&os.ModePerm)) + fileInfo, err := os.Stat(targetDir) require.NoError(t, err) @@ -141,14 +178,21 @@ func TestCachedCSIDriverMountsCacheAtSuffix(t *testing.T) { require.Equal(t, formatFileMode(os.FileMode(0777)), formatFileMode(fileInfo.Mode()&os.ModePerm)) // the cache dir should *not* be writable -- it's managed by the CSI and must remain pristine - cacheFileInfo, err := os.Stat(path.Join(targetDir, "inner_mount")) + cacheFileInfo, err := os.Stat(path.Join(targetDir, "dl_cache")) require.NoError(t, err) require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(cacheFileInfo.Mode()&os.ModePerm)) // files inside cache dir should *not* be writable -- it's managed by the CSI and must remain pristine - cacheFileInfo, err = os.Stat(path.Join(targetDir, fmt.Sprintf("inner_mount/objects/%v/pack/a/1", aHash))) + cacheFileInfo, err = os.Stat(path.Join(targetDir, fmt.Sprintf("dl_cache/objects/%v/pack/a/1", aHash))) require.NoError(t, err) require.Equal(t, formatFileMode(os.FileMode(0755)), formatFileMode(cacheFileInfo.Mode())) + require.Equal(t, targetDir, path.Join(tmpDir, "vol-target")) + + _, err = cached.NodeUnpublishVolume(tc.Context(), &csi.NodeUnpublishVolumeRequest{ + VolumeId: "foobar", + TargetPath: targetDir, + }) + require.NoError(t, err) } func TestCachedCSIDriverProbeFailsUntilPrepared(t *testing.T) { @@ -187,3 +231,26 @@ func TestCachedCSIDriverProbeFailsUntilPrepared(t *testing.T) { func formatFileMode(mode os.FileMode) string { return fmt.Sprintf("%#o", mode) } + +func createTestCachedServer(tc util.TestCtx, tmpDir string) (*api.Cached, string, func()) { + cl, _, closeClient := createTestClient(tc) + _, grpcServer, _ := createTestGRPCServer(tc) + + s := cached.CachedServer{ + Grpc: grpcServer, + } + + cached := tc.CachedApi(cl, path.Join(tmpDir, "cached", "staging")) + s.RegisterCached(cached) + s.RegisterCSI(cached) + + socket := path.Join(tmpDir, "csi.sock") + endpoint := "unix://" + socket + + go func() { + err := s.Serve(endpoint) + require.NoError(tc.T(), err, "CSI Server exited") + }() + + return cached, endpoint, func() { closeClient(); s.Grpc.Stop() } +} diff --git a/test/hardlink_dir_benchmark_test.go b/test/hardlink_dir_benchmark_test.go index d05bcf3..18b4da1 100644 --- a/test/hardlink_dir_benchmark_test.go +++ b/test/hardlink_dir_benchmark_test.go @@ -26,6 +26,7 @@ func TestHardlinkDir(t *testing.T) { require.NoError(t, err, "compareDirectories %s vs %s failed", bigDir, tmpDir) } +//go:bench func BenchmarkHardlinkDir(b *testing.B) { wd, err := os.Getwd() if err != nil { diff --git a/test/hash_benchmark_test.go b/test/hash_benchmark_test.go index 5dfa8b1..7f8e08b 100644 --- a/test/hash_benchmark_test.go +++ b/test/hash_benchmark_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" ) +//go:bench func BenchmarkHex(b *testing.B) { data := make([]byte, 64) _, err := rand.Read(data) diff --git a/test/k8s-local/busy-box-debug.yaml b/test/k8s-local/busy-box-debug.yaml new file mode 100644 index 0000000..2ffcccc --- /dev/null +++ b/test/k8s-local/busy-box-debug.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: busybox-csi +spec: + terminationGracePeriodSeconds: 5 + containers: + - name: busybox + image: busybox + command: + - "/bin/sh" + - "-c" + - "sleep infinity" + volumeMounts: + - name: gadget + mountPath: /gadget + volumes: + - name: gadget + csi: + driver: com.gadget.dateilager.cached + volumeAttributes: + placeCacheAtPath: "dl_cache" + mountCache: "true" + readOnly: false diff --git a/test/k8s-local/cached-csi.yaml b/test/k8s-local/cached-csi.yaml new file mode 100644 index 0000000..87bfaad --- /dev/null +++ b/test/k8s-local/cached-csi.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: com.gadget.dateilager.cached + labels: + origin: krane +spec: + attachRequired: false + volumeLifecycleModes: + - Ephemeral \ No newline at end of file diff --git a/test/k8s-local/cached-daemon.yaml b/test/k8s-local/cached-daemon.yaml new file mode 100644 index 0000000..e44f981 --- /dev/null +++ b/test/k8s-local/cached-daemon.yaml @@ -0,0 +1,95 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dateilager-csi-cached + labels: + app: dateilager-csi-cached +spec: + selector: + matchLabels: + app: dateilager-csi-cached + template: + metadata: + name: datailager-csi-cached + labels: + app: dateilager-csi-cached + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: csi-cached + image: dl-local:dev + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + runAsUser: 0 + command: + [ + "./cached", + "--log-level=info", + "--log-encoding=json", + "--csi-socket=unix:///csi/csi.sock", + "--upstream-host=host.internal", + "--upstream-port=5051", + "--tracing", + "--staging-path=/var/lib/kubelet/dateilager_cache", + ] + env: + - name: DL_ENV + value: "dev" + - name: "LOG_LEVEL" + value: "debug" + - name: DL_PGX_TRACING + value: "1" + - name: DL_TOKEN + value: "v2.public.eyJzdWIiOiJzaGFyZWQtcmVhZGVyIn1CxWdB02s9el0Wt7qReARZ-7JtIb4Zj3D4Oiji1yXHqj0orkpbcVlswVUiekECJC16d1NrHwD2FWSwRORZn8gK" + - name: DL_SKIP_SSL_VERIFICATION + value: "1" + - name: BUMP_TO_ROLL + value: "2" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi/ + - mountPath: "/var/lib/kubelet" + name: host-kubelet-dir + mountPropagation: "Bidirectional" + - name: csi-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 + args: + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/com.gadget.dateilager.cached/csi.sock" + - "--health-port=9809" + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + ports: + - containerPort: 9809 + name: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 + volumes: + # a working directory for the two containers to share when registering the CSI + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/com.gadget.dateilager.cached + type: DirectoryOrCreate + # the directory for the driver-registrar sidecar container to register our driver with the kubelet + # + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + # the host kubelet directory for the CSI driver to mount volumes in + - name: host-kubelet-dir + hostPath: + path: /var/lib/kubelet \ No newline at end of file diff --git a/test/shared_test.go b/test/shared_test.go index 317c753..9931e34 100644 --- a/test/shared_test.go +++ b/test/shared_test.go @@ -10,7 +10,6 @@ import ( "io/fs" "net" "os" - "path" "path/filepath" "sort" "strings" @@ -23,7 +22,6 @@ import ( "github.com/gadget-inc/dateilager/internal/pb" util "github.com/gadget-inc/dateilager/internal/testutil" "github.com/gadget-inc/dateilager/pkg/api" - "github.com/gadget-inc/dateilager/pkg/cached" "github.com/gadget-inc/dateilager/pkg/client" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/klauspost/compress/s2" @@ -415,29 +413,6 @@ func createTestGRPCServer(tc util.TestCtx) (*bufconn.Listener, *grpc.Server, fun return lis, s, getConn } -func createTestCachedServer(tc util.TestCtx, tmpDir string) (*api.Cached, string, func()) { - cl, _, closeClient := createTestClient(tc) - _, grpcServer, _ := createTestGRPCServer(tc) - - s := cached.CachedServer{ - Grpc: grpcServer, - } - - cached := tc.CachedApi(cl, path.Join(tmpDir, "cached", "staging")) - s.RegisterCached(cached) - s.RegisterCSI(cached) - - socket := path.Join(tmpDir, "csi.sock") - endpoint := "unix://" + socket - - go func() { - err := s.Serve(endpoint) - require.NoError(tc.T(), err, "CSI Server exited") - }() - - return cached, endpoint, func() { closeClient(); s.Grpc.Stop() } -} - func createTestClient(tc util.TestCtx) (*client.Client, *api.Fs, func()) { lis, s, getConn := createTestGRPCServer(tc)