From 0a2c7623041076d6bc5df425ea4e893cc17c3fe7 Mon Sep 17 00:00:00 2001 From: aoiasd <45024769+aoiasd@users.noreply.github.com> Date: Thu, 5 Dec 2024 19:02:41 +0800 Subject: [PATCH] enhance: report restful info to hook (#38112) Signed-off-by: aoiasd --- client/go.mod | 2 +- client/go.sum | 4 ++-- go.mod | 6 ++--- go.sum | 8 +++---- .../proxy/httpserver/handler_v2.go | 9 +++++++- .../distributed/proxy/httpserver/utils.go | 6 +++++ internal/distributed/proxy/service.go | 4 ++++ internal/proxy/authentication_interceptor.go | 23 +++++++++++++++++++ internal/util/hookutil/default.go | 4 ++++ pkg/go.mod | 2 +- pkg/go.sum | 4 ++-- 11 files changed, 58 insertions(+), 14 deletions(-) diff --git a/client/go.mod b/client/go.mod index cb64612b6d960..15f6268916414 100644 --- a/client/go.mod +++ b/client/go.mod @@ -6,7 +6,7 @@ require ( github.com/blang/semver/v4 v4.0.0 github.com/cockroachdb/errors v1.9.1 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19 + github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/samber/lo v1.27.0 diff --git a/client/go.sum b/client/go.sum index 29251f9844e71..a3ab79251b375 100644 --- a/client/go.sum +++ b/client/go.sum @@ -400,8 +400,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19 h1:p8YIqJENWFLOnmnuBETjpEPODfJDXIb9NtGJN5SWDQw= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f h1:GlKLZy+wKDvwc+J4CulfCY9x+OKKhPsjcDfpQ59sjKY= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw= github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= diff --git a/go.mod b/go.mod index 80e32051a755d..c53d12d3b4f5b 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,12 @@ require ( github.com/go-playground/validator/v10 v10.14.0 github.com/gofrs/flock v0.8.1 github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/protobuf v1.5.4 github.com/google/btree v1.1.2 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/klauspost/compress v1.17.9 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d - github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19 + github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f github.com/minio/minio-go/v7 v7.0.73 github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/prometheus/client_golang v1.14.0 @@ -252,7 +252,7 @@ replace ( github.com/apache/arrow/go/v12 => github.com/milvus-io/arrow/go/v12 v12.0.1 github.com/apache/pulsar-client-go => github.com/milvus-io/pulsar-client-go v0.6.10 github.com/bketelsen/crypt => github.com/bketelsen/crypt v0.0.4 // Fix security alert for core-os/etcd - github.com/expr-lang/expr => github.com/SimFG/expr v0.0.0-20241226082220-a9a764953bf8 + github.com/expr-lang/expr => github.com/SimFG/expr v0.0.0-20241029080923-21854792f532 github.com/go-kit/kit => github.com/go-kit/kit v0.1.0 // github.com/milvus-io/milvus-storage/go => ../milvus-storage/go github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 diff --git a/go.sum b/go.sum index 534bb1b438b3e..6e078d6034e80 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,8 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/SimFG/expr v0.0.0-20241226082220-a9a764953bf8 h1:boN3QhAWQU9O8EYQWxN7AEYav39PuD29QzZwTiI8Ca0= -github.com/SimFG/expr v0.0.0-20241226082220-a9a764953bf8/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= +github.com/SimFG/expr v0.0.0-20241029080923-21854792f532 h1:fxpBc+wl14Ky70AVgCJ4muWzAzXcxoerlRxA1F69S0M= +github.com/SimFG/expr v0.0.0-20241029080923-21854792f532/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/actgardner/gogen-avro/v10 v10.1.0/go.mod h1:o+ybmVjEa27AAr35FRqU98DJu1fXES56uXniYFv4yDA= github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= github.com/actgardner/gogen-avro/v9 v9.1.0/go.mod h1:nyTj6wPqDJoxM3qdnjcLv+EnMDSDFqE0qDpva2QRmKc= @@ -614,8 +614,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19 h1:p8YIqJENWFLOnmnuBETjpEPODfJDXIb9NtGJN5SWDQw= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f h1:GlKLZy+wKDvwc+J4CulfCY9x+OKKhPsjcDfpQ59sjKY= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70 h1:Z+sp64fmAOxAG7mU0dfVOXvAXlwRB0c8a96rIM5HevI= github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= diff --git a/internal/distributed/proxy/httpserver/handler_v2.go b/internal/distributed/proxy/httpserver/handler_v2.go index 4678d8666f22b..2e916a0034251 100644 --- a/internal/distributed/proxy/httpserver/handler_v2.go +++ b/internal/distributed/proxy/httpserver/handler_v2.go @@ -22,11 +22,13 @@ import ( "google.golang.org/protobuf/proto" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/hook" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proxy" "github.com/milvus-io/milvus/internal/types" + "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" @@ -281,6 +283,7 @@ func checkAuthorizationV2(ctx context.Context, c *gin.Context, ignoreErr bool, r if !ignoreErr { HTTPReturn(c, http.StatusUnauthorized, gin.H{HTTPReturnCode: merr.Code(merr.ErrNeedAuthenticate), HTTPReturnMessage: merr.ErrNeedAuthenticate.Error()}) } + hookutil.GetExtension().ReportRefused(ctx, req, WrapErrorToResponse(merr.ErrNeedAuthenticate), nil, c.FullPath()) return merr.ErrNeedAuthenticate } _, authErr := proxy.PrivilegeInterceptor(ctx, req) @@ -288,6 +291,7 @@ func checkAuthorizationV2(ctx context.Context, c *gin.Context, ignoreErr bool, r if !ignoreErr { HTTPReturn(c, http.StatusForbidden, gin.H{HTTPReturnCode: merr.Code(authErr), HTTPReturnMessage: authErr.Error()}) } + hookutil.GetExtension().ReportRefused(ctx, req, WrapErrorToResponse(authErr), nil, c.FullPath()) return authErr } @@ -313,6 +317,7 @@ func wrapperProxyWithLimit(ctx context.Context, c *gin.Context, req any, checkAu _, err := CheckLimiter(ctx, req, pxy) if err != nil { log.Warn("high level restful api, fail to check limiter", zap.Error(err), zap.String("method", fullMethod)) + hookutil.GetExtension().ReportRefused(ctx, req, WrapErrorToResponse(merr.ErrHTTPRateLimit), nil, c.FullPath()) HTTPAbortReturn(c, http.StatusOK, gin.H{ HTTPReturnCode: merr.Code(merr.ErrHTTPRateLimit), HTTPReturnMessage: merr.ErrHTTPRateLimit.Error() + ", error: " + err.Error(), @@ -325,13 +330,15 @@ func wrapperProxyWithLimit(ctx context.Context, c *gin.Context, req any, checkAu if !ok { username = "" } - response, err := proxy.HookInterceptor(ctx, req, username.(string), fullMethod, handler) + + response, err := proxy.HookInterceptor(context.WithValue(ctx, hook.GinParamsKey, c.Keys), req, username.(string), fullMethod, handler) if err == nil { status, ok := requestutil.GetStatusFromResponse(response) if ok { err = merr.Error(status) } } + if err != nil { log.Ctx(ctx).Warn("high level restful api, grpc call failed", zap.Error(err)) if !ignoreErr { diff --git a/internal/distributed/proxy/httpserver/utils.go b/internal/distributed/proxy/httpserver/utils.go index e6f6edc546658..15481953246eb 100644 --- a/internal/distributed/proxy/httpserver/utils.go +++ b/internal/distributed/proxy/httpserver/utils.go @@ -1507,3 +1507,9 @@ func generateExpressionTemplate(params map[string]interface{}) map[string]*schem } return expressionTemplate } + +func WrapErrorToResponse(err error) *milvuspb.BoolResponse { + return &milvuspb.BoolResponse{ + Status: merr.Status(err), + } +} diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index b82432c5a4f14..16118c7fdcea1 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -63,6 +63,7 @@ import ( "github.com/milvus-io/milvus/internal/util/componentutil" "github.com/milvus-io/milvus/internal/util/dependency" _ "github.com/milvus-io/milvus/internal/util/grpcclient" + "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/tracer" @@ -140,6 +141,9 @@ func authenticate(c *gin.Context) { } log.Warn("fail to verify apikey", zap.Error(err)) } + hookutil.GetExtension().ReportRefused(context.Background(), nil, &milvuspb.BoolResponse{ + Status: merr.Status(merr.ErrNeedAuthenticate), + }, nil, c.FullPath()) c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{httpserver.HTTPReturnCode: merr.Code(merr.ErrNeedAuthenticate), httpserver.HTTPReturnMessage: merr.ErrNeedAuthenticate.Error()}) } diff --git a/internal/proxy/authentication_interceptor.go b/internal/proxy/authentication_interceptor.go index f5369dce4d920..3f90212e2c4e3 100644 --- a/internal/proxy/authentication_interceptor.go +++ b/internal/proxy/authentication_interceptor.go @@ -5,11 +5,15 @@ import ( "fmt" "strings" + grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" "go.uber.org/zap" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" + "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util" @@ -42,6 +46,25 @@ func validSourceID(ctx context.Context, authorization []string) bool { return sourceID == util.MemberCredID } +func GrpcAuthInterceptor(authFunc grpc_auth.AuthFunc) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + var newCtx context.Context + var err error + if overrideSrv, ok := info.Server.(grpc_auth.ServiceAuthFuncOverride); ok { + newCtx, err = overrideSrv.AuthFuncOverride(ctx, info.FullMethod) + } else { + newCtx, err = authFunc(ctx) + } + if err != nil { + hookutil.GetExtension().ReportRefused(context.Background(), req, &milvuspb.BoolResponse{ + Status: merr.Status(err), + }, err, info.FullMethod) + return nil, err + } + return handler(newCtx, req) + } +} + // AuthenticationInterceptor verify based on kv pair <"authorization": "token"> in header func AuthenticationInterceptor(ctx context.Context) (context.Context, error) { // The keys within metadata.MD are normalized to lowercase. diff --git a/internal/util/hookutil/default.go b/internal/util/hookutil/default.go index 7bbd467bb6f6a..546ed80c4388c 100644 --- a/internal/util/hookutil/default.go +++ b/internal/util/hookutil/default.go @@ -59,3 +59,7 @@ var _ hook.Extension = (*DefaultExtension)(nil) func (d DefaultExtension) Report(info any) int { return 0 } + +func (d DefaultExtension) ReportRefused(ctx context.Context, req interface{}, resp interface{}, err error, fullMethod string) error { + return nil +} diff --git a/pkg/go.mod b/pkg/go.mod index 62c7f8d9baa69..efd14c8add224 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -12,7 +12,7 @@ require ( github.com/expr-lang/expr v1.15.7 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/klauspost/compress v1.17.7 - github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19 + github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f github.com/nats-io/nats-server/v2 v2.10.12 github.com/nats-io/nats.go v1.34.1 github.com/panjf2000/ants/v2 v2.7.2 diff --git a/pkg/go.sum b/pkg/go.sum index 48cb2fa88c9c3..3bfadd9a891f0 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -503,8 +503,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19 h1:p8YIqJENWFLOnmnuBETjpEPODfJDXIb9NtGJN5SWDQw= -github.com/milvus-io/milvus-proto/go-api/v2 v2.4.19/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f h1:GlKLZy+wKDvwc+J4CulfCY9x+OKKhPsjcDfpQ59sjKY= +github.com/milvus-io/milvus-proto/go-api/v2 v2.4.20-0.20241231032247-fe5d4dd2235f/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=