diff --git a/.golangci.yml b/.golangci.yml index bf89a5b..6d6cd97 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,4 +19,4 @@ linters: linters-settings: gocyclo: - min-complexity: 16 + min-complexity: 17 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fa6919..ae3fa72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # CHANGELOG +## 0.11.3 + +- Key changes: + - Fixed deduplication for negative non-regexp filters (previously, those would be let through without request modifications); + - Internal refactoring: + - Moved acl + lf logic to a new internal package `querymodifier`; + - Added more tests. + ## 0.11.2 - Key changes: diff --git a/Taskfile.yml b/Taskfile.yml index f42f588..787494f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,9 +2,9 @@ version: '3' tasks: default: - - task: test - - task: lint - task: tidy + - task: lint + - task: test test: cmds: diff --git a/cmd/lfgw/acl.go b/cmd/lfgw/acl.go deleted file mode 100644 index 869298a..0000000 --- a/cmd/lfgw/acl.go +++ /dev/null @@ -1,229 +0,0 @@ -package main - -import ( - "fmt" - "os" - "regexp" - "strings" - "unicode" - - "github.com/VictoriaMetrics/metricsql" - "gopkg.in/yaml.v3" -) - -// ACLMap stores a parsed YAML with role defitions -type ACLMap map[string]*ACL - -// ACL stores a role definition -type ACL struct { - Fullaccess bool - LabelFilter metricsql.LabelFilter - RawACL string -} - -// toSlice converts namespace rules to string slices. -func (a *ACL) toSlice(str string) ([]string, error) { - buffer := []string{} - // yaml values should come already with trimmed leading and trailing spaces - for _, s := range strings.Split(str, ",") { - // in case there are empty elements in between - s := strings.TrimSpace(s) - - // TODO: optionally disable it when things are loaded from a file? - for _, ch := range s { - if unicode.IsSpace(ch) { - return nil, fmt.Errorf("line should not contain spaces within individual elements (%q)", str) - } - } - - if s != "" { - buffer = append(buffer, s) - } - } - - if len(buffer) == 0 { - return nil, fmt.Errorf("line has to contain at least one valid element (%q)", str) - } - - return buffer, nil -} - -// TODO: update description once NormalizedACL field is introduced -// PrepareACL returns an ACL based on a rule definition (non-regexp for one namespace, regexp - for many). .RawACL in the resulting value will contain a normalized value (anchors stripped, implicit admin will have only .*). -func (a *ACL) PrepareACL(rawACL string) (ACL, error) { - var lf = metricsql.LabelFilter{ - Label: "namespace", - IsNegative: false, - IsRegexp: false, - } - - buffer, err := a.toSlice(rawACL) - if err != nil { - return ACL{}, err - } - - // If .* is in the slice, then we can omit any other value - for _, v := range buffer { - // TODO: move to a helper? - if v == ".*" { - // Note: with this approach, we intentionally omit other values in the resulting ACL - return a.getFullaccessACL(), nil - } - } - - if len(buffer) == 1 { - // TODO: move to a helper? - if strings.ContainsAny(buffer[0], `.+*?^$()[]{}|\`) { - lf.IsRegexp = true - // Trim anchors as they're not needed for Prometheus, and not expected in the app.shouldBeModified function - buffer[0] = strings.TrimLeft(buffer[0], "^") - buffer[0] = strings.TrimLeft(buffer[0], "(") - buffer[0] = strings.TrimRight(buffer[0], "$") - buffer[0] = strings.TrimRight(buffer[0], ")") - } - lf.Value = buffer[0] - } else { - // "Regex matches are fully anchored. A match of env=~"foo" is treated as env=~"^foo$"." https://prometheus.io/docs/prometheus/latest/querying/basics/ - lf.Value = strings.Join(buffer, "|") - lf.IsRegexp = true - } - - if lf.IsRegexp { - _, err := regexp.Compile(lf.Value) - if err != nil { - return ACL{}, fmt.Errorf("%s in %q (converted from %q)", err, lf.Value, rawACL) - } - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: lf, - // TODO: that should go to NormalizedACL - RawACL: strings.Join(buffer, ", "), - } - - return acl, nil -} - -// getFullaccessACL returns a fullaccess ACL -func (a *ACL) getFullaccessACL() ACL { - return ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - } -} - -// loadACL loads ACL from a file -func (app *application) loadACL() (ACLMap, error) { - aclMap := make(ACLMap) - - yamlFile, err := os.ReadFile(app.ACLPath) - if err != nil { - return aclMap, err - } - var aclYaml map[string]string - - err = yaml.Unmarshal(yamlFile, &aclYaml) - if err != nil { - return aclMap, err - } - - for role, rawACL := range aclYaml { - acl := ACL{} - - acl, err := acl.PrepareACL(rawACL) - if err != nil { - return ACLMap{}, err - } - - aclMap[role] = &acl - } - - return aclMap, nil -} - -// rolesToRawACL returns a comma-separated list of ACL definitions for all specified roles. Basically, it lets you dynamically generate a raw ACL as if it was supplied through acl.yaml -func (app *application) rolesToRawACL(roles []string) (string, error) { - rawACLs := make([]string, 0, len(roles)) - - for _, role := range roles { - acl, exists := app.ACLMap[role] - if exists { - // NOTE: You should never see an empty definitions in .RawACL as those should be removed by toSlice further down the process. The error check below is not necessary, is left as an additional safeguard for now and might get removed in the future. - if acl.RawACL == "" { - return "", fmt.Errorf("%s role contains empty rawACL", role) - } - rawACLs = append(rawACLs, acl.RawACL) - } else { - // NOTE: Strictly speaking, it's getACL who is expected to pass a filtered list of roles in case Assumed roles are disabled. The error check below is not necessary, is left as an additional safeguard for now and might get removed in the future. - if !app.AssumedRoles { - return "", fmt.Errorf("some of the roles are unknown and assumed roles are not enabled") - } - // NOTE: Role names are not linted, so they may contain regular expressions, including the admin definition: .* - rawACLs = append(rawACLs, role) - } - } - - rawACL := strings.Join(rawACLs, ", ") - if rawACL == "" { - return "", fmt.Errorf("constructed empty rawACL") - } - - return rawACL, nil -} - -// getACL takes a list of roles found in an OIDC claim and constructs and ACL based on them. If assumed roles are disabled, then only known roles (present in app.ACLMap) are considered. -func (app *application) getACL(oidcRoles []string) (ACL, error) { - roles := []string{} - assumedRoles := []string{} - - for _, role := range oidcRoles { - _, exists := app.ACLMap[role] - if exists { - if app.ACLMap[role].Fullaccess { - return *app.ACLMap[role], nil - } - roles = append(roles, role) - } else { - assumedRoles = append(assumedRoles, role) - } - } - - if app.AssumedRoles { - roles = append(roles, assumedRoles...) - } - - if len(roles) == 0 { - return ACL{}, fmt.Errorf("no matching roles found") - } - - // We can return a prebuilt ACL if there's only one role and it's known - if len(roles) == 1 { - role := roles[0] - acl, exists := app.ACLMap[role] - if exists { - return *acl, nil - } - } - - // To simplify creation of composite ACLs, we need to form a raw ACL, so the further process would be equal to what we have for processing acl.yaml - rawACL, err := app.rolesToRawACL(roles) - if err != nil { - return ACL{}, err - } - - acl := ACL{} - - acl, err = acl.PrepareACL(rawACL) - if err != nil { - return ACL{}, err - } - - return acl, nil -} diff --git a/cmd/lfgw/acl_test.go b/cmd/lfgw/acl_test.go deleted file mode 100644 index 592e0e9..0000000 --- a/cmd/lfgw/acl_test.go +++ /dev/null @@ -1,583 +0,0 @@ -package main - -import ( - "os" - "testing" - - "github.com/VictoriaMetrics/metricsql" - "github.com/stretchr/testify/assert" -) - -func TestACL_ToSlice(t *testing.T) { - acl := &ACL{} - - t.Run("a, b", func(t *testing.T) { - want := []string{"a", "b"} - got, err := acl.toSlice("a, b") - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("a, , b (contains empty values)", func(t *testing.T) { - want := []string{"a", "b"} - got, err := acl.toSlice("a, , b") - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("a", func(t *testing.T) { - want := []string{"a"} - got, err := acl.toSlice("a") - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("a b (contains spaces)", func(t *testing.T) { - _, err := acl.toSlice("a b") - assert.NotNil(t, err) - }) - - t.Run("(empty values)", func(t *testing.T) { - _, err := acl.toSlice("") - assert.NotNil(t, err) - }) -} - -func TestACL_PrepareACL(t *testing.T) { - acl := &ACL{false, metricsql.LabelFilter{}, ""} - - tests := []struct { - name string - rawACL string - want ACL - fail bool - }{ - { - name: ".* (full access)", - rawACL: ".*", - want: ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - }, - fail: false, - }, - { - name: "min.*, .*, stolon (implicit full access, same as .*)", - rawACL: "min.*, .*, stolon", - want: ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - }, - fail: false, - }, - { - name: "minio (only minio)", - rawACL: "minio", - want: ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "minio", - IsRegexp: false, - IsNegative: false, - }, - RawACL: "minio", - }, - fail: false, - }, - { - name: "min.* (one regexp)", - rawACL: "min.*", - want: ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*", - }, - fail: false, - }, - { - name: "min.* (one anchored regexp)", - rawACL: "^(min.*)$", - want: ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*", - }, - fail: false, - }, - { - name: "minio, stolon (two namespaces)", - rawACL: "minio, stolon", - want: ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "minio|stolon", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "minio, stolon", - }, - fail: false, - }, - { - name: "min.*, stolon (regexp and non-regexp)", - rawACL: "min.*, stolon", - want: ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|stolon", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, stolon", - }, - fail: false, - }, - // TODO: assign special meaning to this regexp? - { - name: ".+ (is a regexp)", - rawACL: ".+", - want: ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".+", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".+", - }, - fail: false, - }, - { - name: "a,b (is a correct regexp)", - rawACL: "a,b", - want: ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "a|b", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "a, b", - }, - fail: false, - }, - { - name: "[ (incorrect regexp)", - rawACL: "[", - want: ACL{}, - fail: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := acl.PrepareACL(tt.rawACL) - if tt.fail { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - assert.Equal(t, tt.want, got) - } - }) - } -} - -func TestACL_LoadACL(t *testing.T) { - tests := []struct { - name string - content string - want ACLMap - }{ - { - name: "admin", - content: "admin: .*", - want: ACLMap{ - "admin": &ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - }, - }, - }, - { - name: "implicit-admin", - content: `implicit-admin: ku.*, .*, min.*`, - want: ACLMap{ - "implicit-admin": &ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - }, - }, - }, - { - name: "multiple-values", - content: "multiple-values: ku.*, min.*", - want: ACLMap{ - "multiple-values": &ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "ku.*|min.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "ku.*, min.*", - }, - }, - }, - { - name: "single-value", - content: "single-value: default", - want: ACLMap{ - "single-value": &ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "default", - IsRegexp: false, - IsNegative: false, - }, - RawACL: "default", - }, - }, - }, - } - - f, err := os.CreateTemp("", "acl-*.yaml") - if err != nil { - t.Fatal(err) - } - defer os.Remove(f.Name()) - - app := &application{ - ACLPath: f.Name(), - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - saveACLToFile(t, f, tt.content) - got, err := app.loadACL() - assert.Nil(t, err) - assert.Equal(t, tt.want, got) - }) - } - - t.Run("incorrect ACL", func(t *testing.T) { - saveACLToFile(t, f, "test-role:") - _, err := app.loadACL() - assert.NotNil(t, err) - - saveACLToFile(t, f, "test-role: a b") - _, err = app.loadACL() - assert.NotNil(t, err) - }) - - if err := f.Close(); err != nil { - t.Fatal(err) - } -} - -// saveACLToFile writes given content to a file (existing data is deleted) -func saveACLToFile(t testing.TB, f *os.File, content string) { - t.Helper() - if err := f.Truncate(0); err != nil { - f.Close() - t.Fatal(err) - } - - if _, err := f.Seek(0, 0); err != nil { - f.Close() - t.Fatal(err) - } - - if _, err := f.Write([]byte(content)); err != nil { - f.Close() - t.Fatal(err) - } -} - -func TestApplication_rolesToRawACL(t *testing.T) { - app := &application{ - ACLMap: ACLMap{ - "admin": &ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - }, - "multiple-values": &ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "ku.*|min.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "ku.*, min.*", - }, - "single-value": &ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "default", - IsRegexp: false, - IsNegative: false, - }, - RawACL: "default", - }, - }, - } - - t.Run("0 roles", func(t *testing.T) { - roles := []string{} - _, err := app.rolesToRawACL(roles) - assert.NotNil(t, err) - }) - - t.Run("0 known roles", func(t *testing.T) { - roles := []string{"unknown-role"} - _, err := app.rolesToRawACL(roles) - assert.NotNil(t, err) - }) - - t.Run("1 known role", func(t *testing.T) { - roles := []string{"multiple-values"} - want := "ku.*, min.*" - - got, err := app.rolesToRawACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("multiple known roles", func(t *testing.T) { - roles := []string{"multiple-values", "single-value"} - want := "ku.*, min.*, default" - - got, err := app.rolesToRawACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("Empty rawACL", func(t *testing.T) { - app := &application{ - ACLMap: ACLMap{ - "empty-acl": &ACL{}, - }, - } - roles := []string{"empty-acl"} - - _, err := app.rolesToRawACL(roles) - assert.NotNil(t, err) - }) - - // Assumed roles enabled - app.AssumedRoles = true - - t.Run("0 known roles, 1 is unknown (assumed roles enabled)", func(t *testing.T) { - roles := []string{"unknown-role"} - want := "unknown-role" - - got, err := app.rolesToRawACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("multiple roles, 1 is unknown (assumed roles enabled)", func(t *testing.T) { - roles := []string{"multiple-values", "single-value", "unknown-role"} - want := "ku.*, min.*, default, unknown-role" - - got, err := app.rolesToRawACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - -} - -func TestApplication_GetACL(t *testing.T) { - app := &application{ - ACLMap: ACLMap{ - "admin": &ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - }, - "multiple-values": &ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "ku.*|min.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "ku.*, min.*", - }, - "single-value": &ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "default", - IsRegexp: false, - IsNegative: false, - }, - RawACL: "default", - }, - }, - } - - t.Run("0 roles", func(t *testing.T) { - roles := []string{} - _, err := app.getACL(roles) - assert.NotNil(t, err) - }) - - t.Run("0 known roles", func(t *testing.T) { - roles := []string{"unknown-role"} - _, err := app.getACL(roles) - assert.NotNil(t, err) - }) - - t.Run("1 role", func(t *testing.T) { - roles := []string{"single-value"} - want := *app.ACLMap["single-value"] - got, err := app.getACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("multiple roles, full access", func(t *testing.T) { - roles := []string{"admin", "multiple-values"} - want := *app.ACLMap["admin"] - got, err := app.getACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("multiple roles, 1 is unknown, no full access", func(t *testing.T) { - roles := []string{"single-value", "multiple-values", "unknown-role"} - knownRoles := []string{"single-value", "multiple-values"} - - rawACL, err := app.rolesToRawACL(knownRoles) - assert.Nil(t, err) - - want := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "default|ku.*|min.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: rawACL, - } - - got, err := app.getACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - // Assumed roles enabled - app.AssumedRoles = true - - t.Run("0 known roles, 1 is unknown (assumed roles enabled)", func(t *testing.T) { - roles := []string{"unknown-role"} - - want := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "unknown-role", - IsRegexp: false, - IsNegative: false, - }, - RawACL: "unknown-role", - } - - got, err := app.getACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("multiple roles, 1 is unknown (assumed roles enabled)", func(t *testing.T) { - roles := []string{"multiple-values", "single-value", "unknown-role"} - - want := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "ku.*|min.*|default|unknown-role", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "ku.*, min.*, default, unknown-role", - } - - got, err := app.getACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) - - t.Run("multiple roles, 1 is unknown, 1 gives full access (assumed roles enabled)", func(t *testing.T) { - roles := []string{"multiple-values", "admin", "unknown-role"} - - want := ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - } - - got, err := app.getACL(roles) - assert.Nil(t, err) - assert.Equal(t, want, got) - }) -} diff --git a/cmd/lfgw/helpers.go b/cmd/lfgw/helpers.go index dc48d98..b9f73c7 100644 --- a/cmd/lfgw/helpers.go +++ b/cmd/lfgw/helpers.go @@ -31,7 +31,7 @@ func (app *application) clientErrorMessage(w http.ResponseWriter, status int, er // getRawAccessToken returns a raw access token func (app *application) getRawAccessToken(r *http.Request) (string, error) { - headers := []string{"X-Forwarded-Access-Token", "X-Auth-Request-Access-Token", "Authorization"} + headers := []string{"Authorization", "X-Forwarded-Access-Token", "X-Auth-Request-Access-Token"} for _, h := range headers { t := r.Header.Get(h) diff --git a/cmd/lfgw/lf_test.go b/cmd/lfgw/lf_test.go deleted file mode 100644 index 2d521f2..0000000 --- a/cmd/lfgw/lf_test.go +++ /dev/null @@ -1,723 +0,0 @@ -package main - -import ( - "testing" - - "github.com/VictoriaMetrics/metricsql" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" -) - -func TestApplication_modifyMetricExpr(t *testing.T) { - logger := zerolog.New(nil) - - newACLPlain := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "default", - IsRegexp: false, - IsNegative: false, - }, - RawACL: "default", - } - - newACLPositiveRegexp := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|stolon", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, stolon", - } - - // Technically, it's not really possible to create such ACL, but better to keep an eye on it anyway - newACLNegativeRegexp := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|stolon", - IsRegexp: true, - IsNegative: true, - }, - RawACL: "min.*, stolon", - } - - tests := []struct { - name string - query string - EnableDeduplication bool - acl ACL - want string - }{ - { - name: "Complex example, Non-Regexp, no label; append", - query: `(histogram_quantile(0.9, rate (request_duration{job="demo"}[5m])) > 0.05 and rate(demo_api_request_duration_seconds_count{job="demo"}[5m]) > 1)`, - EnableDeduplication: false, - acl: newACLPlain, - want: `(histogram_quantile(0.9, rate(request_duration{job="demo", namespace="default"}[5m])) > 0.05) and (rate(demo_api_request_duration_seconds_count{job="demo", namespace="default"}[5m]) > 1)`, - }, - { - name: "Non-Regexp, no label; append", - query: `request_duration{job="demo"}`, - EnableDeduplication: false, - acl: newACLPlain, - want: `request_duration{job="demo", namespace="default"}`, - }, - { - name: "Non-Regexp, same label name; replace", - query: `request_duration{job="demo", namespace="other"}`, - EnableDeduplication: false, - acl: newACLPlain, - want: `request_duration{job="demo", namespace="default"}`, - }, - { - name: "Regexp, negative; append", - query: `request_duration{job="demo", namespace="other"}`, - EnableDeduplication: false, - acl: newACLNegativeRegexp, - want: `request_duration{job="demo", namespace="other", namespace!~"min.*|stolon"}`, - }, - { - name: "Regexp, negative; merge", - query: `request_duration{job="demo", namespace!~"other.*"}`, - EnableDeduplication: false, - acl: newACLNegativeRegexp, - want: `request_duration{job="demo", namespace!~"other.*|min.*|stolon"}`, - }, - { - name: "Regexp, positive; append", - query: `request_duration{job="demo", namespace="other"}`, - EnableDeduplication: false, - acl: newACLPositiveRegexp, - want: `request_duration{job="demo", namespace="other", namespace=~"min.*|stolon"}`, - }, - { - name: "Regexp, positive; replace", - query: `request_duration{job="demo", namespace=~"other.*"}`, - EnableDeduplication: false, - acl: newACLPositiveRegexp, - want: `request_duration{job="demo", namespace=~"min.*|stolon"}`, - }, - { - name: "Regexp, positive; append (not deduplicated)", - query: `request_duration{job="demo", namespace="default"}`, - EnableDeduplication: true, - acl: newACLPositiveRegexp, - want: `request_duration{job="demo", namespace="default", namespace=~"min.*|stolon"}`, - }, - // Examples from readme, deduplication is enabled - { - name: "Original filter is a non-regexp, matches policy (deduplicated)", - query: `request_duration{namespace="minio"}`, - EnableDeduplication: true, - acl: newACLPositiveRegexp, - want: `request_duration{namespace="minio"}`, - }, - { - name: "Original filter is a fake regexp (deduplicated)", - query: `request_duration{namespace=~"minio"}`, - EnableDeduplication: true, - acl: newACLPositiveRegexp, - want: `request_duration{namespace=~"minio"}`, - }, - { - name: "Original filter is a subfilter of the policy (deduplicated)", - query: `request_duration{namespace=~"min.*"}`, - EnableDeduplication: true, - acl: newACLPositiveRegexp, - want: `request_duration{namespace=~"min.*"}`, - }, - // Same examples, deduplication is disabled - { - name: "Original filter is a non-regexp, matches policy, but deduplication is disabled; append", - query: `request_duration{namespace="minio"}`, - EnableDeduplication: false, - acl: newACLPositiveRegexp, - want: `request_duration{namespace="minio", namespace=~"min.*|stolon"}`, - }, - { - name: "Original filter is a fake regexp, but deduplication is disabled; append", - query: `request_duration{namespace=~"minio"}`, - EnableDeduplication: false, - acl: newACLPositiveRegexp, - want: `request_duration{namespace=~"min.*|stolon"}`, - }, - { - name: "Original filter is a subfilter of the policy, but deduplication is disabled; replace", - query: `request_duration{namespace=~"min.*"}`, - EnableDeduplication: false, - acl: newACLPositiveRegexp, - want: `request_duration{namespace=~"min.*|stolon"}`, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - app := &application{ - logger: &logger, - EnableDeduplication: tt.EnableDeduplication, - } - - expr, err := metricsql.Parse(tt.query) - if err != nil { - t.Fatalf("%s", err) - } - originalExpr := metricsql.Clone(expr) - - newExpr := app.modifyMetricExpr(expr, tt.acl) - assert.Equal(t, originalExpr, expr, "The original expression got modified. Use metricsql.Clone() before modifying any expression.") - - got := string(newExpr.AppendString(nil)) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestApplication_isFakePositiveRegexp(t *testing.T) { - logger := zerolog.New(nil) - app := &application{ - logger: &logger, - } - - t.Run("Not a regexp", func(t *testing.T) { - filter := metricsql.LabelFilter{ - Label: "namespace", - Value: "minio", - IsRegexp: false, - IsNegative: false, - } - - want := false - got := app.isFakePositiveRegexp(filter) - assert.Equal(t, want, got) - }) - - t.Run("Fake positive regexp", func(t *testing.T) { - filter := metricsql.LabelFilter{ - Label: "namespace", - Value: "minio", - IsRegexp: true, - IsNegative: false, - } - - want := true - got := app.isFakePositiveRegexp(filter) - assert.Equal(t, want, got) - }) - - t.Run("Fake negative regexp", func(t *testing.T) { - filter := metricsql.LabelFilter{ - Label: "namespace", - Value: "minio", - IsRegexp: true, - IsNegative: true, - } - - want := false - got := app.isFakePositiveRegexp(filter) - assert.Equal(t, want, got) - }) - - t.Run("Real positive regexp", func(t *testing.T) { - filter := metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - } - - want := false - got := app.isFakePositiveRegexp(filter) - assert.Equal(t, want, got) - }) - - t.Run("Real negative regexp", func(t *testing.T) { - filter := metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: true, - } - - want := false - got := app.isFakePositiveRegexp(filter) - assert.Equal(t, want, got) - }) -} - -func TestApplication_shouldNotBeModified(t *testing.T) { - logger := zerolog.New(nil) - app := &application{ - logger: &logger, - } - - t.Run("Original filters do not contain target label", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "pod", - Value: "minio", - IsRegexp: false, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, control.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the original filters do not contain the target label") - }) - - t.Run("Original filter is a regexp and not a subfilter of the new ACL", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "mini.*", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, control.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the original filter is a regexp and not a subfilter of the new ACL") - }) - - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "minio", - IsRegexp: false, - IsNegative: false, - }, - } - - t.Run("Not a regexp", func(t *testing.T) { - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "default", - }, - RawACL: "default", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the new filter is not a matching positive regexp") - }) - - t.Run("Negative non-matching complex regexp", func(t *testing.T) { - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "kube.*|control.*", - IsRegexp: true, - IsNegative: true, - }, - RawACL: "kube.*, control.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the new filter is not a matching positive regexp") - }) - - t.Run("Negative non-matching simple regexp", func(t *testing.T) { - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "ini.*", - IsRegexp: true, - IsNegative: true, - }, - RawACL: "ini.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the new filter is not a matching positive regexp") - }) - - t.Run("Negative matching complex regexp", func(t *testing.T) { - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: true, - }, - RawACL: "min.*, control.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the new filter is not a matching positive regexp") - }) - - t.Run("Positive non-matching complex regexp", func(t *testing.T) { - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "kube.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "kube.*, control.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the new filter doesn't match original filter") - }) - - t.Run("Positive non-matching simple regexp", func(t *testing.T) { - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "ini.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "ini.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the new filter doesn't match original filter") - }) - - t.Run("Repeating regexp filters (not subfilters)", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - { - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "mini.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "mini.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the original filters contain regexp filters, which are not subfilters of the new filter") - }) - - t.Run("Multiple regexp filters, one of which is not a subfilter of the new filter", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - { - Label: "namespace", - Value: "contro.*", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, control.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because the original filters are regexps, one of which is not a subfilter of the new filter") - }) - - t.Run("Mix of a non-matching regexp and a non-regexp filters", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "minio", - IsRegexp: false, - IsNegative: false, - }, - { - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "mini.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "mini.*", - } - - want := false - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should be modified, because amongst the original filters with the same label (regexp, non-regexp) there is a regexp, which is not a subfilter of the new filter") - }) - - // Matching cases - - t.Run("Original filter is not a regexp, new filter matches", func(t *testing.T) { - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, control.*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter is not a regexp and the new filter is a matching positive regexp") - }) - - t.Run("Original filter is a fake positive regexp (no special symbols), new filter matches", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "minio", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, control.*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter is a fake positive regexp (it doesn't contain any special characters, should have been a non-regexp expression, e.g. namespace=~\"kube-system\") and the new filter is a matching positive regexp") - }) - - t.Run("Original filter is a regexp and a subfilter of the new ACL", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, control.*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter is a regexp subfilter of the ACL") - }) - - t.Run("Multiple regexp filters, new filter matches (subfilters)", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - { - Label: "namespace", - Value: "control.*", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*|control.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*, control.*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filters are subfilters of the new filter") - }) - - t.Run("Repeating filters, new filter matches", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "minio", - IsRegexp: false, - IsNegative: false, - }, - { - Label: "namespace", - Value: "minio", - IsRegexp: false, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "mini.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "mini.*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter contains the same non-regexp label filter multiple times and the new filter matches") - }) - - t.Run("Original filters are a mix of a fake regexp and a non-regexp filters and the new filter matches", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "minio", - IsRegexp: false, - IsNegative: false, - }, - { - Label: "namespace", - Value: "minio", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "mini.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "mini.*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because original filters contain a mix of a fake regexp and a non-regexp filters (basically, they're equal in results)") - }) - - t.Run("Original filter and the new filter contain the same regexp", func(t *testing.T) { - filters := []metricsql.LabelFilter{ - { - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - } - - acl := ACL{ - Fullaccess: false, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: "min.*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: "min.*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because original filter and the new filter contain the same regexp") - }) - - t.Run("The new filter gives full access", func(t *testing.T) { - acl := ACL{ - Fullaccess: true, - LabelFilter: metricsql.LabelFilter{ - Label: "namespace", - Value: ".*", - IsRegexp: true, - IsNegative: false, - }, - RawACL: ".*", - } - - want := true - got := app.shouldNotBeModified(filters, acl) - assert.Equal(t, want, got, "Original expression should NOT be modified, because the new filter gives full access") - }) -} diff --git a/cmd/lfgw/main.go b/cmd/lfgw/main.go index 28bf8f6..5d1c01f 100644 --- a/cmd/lfgw/main.go +++ b/cmd/lfgw/main.go @@ -13,6 +13,7 @@ import ( oidc "github.com/coreos/go-oidc/v3/oidc" "github.com/rs/zerolog" zlog "github.com/rs/zerolog/log" + "github.com/weisdd/lfgw/internal/querymodifier" "go.uber.org/automaxprocs/maxprocs" ) @@ -21,7 +22,7 @@ import ( type application struct { errorLog *log.Logger logger *zerolog.Logger - ACLMap ACLMap + ACLs querymodifier.ACLs proxy *httputil.ReverseProxy verifier *oidc.IDTokenVerifier Debug bool `env:"DEBUG" envDefault:"false"` @@ -35,7 +36,7 @@ type application struct { SetProxyHeaders bool `env:"SET_PROXY_HEADERS" envDefault:"false"` SetGomaxProcs bool `env:"SET_GOMAXPROCS" envDefault:"true"` ACLPath string `env:"ACL_PATH" envDefault:"./acl.yaml"` - AssumedRoles bool `env:"ASSUMED_ROLES" envDefault:"false"` + AssumedRolesEnabled bool `env:"ASSUMED_ROLES" envDefault:"false"` OIDCRealmURL string `env:"OIDC_REALM_URL,required"` OIDCClientID string `env:"OIDC_CLIENT_ID,required"` Port int `env:"PORT" envDefault:"8080"` @@ -89,7 +90,7 @@ func main() { app.logger.Info().Caller(). Msgf("Runtime settings: GOMAXPROCS = %d", runtime.GOMAXPROCS(0)) - if app.AssumedRoles { + if app.AssumedRolesEnabled { app.logger.Info().Caller(). Msg("Assumed roles mode is on") } else { @@ -98,18 +99,18 @@ func main() { } if app.ACLPath != "" { - app.ACLMap, err = app.loadACL() + app.ACLs, err = querymodifier.NewACLsFromFile(app.ACLPath) if err != nil { app.logger.Fatal().Caller(). Err(err).Msgf("Failed to load ACL") } - for role, acl := range app.ACLMap { + for role, acl := range app.ACLs { app.logger.Info().Caller(). Msgf("Loaded role definition for %s: %q (converted to %s)", role, acl.RawACL, acl.LabelFilter.AppendString(nil)) } } else { - if !app.AssumedRoles { + if !app.AssumedRolesEnabled { app.logger.Fatal().Caller(). Msgf("The app cannot run without at least one source of configuration (Non-empty ACL_PATH and/or ASSUMED_ROLES set to true)") } diff --git a/cmd/lfgw/middleware.go b/cmd/lfgw/middleware.go index 13b220d..d9bf467 100644 --- a/cmd/lfgw/middleware.go +++ b/cmd/lfgw/middleware.go @@ -10,6 +10,7 @@ import ( "github.com/VictoriaMetrics/metrics" "github.com/rs/zerolog/hlog" + "github.com/weisdd/lfgw/internal/querymodifier" ) // nonProxiedEndpointsMiddleware is a workaround to support healthz and metrics endpoints while forwarding everything else to an upstream. @@ -133,7 +134,7 @@ func (app *application) oidcModeMiddleware(next http.Handler) http.Handler { // NOTE: The field will contain all roles present in the token, not only those that are considered during ACL generation process app.enrichDebugLogContext(r, "roles", strings.Join(claims.Roles, ", ")) - acl, err := app.getACL(claims.Roles) + acl, err := app.ACLs.GetUserACL(claims.Roles, app.AssumedRolesEnabled) if err != nil { hlog.FromRequest(r).Error().Caller(). Err(err).Msg("") @@ -163,7 +164,7 @@ func (app *application) rewriteRequestMiddleware(next http.Handler) http.Handler return } - acl, ok := r.Context().Value(contextKeyACL).(ACL) + acl, ok := r.Context().Value(contextKeyACL).(querymodifier.ACL) if !ok { // Should never happen. It means OIDC middleware hasn't done it's job app.serverError(w, r, fmt.Errorf("ACL is not set in the context")) @@ -183,9 +184,14 @@ func (app *application) rewriteRequestMiddleware(next http.Handler) http.Handler return } + qm := querymodifier.QueryModifier{ + ACL: acl, + EnableDeduplication: app.EnableDeduplication, + OptimizeExpressions: app.OptimizeExpressions, + } + // Adjust GET params - getParams := r.URL.Query() - newGetParams, err := app.prepareQueryParams(&getParams, acl) + newGetParams, err := qm.GetModifiedEncodedURLValues(r.URL.Query()) if err != nil { hlog.FromRequest(r).Error().Caller(). Err(err).Msg("") @@ -196,7 +202,7 @@ func (app *application) rewriteRequestMiddleware(next http.Handler) http.Handler app.enrichDebugLogContext(r, "new_get_params", app.unescapedURLQuery(newGetParams)) // For PATCH, POST, and PUT requests - newPostParams, err := app.prepareQueryParams(&r.PostForm, acl) + newPostParams, err := qm.GetModifiedEncodedURLValues(r.PostForm) if err != nil { hlog.FromRequest(r).Error().Caller(). Err(err).Msg("") diff --git a/internal/querymodifier/acl.go b/internal/querymodifier/acl.go new file mode 100644 index 0000000..2cfa49a --- /dev/null +++ b/internal/querymodifier/acl.go @@ -0,0 +1,88 @@ +package querymodifier + +import ( + "fmt" + "regexp" + "strings" + + "github.com/VictoriaMetrics/metricsql" +) + +// RegexpSymbols is used to determine whether ACL definition is a regexp or whether LF contains a fake regexp +const RegexpSymbols = `.+*?^$()[]{}|\` + +// ACL stores a role definition +type ACL struct { + Fullaccess bool + LabelFilter metricsql.LabelFilter + RawACL string +} + +// NewACL returns an ACL based on a rule definition (non-regexp for one namespace, regexp - for many). .RawACL in the resulting value will contain a normalized value (anchors stripped, implicit admin will have only .*). +func NewACL(rawACL string) (ACL, error) { + var lf = metricsql.LabelFilter{ + Label: "namespace", + IsNegative: false, + IsRegexp: false, + } + + buffer, err := toSlice(rawACL) + if err != nil { + return ACL{}, err + } + + // If .* is in the slice, then we can omit any other value + for _, v := range buffer { + // TODO: move to a helper? + if v == ".*" { + // Note: with this approach, we intentionally omit other values in the resulting ACL + return getFullaccessACL(), nil + } + } + + if len(buffer) == 1 { + // TODO: move to a helper? + if strings.ContainsAny(buffer[0], RegexpSymbols) { + lf.IsRegexp = true + // Trim anchors as they're not needed for Prometheus, and not expected in the app.shouldBeModified function + buffer[0] = strings.TrimLeft(buffer[0], "^") + buffer[0] = strings.TrimLeft(buffer[0], "(") + buffer[0] = strings.TrimRight(buffer[0], "$") + buffer[0] = strings.TrimRight(buffer[0], ")") + } + lf.Value = buffer[0] + } else { + // "Regex matches are fully anchored. A match of env=~"foo" is treated as env=~"^foo$"." https://prometheus.io/docs/prometheus/latest/querying/basics/ + lf.Value = strings.Join(buffer, "|") + lf.IsRegexp = true + } + + if lf.IsRegexp { + _, err := regexp.Compile(lf.Value) + if err != nil { + return ACL{}, fmt.Errorf("%s in %q (converted from %q)", err, lf.Value, rawACL) + } + } + + acl := ACL{ + Fullaccess: false, + LabelFilter: lf, + RawACL: strings.Join(buffer, ", "), + } + + return acl, nil +} + +// getFullaccessACL returns a fullaccess ACL +func getFullaccessACL() ACL { + return ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + } +} diff --git a/internal/querymodifier/acl_test.go b/internal/querymodifier/acl_test.go new file mode 100644 index 0000000..19c129e --- /dev/null +++ b/internal/querymodifier/acl_test.go @@ -0,0 +1,171 @@ +package querymodifier + +import ( + "testing" + + "github.com/VictoriaMetrics/metricsql" + "github.com/stretchr/testify/assert" +) + +func Test_NewACL(t *testing.T) { + tests := []struct { + name string + rawACL string + want ACL + fail bool + }{ + { + name: ".* (full access)", + rawACL: ".*", + want: ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + }, + fail: false, + }, + { + name: "min.*, .*, stolon (implicit full access, same as .*)", + rawACL: "min.*, .*, stolon", + want: ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + }, + fail: false, + }, + { + name: "minio (only minio)", + rawACL: "minio", + want: ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: false, + }, + RawACL: "minio", + }, + fail: false, + }, + { + name: "min.* (one regexp)", + rawACL: "min.*", + want: ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "min.*", + }, + fail: false, + }, + { + name: "min.* (one anchored regexp)", + rawACL: "^(min.*)$", + want: ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "min.*", + }, + fail: false, + }, + { + name: "minio, stolon (two namespaces)", + rawACL: "minio, stolon", + want: ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "minio|stolon", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "minio, stolon", + }, + fail: false, + }, + { + name: "min.*, stolon (regexp and non-regexp)", + rawACL: "min.*, stolon", + want: ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "min.*|stolon", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "min.*, stolon", + }, + fail: false, + }, + // TODO: assign special meaning to this regexp? + { + name: ".+ (is a regexp)", + rawACL: ".+", + want: ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".+", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".+", + }, + fail: false, + }, + { + name: "a,b (is a correct regexp)", + rawACL: "a,b", + want: ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "a|b", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "a, b", + }, + fail: false, + }, + { + name: "[ (incorrect regexp)", + rawACL: "[", + want: ACL{}, + fail: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewACL(tt.rawACL) + if tt.fail { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} diff --git a/internal/querymodifier/acls.go b/internal/querymodifier/acls.go new file mode 100644 index 0000000..8c1b4de --- /dev/null +++ b/internal/querymodifier/acls.go @@ -0,0 +1,113 @@ +package querymodifier + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// ACLs stores a parsed YAML with role defitions +type ACLs map[string]ACL + +// rolesToRawACL returns a comma-separated list of ACL definitions for all specified roles. Basically, it lets you dynamically generate a raw ACL as if it was supplied through acl.yaml. To support Assumed Roles, unknown roles are treated as ACL definitions. +func (a ACLs) rolesToRawACL(roles []string) (string, error) { + rawACLs := make([]string, 0, len(roles)) + + for _, role := range roles { + acl, exists := a[role] + if exists { + // NOTE: You should never see an empty definitions in .RawACL as those should be removed by toSlice further down the process. The error check below is not necessary, is left as an additional safeguard for now and might get removed in the future. + if acl.RawACL == "" { + return "", fmt.Errorf("%s role contains empty rawACL", role) + } + rawACLs = append(rawACLs, acl.RawACL) + } else { + // NOTE: Role names are not linted, so they may contain regular expressions, including the admin definition: .* + rawACLs = append(rawACLs, role) + } + } + + rawACL := strings.Join(rawACLs, ", ") + if rawACL == "" { + return "", fmt.Errorf("constructed empty rawACL") + } + + return rawACL, nil +} + +// GetUserACL takes a list of roles found in an OIDC claim and constructs and ACL based on them. If assumed roles are disabled, then only known roles (present in app.ACLs) are considered. +func (a ACLs) GetUserACL(oidcRoles []string, assumedRolesEnabled bool) (ACL, error) { + roles := []string{} + assumedRoles := []string{} + + for _, role := range oidcRoles { + _, exists := a[role] + if exists { + if a[role].Fullaccess { + return a[role], nil + } + roles = append(roles, role) + } else { + assumedRoles = append(assumedRoles, role) + } + } + + if assumedRolesEnabled { + roles = append(roles, assumedRoles...) + } + + if len(roles) == 0 { + return ACL{}, fmt.Errorf("no matching roles found") + } + + // We can return a prebuilt ACL if there's only one role and it's known + if len(roles) == 1 { + role := roles[0] + acl, exists := a[role] + if exists { + return acl, nil + } + } + + // To simplify creation of composite ACLs, we need to form a raw ACL, so the further process would be equal to what we have for processing acl.yaml + rawACL, err := a.rolesToRawACL(roles) + if err != nil { + return ACL{}, err + } + + acl, err := NewACL(rawACL) + if err != nil { + return ACL{}, err + } + + return acl, nil +} + +// NewACLsFromFile loads ACL from a file +func NewACLsFromFile(path string) (ACLs, error) { + acls := make(ACLs) + + yamlFile, err := os.ReadFile(path) + if err != nil { + return ACLs{}, err + } + var aclYaml map[string]string + + err = yaml.Unmarshal(yamlFile, &aclYaml) + if err != nil { + return ACLs{}, err + } + + for role, rawACL := range aclYaml { + acl, err := NewACL(rawACL) + if err != nil { + return ACLs{}, err + } + + acls[role] = acl + } + + return acls, nil +} diff --git a/internal/querymodifier/acls_test.go b/internal/querymodifier/acls_test.go new file mode 100644 index 0000000..22e08af --- /dev/null +++ b/internal/querymodifier/acls_test.go @@ -0,0 +1,345 @@ +package querymodifier + +import ( + "os" + "testing" + + "github.com/VictoriaMetrics/metricsql" + "github.com/stretchr/testify/assert" +) + +func TestACL_rolesToRawACL(t *testing.T) { + a := ACLs{ + "admin": ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + }, + "multiple-values": ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "ku.*|min.*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "ku.*, min.*", + }, + "single-value": ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "default", + IsRegexp: false, + IsNegative: false, + }, + RawACL: "default", + }, + } + + t.Run("0 roles", func(t *testing.T) { + roles := []string{} + _, err := a.rolesToRawACL(roles) + assert.NotNil(t, err) + }) + + t.Run("1 known role", func(t *testing.T) { + roles := []string{"multiple-values"} + want := "ku.*, min.*" + + got, err := a.rolesToRawACL(roles) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("multiple known roles", func(t *testing.T) { + roles := []string{"multiple-values", "single-value"} + want := "ku.*, min.*, default" + + got, err := a.rolesToRawACL(roles) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("Empty rawACL", func(t *testing.T) { + a := ACLs{ + "empty-acl": ACL{}, + } + + roles := []string{"empty-acl"} + + _, err := a.rolesToRawACL(roles) + assert.NotNil(t, err) + }) +} + +func TestACL_GetUserACL(t *testing.T) { + a := ACLs{ + "admin": ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + }, + "multiple-values": ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "ku.*|min.*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "ku.*, min.*", + }, + "single-value": ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "default", + IsRegexp: false, + IsNegative: false, + }, + RawACL: "default", + }, + } + + // Assumed roles disabled + t.Run("0 roles", func(t *testing.T) { + roles := []string{} + _, err := a.GetUserACL(roles, false) + assert.NotNil(t, err) + }) + + t.Run("0 known roles", func(t *testing.T) { + roles := []string{"unknown-role"} + _, err := a.GetUserACL(roles, false) + assert.NotNil(t, err) + }) + + t.Run("1 role", func(t *testing.T) { + roles := []string{"single-value"} + want := a["single-value"] + got, err := a.GetUserACL(roles, false) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("multiple roles, full access", func(t *testing.T) { + roles := []string{"admin", "multiple-values"} + want := a["admin"] + got, err := a.GetUserACL(roles, false) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("multiple roles, 1 is unknown, no full access", func(t *testing.T) { + roles := []string{"single-value", "multiple-values", "unknown-role"} + knownRoles := []string{"single-value", "multiple-values"} + + rawACL, err := a.rolesToRawACL(knownRoles) + assert.Nil(t, err) + + want := ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "default|ku.*|min.*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: rawACL, + } + + got, err := a.GetUserACL(roles, false) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + // Assumed roles enabled + t.Run("0 known roles, 1 is unknown (assumed roles enabled)", func(t *testing.T) { + roles := []string{"unknown-role"} + + want := ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "unknown-role", + IsRegexp: false, + IsNegative: false, + }, + RawACL: "unknown-role", + } + + got, err := a.GetUserACL(roles, true) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("multiple roles, 1 is unknown (assumed roles enabled)", func(t *testing.T) { + roles := []string{"multiple-values", "single-value", "unknown-role"} + + want := ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "ku.*|min.*|default|unknown-role", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "ku.*, min.*, default, unknown-role", + } + + got, err := a.GetUserACL(roles, true) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("multiple roles, 1 is unknown, 1 gives full access (assumed roles enabled)", func(t *testing.T) { + roles := []string{"multiple-values", "admin", "unknown-role"} + + want := ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + } + + got, err := a.GetUserACL(roles, true) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) +} + +func TestACL_NewACLsFromFile(t *testing.T) { + tests := []struct { + name string + content string + want ACLs + }{ + { + name: "admin", + content: "admin: .*", + want: ACLs{ + "admin": ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + }, + }, + }, + { + name: "implicit-admin", + content: `implicit-admin: ku.*, .*, min.*`, + want: ACLs{ + "implicit-admin": ACL{ + Fullaccess: true, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: ".*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: ".*", + }, + }, + }, + { + name: "multiple-values", + content: "multiple-values: ku.*, min.*", + want: ACLs{ + "multiple-values": ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "ku.*|min.*", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "ku.*, min.*", + }, + }, + }, + { + name: "single-value", + content: "single-value: default", + want: ACLs{ + "single-value": ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "default", + IsRegexp: false, + IsNegative: false, + }, + RawACL: "default", + }, + }, + }, + } + + f, err := os.CreateTemp("", "acl-*.yaml") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + saveACLToFile(t, f, tt.content) + got, err := NewACLsFromFile(f.Name()) + assert.Nil(t, err) + assert.Equal(t, tt.want, got) + }) + } + + t.Run("incorrect ACL", func(t *testing.T) { + saveACLToFile(t, f, "test-role:") + _, err := NewACLsFromFile(f.Name()) + assert.NotNil(t, err) + + saveACLToFile(t, f, "test-role: a b") + _, err = NewACLsFromFile(f.Name()) + assert.NotNil(t, err) + }) + + if err := f.Close(); err != nil { + t.Fatal(err) + } +} + +// saveACLToFile writes given content to a file (existing data is deleted) +func saveACLToFile(t testing.TB, f *os.File, content string) { + t.Helper() + if err := f.Truncate(0); err != nil { + f.Close() + t.Fatal(err) + } + + if _, err := f.Seek(0, 0); err != nil { + f.Close() + t.Fatal(err) + } + + if _, err := f.Write([]byte(content)); err != nil { + f.Close() + t.Fatal(err) + } +} diff --git a/internal/querymodifier/helpers.go b/internal/querymodifier/helpers.go new file mode 100644 index 0000000..481142a --- /dev/null +++ b/internal/querymodifier/helpers.go @@ -0,0 +1,33 @@ +package querymodifier + +import ( + "fmt" + "strings" + "unicode" +) + +// toSlice converts namespace rules to string slices. +func toSlice(str string) ([]string, error) { + buffer := []string{} + // yaml values should come already with trimmed leading and trailing spaces + for _, s := range strings.Split(str, ",") { + // in case there are empty elements in between + s := strings.TrimSpace(s) + + for _, ch := range s { + if unicode.IsSpace(ch) { + return nil, fmt.Errorf("line should not contain spaces within individual elements (%q)", str) + } + } + + if s != "" { + buffer = append(buffer, s) + } + } + + if len(buffer) == 0 { + return nil, fmt.Errorf("line has to contain at least one valid element (%q)", str) + } + + return buffer, nil +} diff --git a/internal/querymodifier/helpers_test.go b/internal/querymodifier/helpers_test.go new file mode 100644 index 0000000..9816135 --- /dev/null +++ b/internal/querymodifier/helpers_test.go @@ -0,0 +1,40 @@ +package querymodifier + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestACL_ToSlice(t *testing.T) { + t.Run("a, b", func(t *testing.T) { + want := []string{"a", "b"} + got, err := toSlice("a, b") + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("a, , b (contains empty values)", func(t *testing.T) { + want := []string{"a", "b"} + got, err := toSlice("a, , b") + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("a", func(t *testing.T) { + want := []string{"a"} + got, err := toSlice("a") + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("a b (contains spaces)", func(t *testing.T) { + _, err := toSlice("a b") + assert.NotNil(t, err) + }) + + t.Run("(empty values)", func(t *testing.T) { + _, err := toSlice("") + assert.NotNil(t, err) + }) +} diff --git a/cmd/lfgw/lf.go b/internal/querymodifier/qm.go similarity index 64% rename from cmd/lfgw/lf.go rename to internal/querymodifier/qm.go index 2b498d5..64e52eb 100644 --- a/cmd/lfgw/lf.go +++ b/internal/querymodifier/qm.go @@ -1,4 +1,4 @@ -package main +package querymodifier import ( "fmt" @@ -8,36 +8,78 @@ import ( "github.com/VictoriaMetrics/metricsql" ) -// replaceLFByName drops all label filters with the matching name and then appends the supplied filter. -func (app *application) replaceLFByName(filters []metricsql.LabelFilter, newFilter metricsql.LabelFilter) []metricsql.LabelFilter { - newFilters := make([]metricsql.LabelFilter, 0, cap(filters)+1) +// QueryModifier is used for modifying PromQL / MetricsQL requests. The exact changes are determined by an ACL and further tuned by deduplication and expression optimizations. +type QueryModifier struct { + ACL ACL + EnableDeduplication bool + OptimizeExpressions bool +} - // Drop all label filters with the matching name - for _, filter := range filters { - if filter.Label != newFilter.Label { - newFilters = append(newFilters, filter) +// GetModifiedEncodedURLValues rewrites GET/POST "query" and "match" parameters to filter out metrics. +func (qm *QueryModifier) GetModifiedEncodedURLValues(params url.Values) (string, error) { + newParams := url.Values{} + + if qm.ACL.RawACL == "" || string(qm.ACL.LabelFilter.AppendString(nil)) == "" { + return "", fmt.Errorf("ACL cannot be empty") + } + + for k, vv := range params { + switch k { + case "query", "match[]": + for _, v := range vv { + { + expr, err := metricsql.Parse(v) + if err != nil { + return "", err + } + + expr = qm.modifyMetricExpr(expr) + if qm.OptimizeExpressions { + expr = metricsql.Optimize(expr) + } + + newVal := string(expr.AppendString(nil)) + newParams.Add(k, newVal) + } + } + default: + for _, v := range vv { + newParams.Add(k, v) + } } } - newFilters = append(newFilters, newFilter) - return newFilters + return newParams.Encode(), nil } -// isFakePositiveRegexp returns true if the given filter is a positive regexp that doesn't contain special symbols, e.g. namespace=~"kube-system" -func (app *application) isFakePositiveRegexp(filter metricsql.LabelFilter) bool { - if filter.IsRegexp && !filter.IsNegative { - if !strings.ContainsAny(filter.Value, `.+*?^$()[]{}|\`) { - return true +// modifyMetricExpr walks through the query and modifies only metricsql.Expr based on the supplied acl with label filter. +func (qm *QueryModifier) modifyMetricExpr(expr metricsql.Expr) metricsql.Expr { + newExpr := metricsql.Clone(expr) + + // We cannot pass any extra parameters, so we need to use a closure + // to say which label filter to add + modifyLabelFilter := func(expr metricsql.Expr) { + if me, ok := expr.(*metricsql.MetricExpr); ok { + if qm.ACL.LabelFilter.IsRegexp { + if !qm.EnableDeduplication || !qm.shouldNotBeModified(me.LabelFilters) { + me.LabelFilters = appendOrMergeRegexpLF(me.LabelFilters, qm.ACL.LabelFilter) + } + } else { + me.LabelFilters = replaceLFByName(me.LabelFilters, qm.ACL.LabelFilter) + } } } - return false + // Update label filters + metricsql.VisitAll(newExpr, modifyLabelFilter) + + return newExpr } // TODO: simplify description // shouldNotBeModified helps to understand whether the original label filters have to be modified. The function returns false if any of the original filters do not match expectations described further. It returns true if [the list of original filters contains either a fake positive regexp (no special symbols, e.g. namespace=~"kube-system") or a non-regexp filter] and [acl.LabelFilter is a matching positive regexp]. Also, if original filter is a subfilter of the new filter or has the same value; if acl gives full access. Target label is taken from the acl.LabelFilter. -func (app *application) shouldNotBeModified(filters []metricsql.LabelFilter, acl ACL) bool { - if acl.Fullaccess { +func (qm *QueryModifier) shouldNotBeModified(filters []metricsql.LabelFilter) bool { + if qm.ACL.Fullaccess { return true } @@ -45,16 +87,16 @@ func (app *application) shouldNotBeModified(filters []metricsql.LabelFilter, acl seenUnmodified := 0 // TODO: move to a map? Might not be worth doing as filters of the same type are unlikely - // TODO: move to NormalizedACL once it's introduced? - rawSubACLs := strings.Split(acl.RawACL, ", ") - newLF := acl.LabelFilter + rawSubACLs := strings.Split(qm.ACL.RawACL, ", ") + newLF := qm.ACL.LabelFilter for _, filter := range filters { - if filter.Label == newLF.Label && newLF.IsRegexp && !newLF.IsNegative { + // For filter, only positive regexps and non-regexps considered, for newLF - positive regexps. + if filter.Label == newLF.Label && !filter.IsNegative && newLF.IsRegexp && !newLF.IsNegative { seen++ // Target: non-regexps or fake regexps - if !filter.IsRegexp || app.isFakePositiveRegexp(filter) { + if !filter.IsRegexp || isFakePositiveRegexp(filter) { // Prometheus treats all regexp queries as anchored, whereas our raw regexp doesn't have them. So, we should take anchored values. re, err := metricsql.CompileRegexpAnchored(newLF.Value) // There shouldn't be any errors, though, just in case, better to skip deduplication @@ -65,7 +107,7 @@ func (app *application) shouldNotBeModified(filters []metricsql.LabelFilter, acl } // Target: both are positive regexps, filter is a subfilter of the newLF or has the same value - if filter.IsRegexp && !filter.IsNegative { + if filter.IsRegexp { for _, rawSubACL := range rawSubACLs { if filter.Value == rawSubACL { seenUnmodified++ @@ -79,8 +121,14 @@ func (app *application) shouldNotBeModified(filters []metricsql.LabelFilter, acl return seen > 0 && seen == seenUnmodified } +// NewQueryModifier returns a QueryModifier containing an ACL built from rawACL. +func NewQueryModifier(rawACL string) (QueryModifier, error) { + acl, err := NewACL(rawACL) + return QueryModifier{ACL: acl}, err +} + // appendOrMergeRegexpLF appends label filter or merges its value in case it's a regexp with the same label name and of the same type (negative / positive). -func (app *application) appendOrMergeRegexpLF(filters []metricsql.LabelFilter, newFilter metricsql.LabelFilter) []metricsql.LabelFilter { +func appendOrMergeRegexpLF(filters []metricsql.LabelFilter, newFilter metricsql.LabelFilter) []metricsql.LabelFilter { newFilters := make([]metricsql.LabelFilter, 0, cap(filters)+1) // In case we merge original filter value with newFilter, we'd like to skip adding newFilter to the resulting set. @@ -90,7 +138,7 @@ func (app *application) appendOrMergeRegexpLF(filters []metricsql.LabelFilter, n // Inspect label filters with the target name if filter.Label == newFilter.Label { // Inspect regexp filters of the same type (negative, positive) - if filter.IsRegexp && filter.IsNegative == newFilter.IsNegative { + if filter.IsRegexp && newFilter.IsRegexp && filter.IsNegative == newFilter.IsNegative { skipAddingNewFilter = true // Merge only negative regexps, because merge for positive regexp will expose data if filter.Value != "" && filter.IsNegative { @@ -109,59 +157,22 @@ func (app *application) appendOrMergeRegexpLF(filters []metricsql.LabelFilter, n return newFilters } -// modifyMetricExpr walks through the query and modifies only metricsql.Expr based on the supplied acl with label filter. -func (app *application) modifyMetricExpr(expr metricsql.Expr, acl ACL) metricsql.Expr { - newExpr := metricsql.Clone(expr) +// replaceLFByName drops all label filters with the matching name and then appends the supplied filter. +func replaceLFByName(filters []metricsql.LabelFilter, newFilter metricsql.LabelFilter) []metricsql.LabelFilter { + newFilters := make([]metricsql.LabelFilter, 0, cap(filters)+1) - // We cannot pass any extra parameters, so we need to use a closure - // to say which label filter to add - modifyLabelFilter := func(expr metricsql.Expr) { - if me, ok := expr.(*metricsql.MetricExpr); ok { - if acl.LabelFilter.IsRegexp { - if !app.EnableDeduplication || !app.shouldNotBeModified(me.LabelFilters, acl) { - me.LabelFilters = app.appendOrMergeRegexpLF(me.LabelFilters, acl.LabelFilter) - } - } else { - me.LabelFilters = app.replaceLFByName(me.LabelFilters, acl.LabelFilter) - } + // Drop all label filters with the matching name + for _, filter := range filters { + if filter.Label != newFilter.Label { + newFilters = append(newFilters, filter) } } - // Update label filters - metricsql.VisitAll(newExpr, modifyLabelFilter) - - return newExpr + newFilters = append(newFilters, newFilter) + return newFilters } -// prepareQueryParams rewrites GET/POST "query" and "match" parameters to filter out metrics. -func (app *application) prepareQueryParams(params *url.Values, acl ACL) (string, error) { - newParams := &url.Values{} - - for k, vv := range *params { - switch k { - case "query", "match[]": - for _, v := range vv { - { - expr, err := metricsql.Parse(v) - if err != nil { - return "", err - } - - expr = app.modifyMetricExpr(expr, acl) - if app.OptimizeExpressions { - expr = metricsql.Optimize(expr) - } - - newVal := string(expr.AppendString(nil)) - newParams.Add(k, newVal) - } - } - default: - for _, v := range vv { - newParams.Add(k, v) - } - } - } - - return newParams.Encode(), nil +// isFakePositiveRegexp returns true if the given filter is a positive regexp that doesn't contain special symbols, e.g. namespace=~"kube-system" +func isFakePositiveRegexp(filter metricsql.LabelFilter) bool { + return filter.IsRegexp && !filter.IsNegative && !strings.ContainsAny(filter.Value, RegexpSymbols) } diff --git a/internal/querymodifier/qm_test.go b/internal/querymodifier/qm_test.go new file mode 100644 index 0000000..1fd5fff --- /dev/null +++ b/internal/querymodifier/qm_test.go @@ -0,0 +1,1035 @@ +package querymodifier + +import ( + "net/url" + "testing" + + "github.com/VictoriaMetrics/metricsql" + "github.com/stretchr/testify/assert" +) + +func TestQueryModifier_GetModifiedEncodedURLValues(t *testing.T) { + // TODO: test that the original URL.Values haven't changed + + t.Run("Empty ACL", func(t *testing.T) { + params := url.Values{ + "random": []string{"randomvalue"}, + } + + qm := QueryModifier{ + ACL: ACL{}, + EnableDeduplication: false, + OptimizeExpressions: false, + } + + _, err := qm.GetModifiedEncodedURLValues(params) + assert.NotNil(t, err) + }) + + t.Run("No matching parameters", func(t *testing.T) { + params := url.Values{ + "random": []string{"randomvalue"}, + } + + acl, err := NewACL("minio") + if err != nil { + t.Fatal(err) + } + + qm := QueryModifier{ + ACL: acl, + EnableDeduplication: false, + OptimizeExpressions: false, + } + + want := params.Encode() + got, err := qm.GetModifiedEncodedURLValues(params) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("query and match[]", func(t *testing.T) { + query := `request_duration{job="demo", namespace="other"}` + + params := url.Values{ + "query": []string{query}, + "match[]": []string{query}, + } + + newQuery := `request_duration{job="demo", namespace="minio"}` + newParams := url.Values{ + "query": []string{newQuery}, + "match[]": []string{newQuery}, + } + + acl, err := NewACL("minio") + if err != nil { + t.Fatal(err) + } + + qm := QueryModifier{ + ACL: acl, + EnableDeduplication: false, + OptimizeExpressions: false, + } + want := newParams.Encode() + got, err := qm.GetModifiedEncodedURLValues(params) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("Deduplicate", func(t *testing.T) { + query := `request_duration{job="demo", namespace=~"minio"}` + + params := url.Values{ + "query": []string{query}, + "match[]": []string{query}, + } + + newQueryDeduplicated := `request_duration{job="demo", namespace=~"minio"}` + newParamsDeduplicated := url.Values{ + "query": []string{newQueryDeduplicated}, + "match[]": []string{newQueryDeduplicated}, + } + + newQueryNotDeduplicated := `request_duration{job="demo", namespace=~"mini.*"}` + newParamsNotDeduplicated := url.Values{ + "query": []string{newQueryNotDeduplicated}, + "match[]": []string{newQueryNotDeduplicated}, + } + + acl, err := NewACL("mini.*") + if err != nil { + t.Fatal(err) + } + + qm := QueryModifier{ + ACL: acl, + EnableDeduplication: false, + OptimizeExpressions: false, + } + + qm.EnableDeduplication = true + want := newParamsDeduplicated.Encode() + got, err := qm.GetModifiedEncodedURLValues(params) + assert.Nil(t, err) + assert.Equal(t, want, got) + + qm.EnableDeduplication = false + want = newParamsNotDeduplicated.Encode() + got, err = qm.GetModifiedEncodedURLValues(params) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) + + t.Run("Optimize", func(t *testing.T) { + // Example is taken from https://github.com/VictoriaMetrics/metricsql/blob/50340b1c7e599295deafc510f5cb833de0669c20/optimizer_test.go#L149 + query := `foo AND bar{baz="aa"}` + + params := url.Values{ + "query": []string{query}, + "match[]": []string{query}, + } + + newQueryOptimized := `foo{baz="aa", namespace="minio"} and bar{baz="aa", namespace="minio"}` + newParamsOptimized := url.Values{ + "query": []string{newQueryOptimized}, + "match[]": []string{newQueryOptimized}, + } + + newQueryNotOptimized := `foo{namespace="minio"} and bar{baz="aa", namespace="minio"}` + newParamsNotOptimized := url.Values{ + "query": []string{newQueryNotOptimized}, + "match[]": []string{newQueryNotOptimized}, + } + + acl, err := NewACL("minio") + if err != nil { + t.Fatal(err) + } + + qm := QueryModifier{ + ACL: acl, + EnableDeduplication: false, + OptimizeExpressions: false, + } + + qm.OptimizeExpressions = true + want := newParamsOptimized.Encode() + got, err := qm.GetModifiedEncodedURLValues(params) + assert.Nil(t, err) + assert.Equal(t, want, got) + + qm.OptimizeExpressions = false + want = newParamsNotOptimized.Encode() + got, err = qm.GetModifiedEncodedURLValues(params) + assert.Nil(t, err) + assert.Equal(t, want, got) + }) +} + +func TestQueryModifier_modifyMetricExpr(t *testing.T) { + newACLPlain := ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "default", + IsRegexp: false, + IsNegative: false, + }, + RawACL: "default", + } + + newACLPositiveRegexp := ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "min.*|stolon", + IsRegexp: true, + IsNegative: false, + }, + RawACL: "min.*, stolon", + } + + // Technically, it's not really possible to create such ACL, but better to keep an eye on it anyway + newACLNegativeRegexp := ACL{ + Fullaccess: false, + LabelFilter: metricsql.LabelFilter{ + Label: "namespace", + Value: "min.*|stolon", + IsRegexp: true, + IsNegative: true, + }, + RawACL: "min.*, stolon", + } + + tests := []struct { + name string + query string + EnableDeduplication bool + acl ACL + want string + }{ + { + name: "Complex example, Non-Regexp, no label; append", + query: `(histogram_quantile(0.9, rate (request_duration{job="demo"}[5m])) > 0.05 and rate(demo_api_request_duration_seconds_count{job="demo"}[5m]) > 1)`, + EnableDeduplication: false, + acl: newACLPlain, + want: `(histogram_quantile(0.9, rate(request_duration{job="demo", namespace="default"}[5m])) > 0.05) and (rate(demo_api_request_duration_seconds_count{job="demo", namespace="default"}[5m]) > 1)`, + }, + { + name: "Non-Regexp, no label; append", + query: `request_duration{job="demo"}`, + EnableDeduplication: false, + acl: newACLPlain, + want: `request_duration{job="demo", namespace="default"}`, + }, + { + name: "Non-Regexp, same label name; replace", + query: `request_duration{job="demo", namespace="other"}`, + EnableDeduplication: false, + acl: newACLPlain, + want: `request_duration{job="demo", namespace="default"}`, + }, + { + name: "Regexp, negative; append", + query: `request_duration{job="demo", namespace="other"}`, + EnableDeduplication: false, + acl: newACLNegativeRegexp, + want: `request_duration{job="demo", namespace="other", namespace!~"min.*|stolon"}`, + }, + { + name: "Regexp, negative; merge", + query: `request_duration{job="demo", namespace!~"other.*"}`, + EnableDeduplication: false, + acl: newACLNegativeRegexp, + want: `request_duration{job="demo", namespace!~"other.*|min.*|stolon"}`, + }, + { + name: "Regexp, positive; append", + query: `request_duration{job="demo", namespace="other"}`, + EnableDeduplication: false, + acl: newACLPositiveRegexp, + want: `request_duration{job="demo", namespace="other", namespace=~"min.*|stolon"}`, + }, + { + name: "Regexp, positive; replace", + query: `request_duration{job="demo", namespace=~"other.*"}`, + EnableDeduplication: false, + acl: newACLPositiveRegexp, + want: `request_duration{job="demo", namespace=~"min.*|stolon"}`, + }, + { + name: "Regexp, positive; append (not deduplicated)", + query: `request_duration{job="demo", namespace="default"}`, + EnableDeduplication: true, + acl: newACLPositiveRegexp, + want: `request_duration{job="demo", namespace="default", namespace=~"min.*|stolon"}`, + }, + // Examples from readme, deduplication is enabled + { + name: "Original filter is a non-regexp, matches policy (deduplicated)", + query: `request_duration{namespace="minio"}`, + EnableDeduplication: true, + acl: newACLPositiveRegexp, + want: `request_duration{namespace="minio"}`, + }, + { + name: "Original filter is a fake regexp (deduplicated)", + query: `request_duration{namespace=~"minio"}`, + EnableDeduplication: true, + acl: newACLPositiveRegexp, + want: `request_duration{namespace=~"minio"}`, + }, + { + name: "Original filter is a subfilter of the policy (deduplicated)", + query: `request_duration{namespace=~"min.*"}`, + EnableDeduplication: true, + acl: newACLPositiveRegexp, + want: `request_duration{namespace=~"min.*"}`, + }, + // Same examples, deduplication is disabled + { + name: "Original filter is a non-regexp, matches policy, but deduplication is disabled; append", + query: `request_duration{namespace="minio"}`, + EnableDeduplication: false, + acl: newACLPositiveRegexp, + want: `request_duration{namespace="minio", namespace=~"min.*|stolon"}`, + }, + { + name: "Original filter is a fake regexp, but deduplication is disabled; append", + query: `request_duration{namespace=~"minio"}`, + EnableDeduplication: false, + acl: newACLPositiveRegexp, + want: `request_duration{namespace=~"min.*|stolon"}`, + }, + { + name: "Original filter is a subfilter of the policy, but deduplication is disabled; replace", + query: `request_duration{namespace=~"min.*"}`, + EnableDeduplication: false, + acl: newACLPositiveRegexp, + want: `request_duration{namespace=~"min.*|stolon"}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qm := QueryModifier{ + ACL: tt.acl, + EnableDeduplication: tt.EnableDeduplication, + OptimizeExpressions: true, + } + + expr, err := metricsql.Parse(tt.query) + if err != nil { + t.Fatalf("%s", err) + } + originalExpr := metricsql.Clone(expr) + + newExpr := qm.modifyMetricExpr(expr) + assert.Equal(t, originalExpr, expr, "The original expression got modified. Use metricsql.Clone() before modifying any expression.") + + got := string(newExpr.AppendString(nil)) + assert.Equal(t, tt.want, got) + }) + } +} + +//gocyclo:ignore +func TestQueryModifier_shouldNotBeModified(t *testing.T) { + filtersNoTargetLabel := []metricsql.LabelFilter{ + { + Label: "pod", + Value: "minio", + IsRegexp: false, + IsNegative: false, + }, + } + + filtersNonRegexp := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: false, + }, + } + + filtersNegativeNonRegexp := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: true, + }, + } + + tests := []struct { + name string + comment string + rawACL string + isNegativeACL bool // turns a regexp LF into a negative regexp LF + filters []metricsql.LabelFilter + want bool + }{ + // Positive filtersNonRegexp + { + name: "Not a regexp", + comment: "Original expression should be modified, because the new filter is not a matching positive regexp", + rawACL: "default", + isNegativeACL: false, + filters: filtersNonRegexp, + want: false, + }, + { + name: "Positive non-matching complex regexp", + comment: "Original expression should be modified, because the new filter doesn't match original filter", + rawACL: "kube.*, control.*", + isNegativeACL: false, + filters: filtersNonRegexp, + want: false, + }, + { + name: "Positive non-matching simple regexp", + comment: "Original expression should be modified, because the new filter doesn't match original filter", + rawACL: "ini.*", + isNegativeACL: false, + filters: filtersNonRegexp, + want: false, + }, + // Negative filtersNonRegexp + { + name: "Negative non-matching complex regexp", + comment: "Original expression should be modified, because the new filter is not a matching positive regexp", + rawACL: "kube.*, control.*", + isNegativeACL: true, + filters: filtersNonRegexp, + want: false, + }, + { + name: "Negative non-matching simple regexp", + comment: "Original expression should be modified, because the new filter is not a matching positive regexp", + rawACL: "ini.*", + isNegativeACL: true, + filters: filtersNonRegexp, + want: false, + }, + { + name: "Negative matching complex regexp", + comment: "Original expression should be modified, because the new filter is not a matching positive regexp", + rawACL: "min.*, control.*", + isNegativeACL: true, + filters: filtersNonRegexp, + want: false, + }, + // Mixed cases + { + name: "No filters", + comment: "Original expression should be modified, because the list with original filters is empty", + rawACL: "min.*, control.*", + isNegativeACL: false, + filters: []metricsql.LabelFilter{}, + want: false, + }, + { + name: "Original filters do not contain target label", + comment: "Original expression should be modified, because the original filters do not contain the target label", + rawACL: "min.*, control.*", + isNegativeACL: false, + filters: filtersNoTargetLabel, + want: false, + }, + { + name: "Original filter is a negative non-regexp", + comment: "Original expression should be modified, because it is a negative non-regexp", + rawACL: "min.*", + isNegativeACL: false, + filters: filtersNegativeNonRegexp, + want: false, + }, + { + name: "Original filter is a regexp and not a subfilter of the new ACL", + comment: "Original expression should be modified, because the original filter is a regexp and not a subfilter of the new ACL", + rawACL: "min.*, control.*", + isNegativeACL: false, + filters: []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "mini.*", + IsRegexp: true, + IsNegative: false, + }, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qm, err := NewQueryModifier(tt.rawACL) + if err != nil { + t.Fatal(err) + } + + if tt.isNegativeACL { + // 1. NewACL cannot produce negative ACLs + // 2. Cannot convert non-regexp to a negative regexp + if qm.ACL.LabelFilter.IsNegative || !qm.ACL.LabelFilter.IsRegexp { + t.Fatal("Incorrect test data") + } + qm.ACL.LabelFilter.IsNegative = tt.isNegativeACL + } + + got := qm.shouldNotBeModified(tt.filters) + assert.Equal(t, tt.want, got, tt.comment) + }) + } + + t.Run("Repeating regexp filters (not subfilters)", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + { + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("mini.*") + if err != nil { + t.Fatal(err) + } + + want := false + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should be modified, because the original filters contain regexp filters, which are not subfilters of the new filter") + }) + + t.Run("Multiple regexp filters, one of which is not a subfilter of the new filter", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + { + Label: "namespace", + Value: "contro.*", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("min.*, control.*") + if err != nil { + t.Fatal(err) + } + + want := false + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should be modified, because the original filters are regexps, one of which is not a subfilter of the new filter") + }) + + t.Run("Mix of a non-matching regexp and a non-regexp filters", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: false, + }, + { + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("mini.*") + if err != nil { + t.Fatal(err) + } + + want := false + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should be modified, because amongst the original filters with the same label (regexp, non-regexp) there is a regexp, which is not a subfilter of the new filter") + }) + + // Matching cases + + t.Run("Original filter is not a regexp, new filter matches", func(t *testing.T) { + qm, err := NewQueryModifier("min.*, control.*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filtersNonRegexp) + assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter is not a regexp and the new filter is a matching positive regexp") + }) + + t.Run("Original filter is a fake positive regexp (no special symbols), new filter matches", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "minio", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("min.*, control.*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter is a fake positive regexp (it doesn't contain any special characters, should have been a non-regexp expression, e.g. namespace=~\"kube-system\") and the new filter is a matching positive regexp") + }) + + t.Run("Original filter is a regexp and a subfilter of the new ACL", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("min.*, control.*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter is a regexp subfilter of the ACL") + }) + + t.Run("Multiple regexp filters, new filter matches (subfilters)", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + { + Label: "namespace", + Value: "control.*", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("min.*, control.*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filters are subfilters of the new filter") + }) + + t.Run("Repeating filters, new filter matches", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: false, + }, + { + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("mini.*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should NOT be modified, because the original filter contains the same non-regexp label filter multiple times and the new filter matches") + }) + + t.Run("Original filters are a mix of a fake regexp and a non-regexp filters and the new filter matches", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: false, + }, + { + Label: "namespace", + Value: "minio", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("mini.*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should NOT be modified, because original filters contain a mix of a fake regexp and a non-regexp filters (basically, they're equal in results)") + }) + + t.Run("Original filter and the new filter contain the same regexp", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + }, + } + + qm, err := NewQueryModifier("min.*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filters) + assert.Equal(t, want, got, "Original expression should NOT be modified, because original filter and the new filter contain the same regexp") + }) + + t.Run("The new filter gives full access", func(t *testing.T) { + qm, err := NewQueryModifier(".*") + if err != nil { + t.Fatal(err) + } + + want := true + got := qm.shouldNotBeModified(filtersNoTargetLabel) + assert.Equal(t, want, got, "Original expression should NOT be modified, because the new filter gives full access") + }) +} + +func Test_appendOrMergeRegexpLF(t *testing.T) { + t.Run("Non-Regexp LF", func(t *testing.T) { + newFilter := metricsql.LabelFilter{ + Label: "namespace", + Value: "ReplacedValue", + } + + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialVal.*", + IsRegexp: true, + }, + { + Label: "namespace", + Value: "InitialValu.*", + IsRegexp: true, + IsNegative: true, + }, + } + + want := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialVal.*", + IsRegexp: true, + }, + { + Label: "namespace", + Value: "InitialValu.*", + IsRegexp: true, + IsNegative: true, + }, + // The function doesn't take into account non-regexp LFs, that's why it's added + { + Label: "namespace", + Value: "ReplacedValue", + }, + } + got := appendOrMergeRegexpLF(filters, newFilter) + assert.Equal(t, want, got) + }) + + t.Run("Positive Regexp", func(t *testing.T) { + newFilter := metricsql.LabelFilter{ + Label: "namespace", + Value: "ReplacedValue", + IsRegexp: true, + } + + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialVal.*", + IsRegexp: true, + }, + { + Label: "namespace", + Value: "InitialValu.*", + IsRegexp: true, + IsNegative: true, + }, + } + + want := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "InitialValue", + }, + // Only positive regexp should get modified (replaced) + { + Label: "namespace", + Value: "ReplacedValue", + IsRegexp: true, + }, + { + Label: "namespace", + Value: "InitialValu.*", + IsRegexp: true, + IsNegative: true, + }, + } + got := appendOrMergeRegexpLF(filters, newFilter) + assert.Equal(t, want, got) + }) + + t.Run("Negative Regexp", func(t *testing.T) { + newFilter := metricsql.LabelFilter{ + Label: "namespace", + Value: "MergedValue", + IsRegexp: true, + IsNegative: true, + } + + filters := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialVal.*", + IsRegexp: true, + }, + { + Label: "namespace", + Value: "InitialValu.*", + IsRegexp: true, + IsNegative: true, + }, + } + + want := []metricsql.LabelFilter{ + { + Label: "namespace", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialVal.*", + IsRegexp: true, + }, + // Only negative regexp should get modified (Merged) + { + Label: "namespace", + Value: "InitialValu.*|MergedValue", + IsRegexp: true, + IsNegative: true, + }, + } + got := appendOrMergeRegexpLF(filters, newFilter) + assert.Equal(t, want, got) + }) + +} + +func Test_replaceLFByName(t *testing.T) { + newFilter := metricsql.LabelFilter{ + Label: "namespace", + Value: "ReplacedValue", + } + + t.Run("No matching label", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "job", + Value: "InitialValue", + }, + } + + want := []metricsql.LabelFilter{ + { + Label: "job", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "ReplacedValue", + }, + } + got := replaceLFByName(filters, newFilter) + assert.Equal(t, want, got) + }) + + t.Run("1 matching label", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "job", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialValue", + }, + } + + want := []metricsql.LabelFilter{ + { + Label: "job", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "ReplacedValue", + }, + } + got := replaceLFByName(filters, newFilter) + assert.Equal(t, want, got) + + }) + + t.Run("many matching labels", func(t *testing.T) { + filters := []metricsql.LabelFilter{ + { + Label: "job", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "InitialValu.*", + IsRegexp: true, + }, + { + Label: "namespace", + Value: "InitialValue.*", + IsRegexp: true, + IsNegative: true, + }, + } + + want := []metricsql.LabelFilter{ + { + Label: "job", + Value: "InitialValue", + }, + { + Label: "namespace", + Value: "ReplacedValue", + }, + } + got := replaceLFByName(filters, newFilter) + assert.Equal(t, want, got) + }) +} + +func Test_isFakePositiveRegexp(t *testing.T) { + t.Run("Not a regexp", func(t *testing.T) { + filter := metricsql.LabelFilter{ + Label: "namespace", + Value: "minio", + IsRegexp: false, + IsNegative: false, + } + + want := false + got := isFakePositiveRegexp(filter) + assert.Equal(t, want, got) + }) + + t.Run("Fake positive regexp", func(t *testing.T) { + filter := metricsql.LabelFilter{ + Label: "namespace", + Value: "minio", + IsRegexp: true, + IsNegative: false, + } + + want := true + got := isFakePositiveRegexp(filter) + assert.Equal(t, want, got) + }) + + t.Run("Fake negative regexp", func(t *testing.T) { + filter := metricsql.LabelFilter{ + Label: "namespace", + Value: "minio", + IsRegexp: true, + IsNegative: true, + } + + want := false + got := isFakePositiveRegexp(filter) + assert.Equal(t, want, got) + }) + + t.Run("Real positive regexp", func(t *testing.T) { + filter := metricsql.LabelFilter{ + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: false, + } + + want := false + got := isFakePositiveRegexp(filter) + assert.Equal(t, want, got) + }) + + t.Run("Real negative regexp", func(t *testing.T) { + filter := metricsql.LabelFilter{ + Label: "namespace", + Value: "min.*", + IsRegexp: true, + IsNegative: true, + } + + want := false + got := isFakePositiveRegexp(filter) + assert.Equal(t, want, got) + }) +}