Skip to content

Commit 50ef328

Browse files
sparrcsparrc
committed
Refactor and code cleanup of filtering
started working on this with the idea of fixing influxdata#1623, although I realized that this was actually just a documentation issue around a toml eccentricity. closes influxdata#1623
1 parent b63dedb commit 50ef328

File tree

12 files changed

+216
-173
lines changed

12 files changed

+216
-173
lines changed

‎agent/accumulator.go

Lines changed: 14 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,8 @@ func (ac *accumulator) makeMetric(
8383
if len(fields) == 0 || len(measurement) == 0 {
8484
return nil
8585
}
86-
87-
if !ac.inputConfig.Filter.ShouldNamePass(measurement) {
88-
return nil
89-
}
90-
91-
if !ac.inputConfig.Filter.ShouldTagsPass(tags) {
92-
return nil
86+
if tags == nil {
87+
tags = make(map[string]string)
9388
}
9489

9590
// Override measurement name if set
@@ -104,9 +99,6 @@ func (ac *accumulator) makeMetric(
10499
measurement = measurement + ac.inputConfig.MeasurementSuffix
105100
}
106101

107-
if tags == nil {
108-
tags = make(map[string]string)
109-
}
110102
// Apply plugin-wide tags if set
111103
for k, v := range ac.inputConfig.Tags {
112104
if _, ok := tags[k]; !ok {
@@ -119,25 +111,21 @@ func (ac *accumulator) makeMetric(
119111
tags[k] = v
120112
}
121113
}
122-
ac.inputConfig.Filter.FilterTags(tags)
123114

124-
result := make(map[string]interface{})
125-
for k, v := range fields {
126-
// Filter out any filtered fields
127-
if ac.inputConfig != nil {
128-
if !ac.inputConfig.Filter.ShouldFieldsPass(k) {
129-
continue
130-
}
131-
}
115+
// Apply the metric filter(s)
116+
if ok := ac.inputConfig.Filter.Apply(measurement, fields, tags); !ok {
117+
return nil
118+
}
132119

120+
for k, v := range fields {
133121
// Validate uint64 and float64 fields
134122
switch val := v.(type) {
135123
case uint64:
136124
// InfluxDB does not support writing uint64
137125
if val < uint64(9223372036854775808) {
138-
result[k] = int64(val)
126+
fields[k] = int64(val)
139127
} else {
140-
result[k] = int64(9223372036854775807)
128+
fields[k] = int64(9223372036854775807)
141129
}
142130
continue
143131
case float64:
@@ -148,15 +136,12 @@ func (ac *accumulator) makeMetric(
148136
"field, skipping",
149137
measurement, k)
150138
}
139+
delete(fields, k)
151140
continue
152141
}
153142
}
154143

155-
result[k] = v
156-
}
157-
fields = nil
158-
if len(result) == 0 {
159-
return nil
144+
fields[k] = v
160145
}
161146

162147
var timestamp time.Time
@@ -171,11 +156,11 @@ func (ac *accumulator) makeMetric(
171156
var err error
172157
switch mType {
173158
case telegraf.Counter:
174-
m, err = telegraf.NewCounterMetric(measurement, tags, result, timestamp)
159+
m, err = telegraf.NewCounterMetric(measurement, tags, fields, timestamp)
175160
case telegraf.Gauge:
176-
m, err = telegraf.NewGaugeMetric(measurement, tags, result, timestamp)
161+
m, err = telegraf.NewGaugeMetric(measurement, tags, fields, timestamp)
177162
default:
178-
m, err = telegraf.NewMetric(measurement, tags, result, timestamp)
163+
m, err = telegraf.NewMetric(measurement, tags, fields, timestamp)
179164
}
180165
if err != nil {
181166
log.Printf("Error adding point [%s]: %s\n", measurement, err.Error())

‎agent/accumulator_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -560,7 +560,7 @@ func TestAccFilterTags(t *testing.T) {
560560
filter := models.Filter{
561561
TagExclude: []string{"acc"},
562562
}
563-
assert.NoError(t, filter.CompileFilter())
563+
assert.NoError(t, filter.Compile())
564564
a.inputConfig = &models.InputConfig{}
565565
a.inputConfig.Filter = filter
566566

‎docs/CONFIGURATION.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ as it is more efficient to filter out tags at the ingestion point.
8686
* **taginclude**: taginclude is the inverse of tagexclude. It will only include
8787
the tag keys in the final measurement.
8888

89+
**NOTE** `tagpass` and `tagdrop` parameters must be defined at the _end_ of
90+
the plugin definition, otherwise subsequent plugin config options will be
91+
interpreted as part of the tagpass/tagdrop map.
92+
8993
## Input Configuration
9094

9195
Some configuration options are configurable per input:
@@ -129,6 +133,10 @@ fields which begin with `time_`.
129133

130134
#### Input Config: tagpass and tagdrop
131135

136+
**NOTE** `tagpass` and `tagdrop` parameters must be defined at the _end_ of
137+
the plugin definition, otherwise subsequent plugin config options will be
138+
interpreted as part of the tagpass/tagdrop map.
139+
132140
```toml
133141
[[inputs.cpu]]
134142
percpu = true

‎filter/filter.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,16 @@ type Filter interface {
1010
Match(string) bool
1111
}
1212

13-
// CompileFilter takes a list of string filters and returns a Filter interface
13+
// Compile takes a list of string filters and returns a Filter interface
1414
// for matching a given string against the filter list. The filter list
1515
// supports glob matching too, ie:
1616
//
17-
// f, _ := CompileFilter([]string{"cpu", "mem", "net*"})
17+
// f, _ := Compile([]string{"cpu", "mem", "net*"})
1818
// f.Match("cpu") // true
1919
// f.Match("network") // true
2020
// f.Match("memory") // false
2121
//
22-
func CompileFilter(filters []string) (Filter, error) {
22+
func Compile(filters []string) (Filter, error) {
2323
// return if there is nothing to compile
2424
if len(filters) == 0 {
2525
return nil, nil

‎filter/filter_test.go

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,30 @@ import (
66
"github.com/stretchr/testify/assert"
77
)
88

9-
func TestCompileFilter(t *testing.T) {
10-
f, err := CompileFilter([]string{})
9+
func TestCompile(t *testing.T) {
10+
f, err := Compile([]string{})
1111
assert.NoError(t, err)
1212
assert.Nil(t, f)
1313

14-
f, err = CompileFilter([]string{"cpu"})
14+
f, err = Compile([]string{"cpu"})
1515
assert.NoError(t, err)
1616
assert.True(t, f.Match("cpu"))
1717
assert.False(t, f.Match("cpu0"))
1818
assert.False(t, f.Match("mem"))
1919

20-
f, err = CompileFilter([]string{"cpu*"})
20+
f, err = Compile([]string{"cpu*"})
2121
assert.NoError(t, err)
2222
assert.True(t, f.Match("cpu"))
2323
assert.True(t, f.Match("cpu0"))
2424
assert.False(t, f.Match("mem"))
2525

26-
f, err = CompileFilter([]string{"cpu", "mem"})
26+
f, err = Compile([]string{"cpu", "mem"})
2727
assert.NoError(t, err)
2828
assert.True(t, f.Match("cpu"))
2929
assert.False(t, f.Match("cpu0"))
3030
assert.True(t, f.Match("mem"))
3131

32-
f, err = CompileFilter([]string{"cpu", "mem", "net*"})
32+
f, err = Compile([]string{"cpu", "mem", "net*"})
3333
assert.NoError(t, err)
3434
assert.True(t, f.Match("cpu"))
3535
assert.False(t, f.Match("cpu0"))
@@ -40,7 +40,7 @@ func TestCompileFilter(t *testing.T) {
4040
var benchbool bool
4141

4242
func BenchmarkFilterSingleNoGlobFalse(b *testing.B) {
43-
f, _ := CompileFilter([]string{"cpu"})
43+
f, _ := Compile([]string{"cpu"})
4444
var tmp bool
4545
for n := 0; n < b.N; n++ {
4646
tmp = f.Match("network")
@@ -49,7 +49,7 @@ func BenchmarkFilterSingleNoGlobFalse(b *testing.B) {
4949
}
5050

5151
func BenchmarkFilterSingleNoGlobTrue(b *testing.B) {
52-
f, _ := CompileFilter([]string{"cpu"})
52+
f, _ := Compile([]string{"cpu"})
5353
var tmp bool
5454
for n := 0; n < b.N; n++ {
5555
tmp = f.Match("cpu")
@@ -58,7 +58,7 @@ func BenchmarkFilterSingleNoGlobTrue(b *testing.B) {
5858
}
5959

6060
func BenchmarkFilter(b *testing.B) {
61-
f, _ := CompileFilter([]string{"cpu", "mem", "net*"})
61+
f, _ := Compile([]string{"cpu", "mem", "net*"})
6262
var tmp bool
6363
for n := 0; n < b.N; n++ {
6464
tmp = f.Match("network")
@@ -67,7 +67,7 @@ func BenchmarkFilter(b *testing.B) {
6767
}
6868

6969
func BenchmarkFilterNoGlob(b *testing.B) {
70-
f, _ := CompileFilter([]string{"cpu", "mem", "net"})
70+
f, _ := Compile([]string{"cpu", "mem", "net"})
7171
var tmp bool
7272
for n := 0; n < b.N; n++ {
7373
tmp = f.Match("net")
@@ -76,7 +76,7 @@ func BenchmarkFilterNoGlob(b *testing.B) {
7676
}
7777

7878
func BenchmarkFilter2(b *testing.B) {
79-
f, _ := CompileFilter([]string{"aa", "bb", "c", "ad", "ar", "at", "aq",
79+
f, _ := Compile([]string{"aa", "bb", "c", "ad", "ar", "at", "aq",
8080
"aw", "az", "axxx", "ab", "cpu", "mem", "net*"})
8181
var tmp bool
8282
for n := 0; n < b.N; n++ {
@@ -86,7 +86,7 @@ func BenchmarkFilter2(b *testing.B) {
8686
}
8787

8888
func BenchmarkFilter2NoGlob(b *testing.B) {
89-
f, _ := CompileFilter([]string{"aa", "bb", "c", "ad", "ar", "at", "aq",
89+
f, _ := Compile([]string{"aa", "bb", "c", "ad", "ar", "at", "aq",
9090
"aw", "az", "axxx", "ab", "cpu", "mem", "net"})
9191
var tmp bool
9292
for n := 0; n < b.N; n++ {

‎internal/config/config.go

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,6 @@ func buildFilter(tbl *ast.Table) (models.Filter, error) {
665665
for _, elem := range ary.Value {
666666
if str, ok := elem.(*ast.String); ok {
667667
f.NamePass = append(f.NamePass, str.Value)
668-
f.IsActive = true
669668
}
670669
}
671670
}
@@ -678,7 +677,6 @@ func buildFilter(tbl *ast.Table) (models.Filter, error) {
678677
for _, elem := range ary.Value {
679678
if str, ok := elem.(*ast.String); ok {
680679
f.NameDrop = append(f.NameDrop, str.Value)
681-
f.IsActive = true
682680
}
683681
}
684682
}
@@ -693,7 +691,6 @@ func buildFilter(tbl *ast.Table) (models.Filter, error) {
693691
for _, elem := range ary.Value {
694692
if str, ok := elem.(*ast.String); ok {
695693
f.FieldPass = append(f.FieldPass, str.Value)
696-
f.IsActive = true
697694
}
698695
}
699696
}
@@ -709,7 +706,6 @@ func buildFilter(tbl *ast.Table) (models.Filter, error) {
709706
for _, elem := range ary.Value {
710707
if str, ok := elem.(*ast.String); ok {
711708
f.FieldDrop = append(f.FieldDrop, str.Value)
712-
f.IsActive = true
713709
}
714710
}
715711
}
@@ -730,7 +726,6 @@ func buildFilter(tbl *ast.Table) (models.Filter, error) {
730726
}
731727
}
732728
f.TagPass = append(f.TagPass, *tagfilter)
733-
f.IsActive = true
734729
}
735730
}
736731
}
@@ -749,7 +744,6 @@ func buildFilter(tbl *ast.Table) (models.Filter, error) {
749744
}
750745
}
751746
f.TagDrop = append(f.TagDrop, *tagfilter)
752-
f.IsActive = true
753747
}
754748
}
755749
}
@@ -778,7 +772,7 @@ func buildFilter(tbl *ast.Table) (models.Filter, error) {
778772
}
779773
}
780774
}
781-
if err := f.CompileFilter(); err != nil {
775+
if err := f.Compile(); err != nil {
782776
return f, err
783777
}
784778

‎internal/config/config_test.go

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,8 @@ func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
4343
Filter: []string{"mytag"},
4444
},
4545
},
46-
IsActive: true,
4746
}
48-
assert.NoError(t, filter.CompileFilter())
47+
assert.NoError(t, filter.Compile())
4948
mConfig := &models.InputConfig{
5049
Name: "memcached",
5150
Filter: filter,
@@ -83,9 +82,8 @@ func TestConfig_LoadSingleInput(t *testing.T) {
8382
Filter: []string{"mytag"},
8483
},
8584
},
86-
IsActive: true,
8785
}
88-
assert.NoError(t, filter.CompileFilter())
86+
assert.NoError(t, filter.Compile())
8987
mConfig := &models.InputConfig{
9088
Name: "memcached",
9189
Filter: filter,
@@ -130,9 +128,8 @@ func TestConfig_LoadDirectory(t *testing.T) {
130128
Filter: []string{"mytag"},
131129
},
132130
},
133-
IsActive: true,
134131
}
135-
assert.NoError(t, filter.CompileFilter())
132+
assert.NoError(t, filter.Compile())
136133
mConfig := &models.InputConfig{
137134
Name: "memcached",
138135
Filter: filter,

0 commit comments

Comments
 (0)