diff --git a/config/config.go b/config/config.go index 97f8afc974497..bab955e932616 100644 --- a/config/config.go +++ b/config/config.go @@ -1331,7 +1331,9 @@ func (c *Config) buildFilter(tbl *ast.Table) (models.Filter, error) { f := models.Filter{} c.getFieldStringSlice(tbl, "namepass", &f.NamePass) + c.getFieldString(tbl, "namepass_separator", &f.NamePassSeparators) c.getFieldStringSlice(tbl, "namedrop", &f.NameDrop) + c.getFieldString(tbl, "namedrop_separator", &f.NameDropSeparators) c.getFieldStringSlice(tbl, "pass", &f.FieldPass) c.getFieldStringSlice(tbl, "fieldpass", &f.FieldPass) @@ -1446,7 +1448,7 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error { "interval", "lvm", // What is this used for? "metric_batch_size", "metric_buffer_limit", "metricpass", - "name_override", "name_prefix", "name_suffix", "namedrop", "namepass", + "name_override", "name_prefix", "name_suffix", "namedrop", "namedrop_separator", "namepass", "namepass_separator", "order", "pass", "period", "precision", "tagdrop", "tagexclude", "taginclude", "tagpass", "tags": diff --git a/config/config_test.go b/config/config_test.go index 870e7a2b8939e..aba436f2d52bb 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -150,6 +150,49 @@ func TestConfig_LoadSingleInput(t *testing.T) { require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct memcached metadata.") } +func TestConfig_LoadSingleInput_WithSeparators(t *testing.T) { + c := config.NewConfig() + require.NoError(t, c.LoadConfig("./testdata/single_plugin_with_separators.toml")) + + input := inputs.Inputs["memcached"]().(*MockupInputPlugin) + input.Servers = []string{"localhost"} + + filter := models.Filter{ + NameDrop: []string{"metricname2"}, + NameDropSeparators: ".", + NamePass: []string{"metricname1"}, + NamePassSeparators: ".", + FieldDrop: []string{"other", "stuff"}, + FieldPass: []string{"some", "strings"}, + TagDropFilters: []models.TagFilter{ + { + Name: "badtag", + Values: []string{"othertag"}, + }, + }, + TagPassFilters: []models.TagFilter{ + { + Name: "goodtag", + Values: []string{"mytag"}, + }, + }, + } + require.NoError(t, filter.Compile()) + inputConfig := &models.InputConfig{ + Name: "memcached", + Filter: filter, + Interval: 5 * time.Second, + } + inputConfig.Tags = make(map[string]string) + + // Ignore Log, Parser and ID + c.Inputs[0].Input.(*MockupInputPlugin).Log = nil + c.Inputs[0].Input.(*MockupInputPlugin).parser = nil + c.Inputs[0].Config.ID = "" + require.Equal(t, input, c.Inputs[0].Input, "Testdata did not produce a correct memcached struct.") + require.Equal(t, inputConfig, c.Inputs[0].Config, "Testdata did not produce correct memcached metadata.") +} + func TestConfig_LoadDirectory(t *testing.T) { c := config.NewConfig() diff --git a/config/testdata/single_plugin_with_separators.toml b/config/testdata/single_plugin_with_separators.toml new file mode 100644 index 0000000000000..acac08c1e7e47 --- /dev/null +++ b/config/testdata/single_plugin_with_separators.toml @@ -0,0 +1,13 @@ +[[inputs.memcached]] + servers = ["localhost"] + namepass = ["metricname1"] + namepass_separator = "." + namedrop = ["metricname2"] + namedrop_separator = "." + fieldpass = ["some", "strings"] + fielddrop = ["other", "stuff"] + interval = "5s" + [inputs.memcached.tagpass] + goodtag = ["mytag"] + [inputs.memcached.tagdrop] + badtag = ["othertag"] diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index dc00fc8bfa726..36cd80fc80865 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -626,11 +626,15 @@ sent onwards to the next stage of processing. - **namepass**: An array of [glob pattern][] strings. Only metrics whose measurement name -matches a pattern in this list are emitted. +matches a pattern in this list are emitted. Additionally, custom list of +separators can be specified using `namepass_separator`. These separators +are excluded from wildcard glob pattern matching. - **namedrop**: The inverse of `namepass`. If a match is found the metric is discarded. This -is tested on metrics after they have passed the `namepass` test. +is tested on metrics after they have passed the `namepass` test. Additionally, +custom list of separators can be specified using `namedrop_separator`. These +separators are excluded from wildcard glob pattern matching. - **tagpass**: A table mapping tag keys to arrays of [glob pattern][] strings. Only metrics @@ -770,6 +774,26 @@ tags and the agent `host` tag. namepass = ["rest_client_*"] ``` +#### Using namepass and namedrop with separators + +```toml +# Pass all metrics of type 'A.C.B' and drop all others like 'A.C.D.B' +[[inputs.socket_listener]] + data_format = "graphite" + templates = ["measurement*"] + + namepass = ["A.*.B"] + namepass_separator = "." + +# Drop all metrics of type 'A.C.B' and pass all others like 'A.C.D.B' +[[inputs.socket_listener]] + data_format = "graphite" + templates = ["measurement*"] + + namedrop = ["A.*.B"] + namedrop_separator = "." +``` + #### Using taginclude and tagexclude ```toml diff --git a/filter/filter.go b/filter/filter.go index 3e3798a47d2d8..88e946180ad44 100644 --- a/filter/filter.go +++ b/filter/filter.go @@ -12,20 +12,29 @@ type Filter interface { // Compile takes a list of string filters and returns a Filter interface // for matching a given string against the filter list. The filter list -// supports glob matching too, ie: +// supports glob matching with separators too, ie: // // f, _ := Compile([]string{"cpu", "mem", "net*"}) // f.Match("cpu") // true // f.Match("network") // true // f.Match("memory") // false -func Compile(filters []string) (Filter, error) { +// +// separators are only to be used for globbing filters, ie: +// +// f, _ := Compile([]string{"cpu.*.count"}, '.') +// f.Match("cpu.count") // false +// f.Match("cpu.measurement.count") // true +// f.Match("cpu.field.measurement.count") // false +// +// Compile will return nil if the filter list is empty. +func Compile(filters []string, separators ...rune) (Filter, error) { // return if there is nothing to compile if len(filters) == 0 { return nil, nil } // check if we can compile a non-glob filter - noGlob := true + noGlob := len(separators) == 0 for _, filter := range filters { if hasMeta(filter) { noGlob = false @@ -38,14 +47,14 @@ func Compile(filters []string) (Filter, error) { // return non-globbing filter if not needed. return compileFilterNoGlob(filters), nil case len(filters) == 1: - return glob.Compile(filters[0]) + return glob.Compile(filters[0], separators...) default: - return glob.Compile("{" + strings.Join(filters, ",") + "}") + return glob.Compile("{"+strings.Join(filters, ",")+"}", separators...) } } -func MustCompile(filters []string) Filter { - f, err := Compile(filters) +func MustCompile(filters []string, separators ...rune) Filter { + f, err := Compile(filters, separators...) if err != nil { panic(err) } diff --git a/filter/filter_test.go b/filter/filter_test.go index 78918948a8998..4268ccbbcae61 100644 --- a/filter/filter_test.go +++ b/filter/filter_test.go @@ -35,6 +35,18 @@ func TestCompile(t *testing.T) { require.False(t, f.Match("cpu0")) require.True(t, f.Match("mem")) require.True(t, f.Match("network")) + + f, err = Compile([]string{"cpu.*.count"}, '.') + require.NoError(t, err) + require.False(t, f.Match("cpu.count")) + require.True(t, f.Match("cpu.measurement.count")) + require.False(t, f.Match("cpu.field.measurement.count")) + + f, err = Compile([]string{"cpu.*.count"}, '.', ',') + require.NoError(t, err) + require.True(t, f.Match("cpu.measurement.count")) + require.False(t, f.Match("cpu.,.count")) // ',' is not considered under * as it is specified as a separator + require.False(t, f.Match("cpu.field,measurement.count")) } func TestIncludeExclude(t *testing.T) { diff --git a/models/filter.go b/models/filter.go index e595e5be4873d..5c55ae3224cbe 100644 --- a/models/filter.go +++ b/models/filter.go @@ -33,10 +33,12 @@ func (tf *TagFilter) Compile() error { // Filter containing drop/pass and tagdrop/tagpass rules type Filter struct { - NameDrop []string - nameDropFilter filter.Filter - NamePass []string - namePassFilter filter.Filter + NameDrop []string + NameDropSeparators string + nameDropFilter filter.Filter + NamePass []string + NamePassSeparators string + namePassFilter filter.Filter FieldDrop []string fieldDropFilter filter.Filter @@ -78,11 +80,11 @@ func (f *Filter) Compile() error { if f.selectActive { var err error - f.nameDropFilter, err = filter.Compile(f.NameDrop) + f.nameDropFilter, err = filter.Compile(f.NameDrop, []rune(f.NameDropSeparators)...) if err != nil { return fmt.Errorf("error compiling 'namedrop', %w", err) } - f.namePassFilter, err = filter.Compile(f.NamePass) + f.namePassFilter, err = filter.Compile(f.NamePass, []rune(f.NamePassSeparators)...) if err != nil { return fmt.Errorf("error compiling 'namepass', %w", err) } diff --git a/models/filter_test.go b/models/filter_test.go index 80f976809f72b..a1586a5d6540d 100644 --- a/models/filter_test.go +++ b/models/filter_test.go @@ -146,6 +146,42 @@ func TestFilter_NamePass(t *testing.T) { } } +func TestFilter_NamePass_WithSeparator(t *testing.T) { + f := Filter{ + NamePass: []string{"foo.*.bar", "foo.*.abc.*.bar"}, + NamePassSeparators: ".,", + } + require.NoError(t, f.Compile()) + + passes := []string{ + "foo..bar", + "foo.abc.bar", + "foo..abc..bar", + "foo.xyz.abc.xyz-xyz.bar", + } + + drops := []string{ + "foo.bar", + "foo.abc,.bar", // "abc," is not considered under * as ',' is specified as a separator + "foo..abc.bar", // ".abc" shall not be matched under * as '.' is specified as a separator + "foo.abc.abc.bar", + "foo.xyz.abc.xyz.xyz.bar", + "foo.xyz.abc.xyz,xyz.bar", + } + + for _, measurement := range passes { + if !f.shouldNamePass(measurement) { + t.Errorf("Expected measurement %s to pass", measurement) + } + } + + for _, measurement := range drops { + if f.shouldNamePass(measurement) { + t.Errorf("Expected measurement %s to drop", measurement) + } + } +} + func TestFilter_NameDrop(t *testing.T) { f := Filter{ NameDrop: []string{"foo*", "cpu_usage_idle"}, @@ -180,6 +216,42 @@ func TestFilter_NameDrop(t *testing.T) { } } +func TestFilter_NameDrop_WithSeparator(t *testing.T) { + f := Filter{ + NameDrop: []string{"foo.*.bar", "foo.*.abc.*.bar"}, + NameDropSeparators: ".,", + } + require.NoError(t, f.Compile()) + + drops := []string{ + "foo..bar", + "foo.abc.bar", + "foo..abc..bar", + "foo.xyz.abc.xyz-xyz.bar", + } + + passes := []string{ + "foo.bar", + "foo.abc,.bar", // "abc," is not considered under * as ',' is specified as a separator + "foo..abc.bar", // ".abc" shall not be matched under * as '.' is specified as a separator + "foo.abc.abc.bar", + "foo.xyz.abc.xyz.xyz.bar", + "foo.xyz.abc.xyz,xyz.bar", + } + + for _, measurement := range passes { + if !f.shouldNamePass(measurement) { + t.Errorf("Expected measurement %s to pass", measurement) + } + } + + for _, measurement := range drops { + if f.shouldNamePass(measurement) { + t.Errorf("Expected measurement %s to drop", measurement) + } + } +} + func TestFilter_FieldPass(t *testing.T) { f := Filter{ FieldPass: []string{"foo*", "cpu_usage_idle"},