diff --git a/src/META-INF/MANIFEST.MF b/src/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..348f1bdd38 --- /dev/null +++ b/src/META-INF/MANIFEST.MF @@ -0,0 +1 @@ +Manifest-Version: 1.0 \ No newline at end of file diff --git a/src/META-INF/services/net.opentsdb.uid.UniqueIdFilterPlugin b/src/META-INF/services/net.opentsdb.uid.UniqueIdFilterPlugin new file mode 100644 index 0000000000..010e95484c --- /dev/null +++ b/src/META-INF/services/net.opentsdb.uid.UniqueIdFilterPlugin @@ -0,0 +1 @@ +net.opentsdb.uid.UniqueIdWhitelistFilter \ No newline at end of file diff --git a/src/uid/UniqueIdWhitelistFilter.java b/src/uid/UniqueIdWhitelistFilter.java new file mode 100644 index 0000000000..371ea7bd50 --- /dev/null +++ b/src/uid/UniqueIdWhitelistFilter.java @@ -0,0 +1,200 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.uid; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import com.google.common.annotations.VisibleForTesting; +import com.stumbleupon.async.Deferred; + +import net.opentsdb.core.TSDB; +import net.opentsdb.stats.StatsCollector; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +/** + * A UID filter implementation using regular expression based whitelists. + * Multiple regular expressions can be provided in the configuration file with + * a configurable delimiter. Each expression is compiled into a list per UID + * type and when a new UID passes through the filter, each expression in the + * list is compared to make sure the name satisfies all expressions. + */ +public class UniqueIdWhitelistFilter extends UniqueIdFilterPlugin { + + /** Default delimiter */ + public static final String DEFAULT_REGEX_DELIMITER = ","; + + /** Lists of patterns for each type. */ + private List metric_patterns; + private List tagk_patterns; + private List tagv_patterns; + + /** Counters for tracking stats */ + private final AtomicLong metrics_rejected = new AtomicLong(); + private final AtomicLong metrics_allowed = new AtomicLong(); + private final AtomicLong tagks_rejected = new AtomicLong(); + private final AtomicLong tagks_allowed = new AtomicLong(); + private final AtomicLong tagvs_rejected = new AtomicLong(); + private final AtomicLong tagvs_allowed = new AtomicLong(); + + @Override + public void initialize(final TSDB tsdb) { + final Config config = tsdb.getConfig(); + String delimiter = config.getString("tsd.uidfilter.whitelist.delimiter"); + if (delimiter == null) { + delimiter = DEFAULT_REGEX_DELIMITER; + } + + String raw = config.getString("tsd.uidfilter.whitelist.metric_patterns"); + if (raw != null) { + final String[] splits = raw.split(delimiter); + metric_patterns = new ArrayList(splits.length); + for (final String pattern : splits) { + try { + metric_patterns.add(Pattern.compile(pattern)); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("The metric whitelist pattern [" + + pattern + "] does not compile.", e); + } + } + } + + raw = config.getString("tsd.uidfilter.whitelist.tagk_patterns"); + if (raw != null) { + final String[] splits = raw.split(delimiter); + tagk_patterns = new ArrayList(splits.length); + for (final String pattern : splits) { + try { + tagk_patterns.add(Pattern.compile(pattern)); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("The tagk whitelist pattern [" + + pattern + "] does not compile.", e); + } + } + } + + raw = config.getString("tsd.uidfilter.whitelist.tagv_patterns"); + if (raw != null) { + final String[] splits = raw.split(delimiter); + tagv_patterns = new ArrayList(splits.length); + for (final String pattern : splits) { + try { + tagv_patterns.add(Pattern.compile(pattern)); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("The tagv whitelist pattern [" + + pattern + "] does not compile.", e); + } + } + } + } + + @Override + public Deferred shutdown() { + return Deferred.fromResult(null); + } + + @Override + public String version() { + return "2.3.0"; + } + + @Override + public void collectStats(final StatsCollector collector) { + collector.record("uid.filter.whitelist.accepted", metrics_allowed.get(), + "type=metrics"); + collector.record("uid.filter.whitelist.accepted", tagks_allowed.get(), + "type=tagk"); + collector.record("uid.filter.whitelist.accepted", tagvs_allowed.get(), + "type=tagv"); + collector.record("uid.filter.whitelist.rejected", metrics_rejected.get(), + "type=metrics"); + collector.record("uid.filter.whitelist.rejected", tagks_rejected.get(), + "type=tagk"); + collector.record("uid.filter.whitelist.rejected", tagvs_rejected.get(), + "type=tagv"); + } + + @Override + public Deferred allowUIDAssignment( + final UniqueIdType type, + final String value, + final String metric, + final Map tags) { + + switch (type) { + case METRIC: + if (metric_patterns != null) { + for (final Pattern pattern : metric_patterns) { + if (!pattern.matcher(value).find()) { + metrics_rejected.incrementAndGet(); + return Deferred.fromResult(false); + } + } + } + metrics_allowed.incrementAndGet(); + break; + + case TAGK: + if (tagk_patterns != null) { + for (final Pattern pattern : tagk_patterns) { + if (!pattern.matcher(value).find()) { + tagks_rejected.incrementAndGet(); + return Deferred.fromResult(false); + } + } + } + tagks_allowed.incrementAndGet(); + break; + + case TAGV: + if (tagv_patterns != null) { + for (final Pattern pattern : tagv_patterns) { + if (!pattern.matcher(value).find()) { + tagvs_rejected.incrementAndGet(); + return Deferred.fromResult(false); + } + } + } + tagvs_allowed.incrementAndGet(); + break; + } + + // all patterns passed, yay! + return Deferred.fromResult(true); + } + + @Override + public boolean fillterUIDAssignments() { + return true; + } + + @VisibleForTesting + List metricPatterns() { + return metric_patterns; + } + + @VisibleForTesting + List tagkPatterns() { + return tagk_patterns; + } + + @VisibleForTesting + List tagvPatterns() { + return tagv_patterns; + } +} diff --git a/test/uid/TestUniqueIdWhitelistFilter.java b/test/uid/TestUniqueIdWhitelistFilter.java new file mode 100644 index 0000000000..c07f9c26e2 --- /dev/null +++ b/test/uid/TestUniqueIdWhitelistFilter.java @@ -0,0 +1,165 @@ +// This file is part of OpenTSDB. +// Copyright (C) 2016 The OpenTSDB Authors. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 2.1 of the License, or (at your +// option) any later version. This program is distributed in the hope that it +// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty +// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. You should have received a copy +// of the GNU Lesser General Public License along with this program. If not, +// see . +package net.opentsdb.uid; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import net.opentsdb.core.TSDB; +import net.opentsdb.uid.UniqueId.UniqueIdType; +import net.opentsdb.utils.Config; + +@RunWith(PowerMockRunner.class) +//"Classloader hell"... It's real. Tell PowerMock to ignore these classes +//because they fiddle with the class loader. We don't test them anyway. +@PowerMockIgnore({"javax.management.*", "javax.xml.*", + "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"}) +@PrepareForTest({ TSDB.class, Config.class }) +public class TestUniqueIdWhitelistFilter { + + private TSDB tsdb; + private Config config; + private UniqueIdWhitelistFilter filter; + + @Before + public void before() throws Exception { + tsdb = PowerMockito.mock(TSDB.class); + config = new Config(false); + when(tsdb.getConfig()).thenReturn(config); + filter = new UniqueIdWhitelistFilter(); + + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*"); + } + + @Test + public void ctor() throws Exception { + assertNull(filter.metricPatterns()); + assertNull(filter.tagkPatterns()); + assertNull(filter.tagvPatterns()); + } + + @Test + public void initalize() throws Exception { + filter.initialize(tsdb); + assertEquals(1, filter.metricPatterns().size()); + assertEquals(".*", filter.metricPatterns().get(0).pattern()); + assertEquals(1, filter.tagkPatterns().size()); + assertEquals(".*", filter.tagkPatterns().get(0).pattern()); + assertEquals(1, filter.tagvPatterns().size()); + assertEquals(".*", filter.tagvPatterns().get(0).pattern()); + } + + @Test + public void initalizeMultiplePatterns() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*,^test.*"); + filter.initialize(tsdb); + assertEquals(2, filter.metricPatterns().size()); + assertEquals(".*", filter.metricPatterns().get(0).pattern()); + assertEquals("^test.*", filter.metricPatterns().get(1).pattern()); + assertEquals(2, filter.tagkPatterns().size()); + assertEquals(".*", filter.tagkPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagkPatterns().get(1).pattern()); + assertEquals(2, filter.tagvPatterns().size()); + assertEquals(".*", filter.tagvPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagvPatterns().get(1).pattern()); + } + + @Test + public void initalizeMultiplePatternsAlternateDelimiter() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.delimiter", "\\|"); + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*|^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*|^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*|^test.*"); + filter.initialize(tsdb); + assertEquals(2, filter.metricPatterns().size()); + assertEquals(".*", filter.metricPatterns().get(0).pattern()); + assertEquals("^test.*", filter.metricPatterns().get(1).pattern()); + assertEquals(2, filter.tagkPatterns().size()); + assertEquals(".*", filter.tagkPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagkPatterns().get(1).pattern()); + assertEquals(2, filter.tagvPatterns().size()); + assertEquals(".*", filter.tagvPatterns().get(0).pattern()); + assertEquals("^test.*", filter.tagvPatterns().get(1).pattern()); + } + + @Test (expected = IllegalArgumentException.class) + public void initalizeBadRegex() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", "grp[start"); + filter.initialize(tsdb); + } + + @Test + public void shutdown() throws Exception { + assertNull(filter.shutdown().join()); + } + + @Test + public void allowUIDAssignment() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", "^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", "^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", "^test.*"); + filter.initialize(tsdb); + assertTrue(filter.allowUIDAssignment(UniqueIdType.METRIC, "test_metric", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.METRIC, "metric", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGK, "test_tagk", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGK, "tagk", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGV, "test_tagv", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGV, "tagv", + null, null).join()); + } + + @Test + public void allowUIDAssignmentMultiplePaterns() throws Exception { + config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*,^test.*"); + config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*,^test.*"); + filter.initialize(tsdb); + assertTrue(filter.allowUIDAssignment(UniqueIdType.METRIC, "test_metric", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.METRIC, "metric", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGK, "test_tagk", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGK, "tagk", + null, null).join()); + assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGV, "test_tagv", + null, null).join()); + assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGV, "tagv", + null, null).join()); + } + + @Test + public void fillterUIDAssignments() throws Exception { + assertTrue(filter.fillterUIDAssignments()); + } +}