Add the UniqueIdWhitelistFilter plugin directly to the TSD code

base so users can instantiate it without loading a separate file.

Signed-off-by: Chris Larsen <[email protected]>

src/META-INF/MANIFEST.MF

@@ -0,0 +1 @@
+Manifest-Version: 1.0

src/META-INF/services/net.opentsdb.uid.UniqueIdFilterPlugin

@@ -0,0 +1 @@
+net.opentsdb.uid.UniqueIdWhitelistFilter

src/uid/UniqueIdWhitelistFilter.java

@@ -0,0 +1,200 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2016  The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version.  This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+// General Public License for more details.  You should have received a copy
+// of the GNU Lesser General Public License along with this program.  If not,
+// see <http://www.gnu.org/licenses/>.
+package net.opentsdb.uid;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.stumbleupon.async.Deferred;
+
+import net.opentsdb.core.TSDB;
+import net.opentsdb.stats.StatsCollector;
+import net.opentsdb.uid.UniqueId.UniqueIdType;
+import net.opentsdb.utils.Config;
+
+/**
+ * A UID filter implementation using regular expression based whitelists.
+ * Multiple regular expressions can be provided in the configuration file with
+ * a configurable delimiter. Each expression is compiled into a list per UID
+ * type and when a new UID passes through the filter, each expression in the
+ * list is compared to make sure the name satisfies all expressions.
+ */
+public class UniqueIdWhitelistFilter extends UniqueIdFilterPlugin {
+
+  /** Default delimiter */
+  public static final String DEFAULT_REGEX_DELIMITER = ",";
+
+  /** Lists of patterns for each type. */
+  private List<Pattern> metric_patterns;
+  private List<Pattern> tagk_patterns;
+  private List<Pattern> tagv_patterns;
+
+  /** Counters for tracking stats */
+  private final AtomicLong metrics_rejected = new AtomicLong();
+  private final AtomicLong metrics_allowed = new AtomicLong();
+  private final AtomicLong tagks_rejected = new AtomicLong();
+  private final AtomicLong tagks_allowed = new AtomicLong();
+  private final AtomicLong tagvs_rejected = new AtomicLong();
+  private final AtomicLong tagvs_allowed = new AtomicLong();
+
+  @Override
+  public void initialize(final TSDB tsdb) {
+    final Config config = tsdb.getConfig();
+    String delimiter = config.getString("tsd.uidfilter.whitelist.delimiter");
+    if (delimiter == null) {
+      delimiter = DEFAULT_REGEX_DELIMITER;
+    }
+
+    String raw = config.getString("tsd.uidfilter.whitelist.metric_patterns");
+    if (raw != null) {
+      final String[] splits = raw.split(delimiter);
+      metric_patterns = new ArrayList<Pattern>(splits.length);
+      for (final String pattern : splits) {
+        try {
+          metric_patterns.add(Pattern.compile(pattern));
+        } catch (PatternSyntaxException e) {
+          throw new IllegalArgumentException("The metric whitelist pattern [" + 
+              pattern + "] does not compile.", e);
+        }
+      }
+    }
+
+    raw = config.getString("tsd.uidfilter.whitelist.tagk_patterns");
+    if (raw != null) {
+      final String[] splits = raw.split(delimiter);
+      tagk_patterns = new ArrayList<Pattern>(splits.length);
+      for (final String pattern : splits) {
+        try {
+          tagk_patterns.add(Pattern.compile(pattern));
+        } catch (PatternSyntaxException e) {
+          throw new IllegalArgumentException("The tagk whitelist pattern [" + 
+              pattern + "] does not compile.", e);
+        }
+      }
+    }
+
+    raw = config.getString("tsd.uidfilter.whitelist.tagv_patterns");
+    if (raw != null) {
+      final String[] splits = raw.split(delimiter);
+      tagv_patterns = new ArrayList<Pattern>(splits.length);
+      for (final String pattern : splits) {
+        try {
+          tagv_patterns.add(Pattern.compile(pattern));
+        } catch (PatternSyntaxException e) {
+          throw new IllegalArgumentException("The tagv whitelist pattern [" + 
+              pattern + "] does not compile.", e);
+        }
+      }
+    }
+  }
+
+  @Override
+  public Deferred<Object> shutdown() {
+    return Deferred.fromResult(null);
+  }
+
+  @Override
+  public String version() {
+    return "2.3.0";
+  }
+
+  @Override
+  public void collectStats(final StatsCollector collector) {
+    collector.record("uid.filter.whitelist.accepted", metrics_allowed.get(), 
+        "type=metrics");
+    collector.record("uid.filter.whitelist.accepted", tagks_allowed.get(), 
+        "type=tagk");
+    collector.record("uid.filter.whitelist.accepted", tagvs_allowed.get(), 
+        "type=tagv");
+    collector.record("uid.filter.whitelist.rejected", metrics_rejected.get(), 
+        "type=metrics");
+    collector.record("uid.filter.whitelist.rejected", tagks_rejected.get(), 
+        "type=tagk");
+    collector.record("uid.filter.whitelist.rejected", tagvs_rejected.get(), 
+        "type=tagv");
+  }
+
+  @Override
+  public Deferred<Boolean> allowUIDAssignment(
+      final UniqueIdType type, 
+      final String value,
+      final String metric, 
+      final Map<String, String> tags) {
+
+    switch (type) {
+      case METRIC:
+        if (metric_patterns != null) {
+          for (final Pattern pattern : metric_patterns) {
+            if (!pattern.matcher(value).find()) {
+              metrics_rejected.incrementAndGet();
+              return Deferred.fromResult(false);
+            }
+          }
+        }
+        metrics_allowed.incrementAndGet();
+        break;
+
+      case TAGK:
+        if (tagk_patterns != null) {
+          for (final Pattern pattern : tagk_patterns) {
+            if (!pattern.matcher(value).find()) {
+              tagks_rejected.incrementAndGet();
+              return Deferred.fromResult(false);
+            }
+          }
+        }
+        tagks_allowed.incrementAndGet();
+        break;
+
+      case TAGV:
+        if (tagv_patterns != null) {
+          for (final Pattern pattern : tagv_patterns) {
+            if (!pattern.matcher(value).find()) {
+              tagvs_rejected.incrementAndGet();
+              return Deferred.fromResult(false);
+            }
+          }
+        }
+        tagvs_allowed.incrementAndGet();
+        break;
+    }
+
+    // all patterns passed, yay!
+    return Deferred.fromResult(true);
+  }
+
+  @Override
+  public boolean fillterUIDAssignments() {
+    return true;
+  }
+
+  @VisibleForTesting
+  List<Pattern> metricPatterns() {
+    return metric_patterns;
+  }
+
+  @VisibleForTesting
+  List<Pattern> tagkPatterns() {
+    return tagk_patterns;
+  }
+
+  @VisibleForTesting
+  List<Pattern> tagvPatterns() {
+    return tagv_patterns;
+  }
+}

test/uid/TestUniqueIdWhitelistFilter.java

@@ -0,0 +1,165 @@
+// This file is part of OpenTSDB.
+// Copyright (C) 2016  The OpenTSDB Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 2.1 of the License, or (at your
+// option) any later version.  This program is distributed in the hope that it
+// will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
+// General Public License for more details.  You should have received a copy
+// of the GNU Lesser General Public License along with this program.  If not,
+// see <http://www.gnu.org/licenses/>.
+package net.opentsdb.uid;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import net.opentsdb.core.TSDB;
+import net.opentsdb.uid.UniqueId.UniqueIdType;
+import net.opentsdb.utils.Config;
+
+@RunWith(PowerMockRunner.class)
+//"Classloader hell"...  It's real.  Tell PowerMock to ignore these classes
+//because they fiddle with the class loader.  We don't test them anyway.
+@PowerMockIgnore({"javax.management.*", "javax.xml.*",
+               "ch.qos.*", "org.slf4j.*", "com.sum.*", "org.xml.*"})
+@PrepareForTest({ TSDB.class, Config.class })
+public class TestUniqueIdWhitelistFilter {
+
+  private TSDB tsdb;
+  private Config config;
+  private UniqueIdWhitelistFilter filter;
+
+  @Before
+  public void before() throws Exception {
+    tsdb = PowerMockito.mock(TSDB.class);
+    config = new Config(false);
+    when(tsdb.getConfig()).thenReturn(config);
+    filter = new UniqueIdWhitelistFilter();
+
+    config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*");
+  }
+
+  @Test
+  public void ctor() throws Exception {
+    assertNull(filter.metricPatterns());
+    assertNull(filter.tagkPatterns());
+    assertNull(filter.tagvPatterns());
+  }
+
+  @Test
+  public void initalize() throws Exception {
+    filter.initialize(tsdb);
+    assertEquals(1, filter.metricPatterns().size());
+    assertEquals(".*", filter.metricPatterns().get(0).pattern());
+    assertEquals(1, filter.tagkPatterns().size());
+    assertEquals(".*", filter.tagkPatterns().get(0).pattern());
+    assertEquals(1, filter.tagvPatterns().size());
+    assertEquals(".*", filter.tagvPatterns().get(0).pattern());
+  }
+
+  @Test
+  public void initalizeMultiplePatterns() throws Exception {
+    config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*,^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*,^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*,^test.*");
+    filter.initialize(tsdb);
+    assertEquals(2, filter.metricPatterns().size());
+    assertEquals(".*", filter.metricPatterns().get(0).pattern());
+    assertEquals("^test.*", filter.metricPatterns().get(1).pattern());
+    assertEquals(2, filter.tagkPatterns().size());
+    assertEquals(".*", filter.tagkPatterns().get(0).pattern());
+    assertEquals("^test.*", filter.tagkPatterns().get(1).pattern());
+    assertEquals(2, filter.tagvPatterns().size());
+    assertEquals(".*", filter.tagvPatterns().get(0).pattern());
+    assertEquals("^test.*", filter.tagvPatterns().get(1).pattern());
+  }
+
+  @Test
+  public void initalizeMultiplePatternsAlternateDelimiter() throws Exception {
+    config.overrideConfig("tsd.uidfilter.whitelist.delimiter", "\\|");
+    config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*|^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*|^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*|^test.*");
+    filter.initialize(tsdb);
+    assertEquals(2, filter.metricPatterns().size());
+    assertEquals(".*", filter.metricPatterns().get(0).pattern());
+    assertEquals("^test.*", filter.metricPatterns().get(1).pattern());
+    assertEquals(2, filter.tagkPatterns().size());
+    assertEquals(".*", filter.tagkPatterns().get(0).pattern());
+    assertEquals("^test.*", filter.tagkPatterns().get(1).pattern());
+    assertEquals(2, filter.tagvPatterns().size());
+    assertEquals(".*", filter.tagvPatterns().get(0).pattern());
+    assertEquals("^test.*", filter.tagvPatterns().get(1).pattern());
+  }
+
+  @Test (expected = IllegalArgumentException.class)
+  public void initalizeBadRegex() throws Exception {
+    config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", "grp[start");
+    filter.initialize(tsdb);
+  }
+
+  @Test
+  public void shutdown() throws Exception {
+    assertNull(filter.shutdown().join());
+  }
+
+  @Test
+  public void allowUIDAssignment() throws Exception {
+    config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", "^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", "^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", "^test.*");
+    filter.initialize(tsdb);
+    assertTrue(filter.allowUIDAssignment(UniqueIdType.METRIC, "test_metric", 
+        null, null).join());
+    assertFalse(filter.allowUIDAssignment(UniqueIdType.METRIC, "metric", 
+        null, null).join());
+    assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGK, "test_tagk", 
+        null, null).join());
+    assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGK, "tagk", 
+        null, null).join());
+    assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGV, "test_tagv", 
+        null, null).join());
+    assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGV, "tagv", 
+        null, null).join());
+  }
+
+  @Test
+  public void allowUIDAssignmentMultiplePaterns() throws Exception {
+    config.overrideConfig("tsd.uidfilter.whitelist.metric_patterns", ".*,^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagk_patterns", ".*,^test.*");
+    config.overrideConfig("tsd.uidfilter.whitelist.tagv_patterns", ".*,^test.*");
+    filter.initialize(tsdb);
+    assertTrue(filter.allowUIDAssignment(UniqueIdType.METRIC, "test_metric", 
+        null, null).join());
+    assertFalse(filter.allowUIDAssignment(UniqueIdType.METRIC, "metric", 
+        null, null).join());
+    assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGK, "test_tagk", 
+        null, null).join());
+    assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGK, "tagk", 
+        null, null).join());
+    assertTrue(filter.allowUIDAssignment(UniqueIdType.TAGV, "test_tagv", 
+        null, null).join());
+    assertFalse(filter.allowUIDAssignment(UniqueIdType.TAGV, "tagv", 
+        null, null).join());
+  }
+
+  @Test
+  public void fillterUIDAssignments() throws Exception {
+    assertTrue(filter.fillterUIDAssignments());
+  }
+}