Add the Jackson dependency for de/serializing objects with JSON forma…

…t which

will be used in the API as well as HBase data storage.
Add an src/utils directory for shared utilities that are not a part of the
OpenTSDB core behavior
Add utils/JSON that provides instantiation of a static Jackson ObjectMaper
and some helper methods
Tested standard and Maven builds
Add mygnuplot.bat that works on Windows machines with gnuplot installed
Modify GraphHandler to support Windows machines. Really simple, just had to
tweak the line endings. Tested changes on Linux and Windows, works fine
Add a Java Properties format configuration file for OpenTSDB. It's fully
backwards compatible in that you can still use the existing command line
options, which will overload defaults or config file options, but it can let
you save a config file in a certain location and use any of the TSD CLI tools
without passing extra parameters.
Sample config file is at src/opentsdb.conf
The Config class has helpers to convert values to numbers or boolean
Modify all of the code that used to reply on System properties or other means
to use the new Config class
Mark the start of 2.0.0 in configure.ac and NEWS

Signed-off-by: Chris Larsen <[email protected]>

Makefile.am

@@ -25,7 +25,7 @@ builddata_SRC := src/BuildData.java
 BUILT_SOURCES = $(builddata_SRC)
 nodist_bin_SCRIPTS = tsdb
 dist_noinst_SCRIPTS = src/create_table.sh
-dist_pkgdata_SCRIPTS = src/mygnuplot.sh
+dist_pkgdata_SCRIPTS := src/mygnuplot.sh src/mygnuplot.bat src/opentsdb.conf
 dist_noinst_DATA = pom.xml.in
 tsdb_SRC := \
 	src/core/Aggregator.java	\
@@ -76,14 +76,18 @@ tsdb_SRC := \
 	src/uid/NoSuchUniqueId.java	\
 	src/uid/NoSuchUniqueName.java	\
 	src/uid/UniqueId.java	\
-	src/uid/UniqueIdInterface.java
+	src/uid/UniqueIdInterface.java \
+	src/utils/Config.java \ 
+	src/utils/JSON.java
 
 tsdb_DEPS = \
 	$(ASYNCHBASE)	\
 	$(GUAVA)	\
 	$(LOG4J_OVER_SLF4J)	\
 	$(LOGBACK_CLASSIC)	\
 	$(LOGBACK_CORE)	\
+	$(JACKSON_CORE)	\
+	$(JACKSON_MAPPER)	\
 	$(NETTY)	\
 	$(SLF4J_API)	\
 	$(SUASYNC)	\
@@ -99,7 +103,9 @@ test_SRC := \
 	test/stats/TestHistogram.java	\
 	test/tsd/TestGraphHandler.java	\
 	test/uid/TestNoSuchUniqueId.java	\
-	test/uid/TestUniqueId.java
+	test/uid/TestUniqueId.java \
+	test/utils/TestConfig.java \ 
+	test/utils/TestJSON.java
 
 test_DEPS = \
 	$(tsdb_DEPS) \
@@ -377,6 +383,7 @@ pom.xml: pom.xml.in Makefile
 	    -e 's/@GUAVA_VERSION@/$(GUAVA_VERSION)/' \
 	    -e 's/@GWT_VERSION@/$(GWT_VERSION)/' \
 	    -e 's/@HAMCREST_VERSION@/$(HAMCREST_VERSION)/' \
+	    -e 's/@JACKSON_VERSION@/$(JACKSON_VERSION)/' \
 	    -e 's/@JAVASSIST_VERSION@/$(JAVASSIST_VERSION)/' \
 	    -e 's/@JUNIT_VERSION@/$(JUNIT_VERSION)/' \
 	    -e 's/@LOG4J_OVER_SLF4J_VERSION@/$(LOG4J_OVER_SLF4J_VERSION)/' \

NEWS

@@ -1,5 +1,12 @@
 OpenTSDB - User visible changes.
 
+* Version 2.0.0 (2013-07-?)
+
+Noteworthy changes:
+  - Configuration can be provided in a properties file
+  - New Jackson JSON helper class
+  - GnuPlot batch file for Windows compatability
+
 * Version 1.1.0 (2013-03-08) [12879d7]
 
 Noteworthy changes:

configure.ac

@@ -14,7 +14,7 @@
 # along with this library.  If not, see <http://www.gnu.org/licenses/>.
 
 # Semantic Versioning (see http://semver.org/).
-AC_INIT([opentsdb], [1.1.0], [[email protected]])
+AC_INIT([opentsdb], [2.0.0], [[email protected]])
 AC_CONFIG_AUX_DIR([build-aux])
 AM_INIT_AUTOMAKE([foreign])
 

pom.xml.in

@@ -136,6 +136,8 @@
               <target>
                 <copy file="${basedir}/src/mygnuplot.sh" todir="${basedir}/target/classes"/>
                 <chmod file="${basedir}/target/classes/mygnuplot.sh" perm="ugo+rx"/>
+                <copy file="${basedir}/src/mygnuplot.bat" todir="${basedir}/target/classes"/>
+                <chmod file="${basedir}/target/classes/mygnuplot.bat" perm="ugo+rx"/>
               </target>
             </configuration>
             <goals>
@@ -245,6 +247,18 @@
       <artifactId>guava</artifactId>
       <version>@GUAVA_VERSION@</version>
     </dependency>
+
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-core-lgpl</artifactId>
+      <version>@JACKSON_VERSION@</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-mapper-lgpl</artifactId>
+      <version>@JACKSON_VERSION@</version>
+    </dependency>
 
     <dependency>
       <groupId>io.netty</groupId>

src/core/CompactionQueue.java

@@ -32,6 +32,7 @@
 import org.hbase.async.PleaseThrottleException;
 
 import net.opentsdb.stats.StatsCollector;
+import net.opentsdb.utils.Config;
 
 /**
  * "Queue" of rows to compact.
@@ -79,7 +80,7 @@ public CompactionQueue(final TSDB tsdb) {
     super(new Cmp(tsdb));
     this.tsdb = tsdb;
     metric_width = tsdb.metrics.width();
-    if (TSDB.enable_compactions) {
+    if (tsdb.config.enable_compactions()) {
       startCompactionThread();
     }
   }
@@ -118,7 +119,7 @@ public Deferred<ArrayList<Object>> flush() {
   void collectStats(final StatsCollector collector) {
     collector.record("compaction.count", trivial_compactions, "type=trivial");
     collector.record("compaction.count", complex_compactions, "type=complex");
-    if (!TSDB.enable_compactions) {
+    if (!tsdb.config.enable_compactions()) {
       return;
     }
     // The remaining stats only make sense with compactions enabled.
@@ -412,7 +413,7 @@ private Deferred<Object> compact(final ArrayList<KeyValue> row,
         return null;              // ... Don't write back compacted.
       }
     }
-    if (!TSDB.enable_compactions) {
+    if (!tsdb.config.enable_compactions()) {
       return null;
     }
 

src/core/IncomingDataPoints.java

@@ -24,16 +24,13 @@
 import org.hbase.async.PutRequest;
 
 import net.opentsdb.stats.Histogram;
+import net.opentsdb.utils.Config;
 
 /**
  * Receives new data points and stores them in HBase.
  */
 final class IncomingDataPoints implements WritableDataPoints {
 
-  /** For auto create metrics mode, set by --auto-metric flag in TSDMain.  */
-  private static final boolean AUTO_METRIC =
-    System.getProperty("tsd.core.auto_create_metrics") != null;
-
   /** For how long to buffer edits when doing batch imports (in ms).  */
   private static final short DEFAULT_BATCH_IMPORT_BUFFER_INTERVAL = 5000;
 
@@ -121,8 +118,9 @@ static byte[] rowKeyTemplate(final TSDB tsdb,
 
     short pos = 0;
 
-    copyInRowKey(row, pos, (AUTO_METRIC ? tsdb.metrics.getOrCreateId(metric)
-                       : tsdb.metrics.getId(metric)));
+    copyInRowKey(row, pos, (tsdb.config.auto_metric() ? 
+        tsdb.metrics.getOrCreateId(metric)
+        : tsdb.metrics.getId(metric)));
     pos += metric_width;
 
     pos += Const.TIMESTAMP_BYTES;

src/core/TSDB.java

@@ -34,6 +34,7 @@
 import org.hbase.async.PutRequest;
 
 import net.opentsdb.uid.UniqueId;
+import net.opentsdb.utils.Config;
 import net.opentsdb.stats.Histogram;
 import net.opentsdb.stats.StatsCollector;
 
@@ -44,7 +45,8 @@
  * points or query the database.
  */
 public final class TSDB {
-
+  private static final Logger LOG = LoggerFactory.getLogger(TSDB.class);
+
   static final byte[] FAMILY = { 't' };
 
   private static final String METRICS_QUAL = "metrics";
@@ -54,13 +56,6 @@ public final class TSDB {
   private static final String TAG_VALUE_QUAL = "tagv";
   private static final short TAG_VALUE_WIDTH = 3;
 
-  static final boolean enable_compactions;
-  static {
-    final String compactions = System.getProperty("tsd.feature.compactions");
-    // If not set, or set to anything but "false", defaults to true.
-    enable_compactions = !"false".equals(compactions);
-  }
-
   /** Client for the HBase cluster to use.  */
   final HBaseClient client;
 
@@ -74,6 +69,9 @@ public final class TSDB {
   /** Unique IDs for the tag values. */
   final UniqueId tag_values;
 
+  /** Configuration object for all TSDB components */
+  final Config config;
+
   /**
    * Row keys that need to be compacted.
    * Whenever we write a new data point to a row, we add the row key to this
@@ -83,27 +81,60 @@ public final class TSDB {
   private final CompactionQueue compactionq;
 
   /**
-   * Constructor.
-   * @param client The HBase client to use.
-   * @param timeseries_table The name of the HBase table where time series
-   * data is stored.
-   * @param uniqueids_table The name of the HBase table where the unique IDs
-   * are stored.
+   * Constructor
+   * @param config An initialized configuration object
+   * @since 2.0
    */
-  public TSDB(final HBaseClient client,
-              final String timeseries_table,
-              final String uniqueids_table) {
-    this.client = client;
-    table = timeseries_table.getBytes();
+  public TSDB(final Config config) {
+    this.config = config;
+    this.client = new HBaseClient(
+        config.getString("tsd.storage.hbase.zk_quorum"),
+        config.getString("tsd.storage.hbase.zk_basedir"));
+    this.client.setFlushInterval(config.getShort("tsd.storage.flush_interval"));
+    table = config.getString("tsd.storage.hbase.data_table").getBytes();
+
+    final byte[] uidtable = config.getString("tsd.storage.hbase.uid_table")
+        .getBytes();
 
-    final byte[] uidtable = uniqueids_table.getBytes();
     metrics = new UniqueId(client, uidtable, METRICS_QUAL, METRICS_WIDTH);
     tag_names = new UniqueId(client, uidtable, TAG_NAME_QUAL, TAG_NAME_WIDTH);
-    tag_values = new UniqueId(client, uidtable, TAG_VALUE_QUAL,
-                              TAG_VALUE_WIDTH);
+    tag_values = new UniqueId(client, uidtable, TAG_VALUE_QUAL, TAG_VALUE_WIDTH);
     compactionq = new CompactionQueue(this);
+
+    LOG.debug(config.dumpConfiguration());
+  }
+
+  /** 
+   * Returns the configured HBase client 
+   * @return The HBase client
+   * @since 2.0 
+   */
+  public final HBaseClient getClient() {
+    return this.client;
+  }
+
+  /** 
+   * Getter that returns the configuration object
+   * @return The configuration object
+   * @since 2.0 
+   */
+  public final Config getConfig() {
+    return this.config;
   }
 
+  /**
+   * Verifies that the data and UID tables exist in HBase
+   * @return An ArrayList of objects to wait for
+   * @throws TableNotFoundException
+   * @since 2.0
+   */
+  public Deferred<ArrayList<Object>> checkNecessaryTablesExist() {
+    return Deferred.group(client.ensureTableExists(
+        config.getString("tsd.storage.hbase.data_table")),
+        client.ensureTableExists(
+            config.getString("tsd.storage.hbase.uid_table")));
+  }
+
   /** Number of cache hits during lookups involving UIDs. */
   public int uidCacheHits() {
     return (metrics.cacheHits() + tag_names.cacheHits()
@@ -364,7 +395,7 @@ public String toString() {
       }
     }
     // First flush the compaction queue, then shutdown the HBase client.
-    return enable_compactions
+    return config.enable_compactions()
       ? compactionq.flush().addCallbacks(new HClientShutdown(),
                                          new ShutdownErrback())
       : client.shutdown();
@@ -421,7 +452,7 @@ final KeyValue compact(final ArrayList<KeyValue> row) {
    * @param base_time The 32-bit unsigned UNIX timestamp.
    */
   final void scheduleForCompaction(final byte[] row, final int base_time) {
-    if (enable_compactions) {
+    if (config.enable_compactions()) {
       compactionq.add(row);
     }
   }

src/mygnuplot.bat

@@ -0,0 +1,6 @@
+set -e
+stdout=$1
+shift
+stderr=$1
+shift
+gnuplot %1 2>&1

src/opentsdb.conf

@@ -0,0 +1,60 @@
+# --------- NETWORK ----------
+# The TCP port TSD should use for communications
+# *** REQUIRED ***
+tsd.network.port =
+
+# The IPv4 network address to bind to, defaults to all addresses
+# tsd.network.bind = 0.0.0.0
+
+# Enables Nagel's algorithm to reduce the number of packets sent over the
+# network, default is True
+#tsd.network.tcpnodelay = true
+
+# Determines whether or not to send keepalive packets to peers, default 
+# is True
+#tsd.network.keepalive = true
+
+# Determines if the same socket should be used for new connections, default 
+# is True
+#tsd.network.reuseaddress = true
+
+# Number of worker threads dedicated to Netty, defaults to # of CPUs * 2
+#tsd.network.worker_threads = 8
+
+# Whether or not to use NIO or tradditional blocking IO, defaults to True
+#tsd.network.async_io = true
+
+# ----------- HTTP -----------
+# The location of static files for the HTTP GUI interface.
+# *** REQUIRED ***
+tsd.http.staticroot =
+
+# Where TSD should write it's cache files to
+# *** REQUIRED ***
+tsd.http.cachedir =
+
+# --------- CORE ----------
+# Whether or not to automatically create UIDs for new metric types, default
+# is False
+#tsd.core.auto_create_metrics = false
+
+# --------- STORAGE ----------
+# Whether or not to enable data compaction in HBase, default is True
+#tsd.storage.enable_compaction = true
+
+# How often, in milliseconds, to flush the data point queue to storage, 
+# default is 1,000
+# tsd.storage.flush_interval = 1000
+
+# Name of the HBase table where data points are stored, default is "tsdb"
+#tsd.storage.hbase.data_table = tsdb
+
+# Name of the HBase table where UID information is stored, default is "tsdb-uid"
+#tsd.storage.hbase.uid_table = tsdb-uid
+
+# Path under which the znode for the -ROOT- region is located, default is "/hbase"
+#tsd.storage.hbase.zk_basedir = /hbase
+
+# A space separated list of Zookeeper hosts to connect to, with or without 
+# port specifiers, default is "localhost"
+#tsd.storage.hbase.zk_quorum = localhost

src/tools/ArgP.java

@@ -248,6 +248,11 @@ public void addUsageTo(final StringBuilder buf) {
     }
   }
 
+  /** Returns a the parsed options and values */
+  public HashMap<String, String> getParsed() {
+    return this.parsed;
+  }
+
   /**
    * Returns a usage string.
    */