Skip to content

Commit

Permalink
Add some really ugly makefile code to compile the expression parser
Browse files Browse the repository at this point in the history
source files and add the source and classes to the TSDB jar.
TODO - clean this up! It's not pretty but it seems to function
and it doesn't cleanup after itself yet.

Signed-off-by: Chris Larsen <[email protected]>
  • Loading branch information
manolama committed Nov 3, 2015
1 parent acfe754 commit 8c74b89
Show file tree
Hide file tree
Showing 6 changed files with 113 additions and 8 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,4 @@ guava-rpm-maker/\.project
src-main
src-test
plugin_test.jar
/bin/
bin/
26 changes: 19 additions & 7 deletions Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ tsdb_DEPS = \
$(JACKSON_ANNOTATIONS) \
$(JACKSON_CORE) \
$(JACKSON_DATABIND) \
$(JAVACC) \
$(NETTY) \
$(SLF4J_API) \
$(SUASYNC) \
Expand All @@ -194,7 +195,7 @@ else
tsdb_DEPS += \
$(ASYNCHBASE) \
$(PROTOBUF) \
$(ZOOKEEPER)
$(ZOOKEEPER)
maven_profile_bigtable := false
maven_profile_hbase := true
maven_profile_cassandra := false
Expand Down Expand Up @@ -329,11 +330,11 @@ test_DEPS = \
$(tsdb_DEPS) \
$(JAVASSIST) \
$(JUNIT) \
$(HAMCREST) \
$(HAMCREST) \
$(MOCKITO) \
$(OBJENESIS) \
$(OBJENESIS) \
$(POWERMOCK_MOCKITO) \
$(jar)
$(jar)

httpui_SRC := \
src/tsd/client/DateTimeBox.java \
Expand All @@ -347,6 +348,11 @@ httpui_SRC := \

httpui_DEPS = src/tsd/QueryUi.gwt.xml

# TODO(CL) - There is likely a MUCH better way to compile and add the expression sources and jars.
expr_package = net/opentsdb/query/expression/parser
expr_src_dir = $(abs_builddir)/src/$(expr_package)
get_expr_classes = `classes=''; for f in $(packagedir)$(expr_package)/*.class; do classes="$$classes $$f"; done; echo $$classes;`

#dist_pkgdata_DATA = src/logback.xml
dist_static_DATA = src/tsd/static/favicon.ico

Expand Down Expand Up @@ -405,7 +411,7 @@ install-exec-hook:
$(builddata_SRC): .git/HEAD $(tsdb_SRC) $(top_srcdir)/build-aux/gen_build_data.sh
$(srcdir)/build-aux/gen_build_data.sh $(builddata_SRC) $(package).$(builddata_subpackage) $(PACKAGE_VERSION)

jar: $(jar) .javac-unittests-stamp .gwtc-stamp
jar: runjavacc $(jar) .javac-unittests-stamp .gwtc-stamp

JAVA_COMPILE := $(JAVAC) $(AM_JAVACFLAGS) -d .

Expand All @@ -420,6 +426,9 @@ filter_src = \
src="$$src $$i";; \
esac; \
done; \
for f in $(expr_src_dir)/*.java; do \
src="$$src $$f"; \
done; \
test -n "$$src" || exit 0
# Touches all the targets if any of the dependencies are newer.
# This is useful to force-recompile all files if one of the
Expand All @@ -438,7 +447,7 @@ find_jar = test -f "$$jar" && echo "$$jar" || echo "$(srcdir)/$$jar"
get_dep_classpath = `for jar in $(tsdb_DEPS); do $(find_jar); done | tr '\n' ':'`
.javac-stamp: $(tsdb_SRC) $(builddata_SRC)
@$(filter_src); cp=$(get_dep_classpath); \
echo "$(JAVA_COMPILE) -cp $$cp $$src"; \
echo "DO THA COMPILE!!! $(JAVA_COMPILE) -cp $$cp $$src"; \
$(JAVA_COMPILE) -cp $$cp $$src
@touch "$@"

Expand Down Expand Up @@ -653,7 +662,7 @@ manifest: .javac-stamp .git/HEAD
echo "Implementation-Vendor: $(spec_vendor)"; } >"$@"

$(jar): manifest .javac-stamp $(classes)
$(JAR) cfm `basename $(jar)` manifest $(classes_with_nested_classes) \
$(JAR) cfm `basename $(jar)` manifest $(classes_with_nested_classes) $(get_expr_classes) \
|| { rv=$$? && rm -f `basename $(jar)` && exit $$rv; }
# ^^^^^^^^^^^^^^^^^^^^^^^
# I've seen cases where `jar' exits with an error but leaves a partially built .jar file!
Expand All @@ -677,6 +686,9 @@ $(JAVADOC_DIR)/index.html: $(tsdb_SRC)
-link $(JDK_JAVADOC) -link $(NETTY_JAVADOC) -link $(SUASYNC_JAVADOC) \
$? $(builddata_SRC)

runjavacc:
$(JAVA) -cp $(JAVACC) javacc -STATIC:false -LOOKAHEAD:5 -OUTPUT_DIRECTORY:$(expr_src_dir) $(abs_srcdir)/src/parser.jj; echo PWD: `pwd`;

dist-hook:
$(mkdir_p) $(distdir)/.git
echo $(git_version) >$(distdir)/.git/HEAD
Expand Down
68 changes: 68 additions & 0 deletions src/parser.jj
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
PARSER_BEGIN(SyntaxChecker)
package net.opentsdb.query.expression.parser;

import net.opentsdb.query.expression.ExpressionTree;
import net.opentsdb.core.TSQuery;

import java.util.List;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;

/**
* A simple class for validating the expressions
* @since 2.3
*/
public class SyntaxChecker {

private TSQuery data_query;
private List<String> metricQueries;

public void setTSQuery(TSQuery data_query) {
this.data_query = data_query;
}

public void setMetricQueries(List<String> metricQueries) {
this.metricQueries = metricQueries;
}

public static void main(String[] args) {
try {
new SyntaxChecker(new java.io.StringReader(args[0])).EXPRESSION();
System.out.println("Syntax is okay");
} catch (Throwable e) {
// Catching Throwable is ugly but JavaCC throws Error objects!
System.out.println("Syntax check failed: " + e.getMessage());
}
}
}

PARSER_END(SyntaxChecker)

SKIP: { " " | "\t" | "\n" | "\r" }
TOKEN: { <NAME: ("*" | (["a" - "z"] | ["A" - "Z"]| ["0"-"9"] | "-" | "_" | "#" | "/" | "$" | "@" | "|" | "." | "'" | "]" | "[")+ )> }
TOKEN: { <PARAM: ("&&")> }

ExpressionTree EXPRESSION(): {Token name; int paramIndex=0;} {
name=<NAME> { ExpressionTree tree=new ExpressionTree(name.image,data_query); }
"(" PARAMETER(tree, paramIndex++) ("," PARAMETER(tree, paramIndex++))* ")"
{return tree;}
}

void PARAMETER(ExpressionTree tree, int paramIndex): {String metric; Token param; ExpressionTree subTree;} {
subTree=EXPRESSION() {tree.addSubExpression(subTree, paramIndex);} |
metric=METRIC() {metricQueries.add(metric); tree.addSubMetricQuery(metric, metricQueries.size()-1, paramIndex);} |
param=<NAME> {tree.addFunctionParameter(param.image);}
}

// metric is agg:[interval-agg:][rate:]metric[{tag=value,...}]
String METRIC() : {Token agg,itvl,rate,metric,tagk,tagv; StringBuilder builder = new StringBuilder();
Joiner JOINER = Joiner.on(",").skipNulls();
List<String> tagPairs = Lists.newArrayList();
} {
agg = <NAME> ":" { builder.append(agg.image).append(":"); }
(itvl=<NAME> ":" { builder.append(itvl.image).append(":"); })?
(rate=<NAME> ":" { builder.append(rate.image).append(":"); })?
metric=<NAME> { builder.append(metric.image); }
("{" tagk=<NAME> "=" tagv=<NAME> {tagPairs.add(tagk+"="+tagv);}
("," tagk=<NAME> "=" tagv=<NAME> {tagPairs.add(tagk+"="+tagv);})* "}")?
{if (tagPairs.size() > 0) builder.append("{").append(JOINER.join(tagPairs)).append("}"); return builder.toString();} }
1 change: 1 addition & 0 deletions third_party/include.mk
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ include third_party/guava/include.mk
include third_party/gwt/include.mk
include third_party/hamcrest/include.mk
include third_party/jackson/include.mk
include third_party/javacc/include.mk
include third_party/javassist/include.mk
include third_party/junit/include.mk
include third_party/logback/include.mk
Expand Down
23 changes: 23 additions & 0 deletions third_party/javacc/include.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Copyright (C) 2015 The OpenTSDB Authors.
#
# This library is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 2.1 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library. If not, see <http://www.gnu.org/licenses/>.

JAVACC_VERSION := 6.1.2
JAVACC := third_party/javacc/javacc-$(JAVACC_VERSION).jar
JAVACC_BASE_URL := http://central.maven.org/maven2/net/java/dev/javacc/javacc/$(JAVACC_VERSION)

$(JAVACC): $(JAVACC).md5
set dummy "$(JAVACC_BASE_URL)" "$(JAVACC)"; shift; $(FETCH_DEPENDENCY)

THIRD_PARTY += $(JAVACC)
1 change: 1 addition & 0 deletions third_party/javacc/javacc-6.1.2.jar.md5
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c74b2df75b4c46209d6da22bf4dad976

0 comments on commit 8c74b89

Please sign in to comment.