feat(log4j):add rollbar support

build.sbt

@@ -69,7 +69,8 @@ libraryDependencies ++= Seq(
   "org.apache.avro" % "avro" % "1.8.2" % "provided",
   "org.apache.hive" % "hive-jdbc" % "2.3.3" % "provided" excludeAll(excludeNetty, excludeNettyAll, excludeLog4j, excludeParquet),
   "org.apache.hadoop" % "hadoop-aws" % "2.7.3" % "provided",
-  "com.amazon.deequ" % "deequ" % "1.0.4" excludeAll(excludeSpark)
+  "com.amazon.deequ" % "deequ" % "1.0.4" excludeAll(excludeSpark),
+  "com.nextdoor.rollbar" % "rollbar-log4j" % "1.0.2"
 )
 
 // Temporary fix for https://github.com/databricks/spark-redshift/issues/315#issuecomment-285294306

docker/spark/log4j.rollbar.properties

@@ -0,0 +1,5 @@
+log4j.appender.ROLLBAR=com.nextdoor.rollbar.RollbarLog4jAppender
+log4j.appender.ROLLBAR.AccessToken=${ROLLBAR_ACCESS_TOKEN}
+log4j.appender.ROLLBAR.Environment=${ROLLBAR_ENV:-production}
+log4j.appender.ROLLBAR.layout=org.apache.log4j.PatternLayout
+log4j.appender.ROLLBAR.Threshold=${ROLLBAR_ENV:-WARN}

docker/spark/scripts/init-logs-metrics.sh

@@ -15,3 +15,8 @@ if [ ! -z ${JSON_LOG} ] ; then
     echo "Setting Log type to JSON"
     eval "echo \"$(< /spark/conf/log4j.json.properties)\"" >> /spark/conf/log4j.properties
 fi
+
+if [ ! -z ${USE_ROLLBAR} ] ; then
+    echo "Appending Rollbar Logger"
+    eval "echo \"$(< /spark/conf/log4j.rollbar.properties)\"" >> /spark/conf/log4j.properties
+fi

src/main/resources/log4j.properties

@@ -16,7 +16,7 @@
 #
 
 # Set everything to be logged to the console
-log4j.rootLogger=INFO,console
+log4j.rootLogger=INFO,console,ROLLBAR
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
@@ -38,4 +38,4 @@ log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
 log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
 
 log4j.logger.com.yotpo.metorikku=INFO
-log4j.logger.org.apache.hudi=INFO
+log4j.logger.org.apache.hudi=INFO