[SW-2146] Cleanup in tests -> move to right ai.h2o.sparkling packages (#2012)

Author: jakubhava

build.gradle

@@ -47,7 +47,11 @@ ext {
     project(':sparkling-water-extensions')
   ]
   // Projects with integration tests
-  integTestProjects = [project(':sparkling-water-core'), project(':sparkling-water-examples')]
+  integTestProjects = [
+    project(':sparkling-water-core'),
+    project(':sparkling-water-examples'),
+    project(':sparkling-water-ml')
+  ]
 
   // Projects with benchmarks
   benchProjects = [project(':sparkling-water-core'), project(':sparkling-water-benchmarks')]

core/build.gradle

@@ -97,10 +97,6 @@ dependencies {
 
   benchImplementation("org.scalatest:scalatest_${scalaBaseVersion}:${scalaTestVersion}")
   benchImplementation("junit:junit:4.11")
-
-  // Put Spark Assembly on runtime path
-  integTestRuntimeOnly(fileTree(dir: new File((String) sparkHome, "lib/"), include: '*.jar'))
-  benchRuntimeOnly(fileTree(dir: new File((String) sparkHome, "lib/"), include: '*.jar'))
 }
 
 task createSparkVersionFile {

core/src/test/scala/org/apache/spark/h2o/utils/BenchUtils.scala renamed to core/src/bench/scala/ai/h2o/sparkling/bench/BenchResult.scala

@@ -15,52 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.spark.h2o.utils
-
-import java.util.concurrent.TimeUnit
+package ai.h2o.sparkling.bench
 
 import scala.concurrent.duration.TimeUnit
 
-object BenchUtils {
-
-  /**
-    * Measure execution time of given block in nanoseconds.
-    *
-    * @param block block to measure
-    * @return number of ns to execute given block
-    */
-  def timer(block: => Unit): Long = {
-    val now = System.nanoTime()
-    block
-    System.nanoTime() - now
-  }
-
-  /**
-    * Benchmark given block of code.
-    *
-    * @param iterations number of iterations to execute the block of code
-    * @param block      block to execute as benchmark
-    * @return
-    */
-  def bench(iterations: Int, warmup: Int = 4, outputTimeUnit: TimeUnit = TimeUnit.MILLISECONDS)(
-      block: => Unit): BenchResult = {
-    val times = new Array[Long](iterations)
-    // Warmup
-    for (i <- 0 until warmup) {
-      timer(block)
-    }
-    // Measure
-    for (i <- 0 until iterations) {
-      times(i) = timer(block)
-    }
-
-    BenchResult(times, TimeUnit.NANOSECONDS, outputTimeUnit)
-  }
-}
-
 case class BenchResult(mean: Float, stdDev: Float, min: Float, max: Float, unit: TimeUnit) {
   def show(): String = {
-    f"${mean}%4f ± ${stdDev}%4f (${min}%4f, ${max}%4f)"
+    f"$mean%4f ± $stdDev%4f ($min%4f, $max%4f)"
   }
 }
 
@@ -69,7 +30,7 @@ object BenchResult {
     val convMeasurements = measurements.map(x => outputUnit.convert(x, inputUnit))
     val mean = convMeasurements.sum.toFloat / convMeasurements.length
     val stdev =
-      (Math.sqrt(convMeasurements.map(x => (x - mean) * (x - mean)).sum / (convMeasurements.length - 1))).toFloat
+      Math.sqrt(convMeasurements.map(x => (x - mean) * (x - mean)).sum / (convMeasurements.length - 1)).toFloat
     new BenchResult(mean, stdev, convMeasurements.min, convMeasurements.max, outputUnit)
   }
 }

core/src/bench/scala/ai/h2o/sparkling/bench/BenchSuite.scala

@@ -19,23 +19,57 @@ package ai.h2o.sparkling.bench
 
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.h2o.utils.BenchUtils.bench
 import org.scalatest.FunSuite
 
+import scala.concurrent.duration.TimeUnit
+
 class BenchSuite extends FunSuite {
 
   protected def benchTest(
       testName: String,
       iterations: Int = 5,
       warmUp: Int = 1,
       outputTimeUnit: TimeUnit = TimeUnit.MILLISECONDS)(testFun: => Unit): Unit = {
-    def body: Unit = {
+    def body(): Unit = {
       val result = bench(iterations, warmUp, outputTimeUnit) {
-        val evaluated = testFun
+        testFun
       }
       println(s"$testName: ${result.show()}")
     }
+    registerTest(testName)(body())
+  }
+
+  /**
+    * Measure execution time of given block in nanoseconds.
+    *
+    * @param block block to measure
+    * @return number of ns to execute given block
+    */
+  private def timer(block: => Unit): Long = {
+    val now = System.nanoTime()
+    block
+    System.nanoTime() - now
+  }
+
+  /**
+    * Benchmark given block of code.
+    *
+    * @param iterations number of iterations to execute the block of code
+    * @param block      block to execute as benchmark
+    * @return
+    */
+  private def bench(iterations: Int, warmup: Int = 4, outputTimeUnit: TimeUnit = TimeUnit.MILLISECONDS)(
+      block: => Unit): BenchResult = {
+    val times = new Array[Long](iterations)
+    // Warmup
+    for (_ <- 0 until warmup) {
+      timer(block)
+    }
+    // Measure
+    for (i <- 0 until iterations) {
+      times(i) = timer(block)
+    }
 
-    registerTest(testName)(body)
+    BenchResult(times, TimeUnit.NANOSECONDS, outputTimeUnit)
   }
 }

core/src/bench/scala/ai/h2o/sparkling/bench/DataFrameConverterBenchSuite.scala

@@ -17,11 +17,11 @@
 
 package ai.h2o.sparkling.bench
 
-import ai.h2o.sparkling.ml.utils.{FlatArraysOnlySchema, FlatSchema, SchemaUtils, StructsOnlySchema}
-import org.apache.spark.SparkContext
-import org.apache.spark.h2o.testdata.{DenseVectorHolder, SparseVectorHolder}
-import org.apache.spark.h2o.utils.{SharedH2OTestContext, TestFrameUtils}
+import ai.h2o.sparkling.TestUtils.{DenseVectorHolder, SparseVectorHolder}
+import ai.h2o.sparkling.ml.utils.SchemaUtils
+import ai.h2o.sparkling.{SharedH2OTestContext, TestUtils}
 import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vectors}
+import org.apache.spark.sql.SparkSession
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner
 
@@ -30,10 +30,10 @@ import scala.util.Random
 @RunWith(classOf[JUnitRunner])
 class DataFrameConverterBenchSuite extends BenchSuite with SharedH2OTestContext {
 
-  override def createSparkContext =
-    new SparkContext("local-cluster[2, 1, 2048]", getClass.getSimpleName, defaultSparkConf)
+  override def createSparkSession(): SparkSession = sparkSession("local-cluster[2, 1, 2048]")
+  import spark.implicits._
 
-  private val settings = TestFrameUtils.GenerateDataFrameSettings(
+  private val settings = TestUtils.GenerateDataFrameSettings(
     numberOfRows = 8000,
     rowsPerPartition = 500,
     maxCollectionSize = 100,
@@ -75,29 +75,28 @@ class DataFrameConverterBenchSuite extends BenchSuite with SharedH2OTestContext
     rowToSchema(FlatArraysOnlySchema)
   }
 
-  private def testPerSchema(schemaHolder: TestFrameUtils.SchemaHolder): Unit = {
-    val df = TestFrameUtils.generateDataFrame(spark, schemaHolder, settings)
+  private def testPerSchema(schemaHolder: TestUtils.SchemaHolder): Unit = {
+    val df = TestUtils.generateDataFrame(spark, schemaHolder, settings)
     val hf = hc.asH2OFrame(df)
     hf.remove()
   }
 
-  private def testflattenOnlyPerSchema(schemaHolder: TestFrameUtils.SchemaHolder): Unit = {
-    val df = TestFrameUtils.generateDataFrame(spark, schemaHolder, settings)
+  private def testflattenOnlyPerSchema(schemaHolder: TestUtils.SchemaHolder): Unit = {
+    val df = TestUtils.generateDataFrame(spark, schemaHolder, settings)
     SchemaUtils.flattenDataFrame(df).foreach(_ => {})
   }
 
-  private def testflattenSchema(schemaHolder: TestFrameUtils.SchemaHolder): Unit = {
-    val df = TestFrameUtils.generateDataFrame(spark, schemaHolder, settings)
+  private def testflattenSchema(schemaHolder: TestUtils.SchemaHolder): Unit = {
+    val df = TestUtils.generateDataFrame(spark, schemaHolder, settings)
     SchemaUtils.flattenSchema(df)
   }
 
-  private def rowToSchema(schemaHolder: TestFrameUtils.SchemaHolder): Unit = {
-    val df = TestFrameUtils.generateDataFrame(spark, schemaHolder, settings)
+  private def rowToSchema(schemaHolder: TestUtils.SchemaHolder): Unit = {
+    val df = TestUtils.generateDataFrame(spark, schemaHolder, settings)
     SchemaUtils.rowsToRowSchemas(df).foreach(_ => {})
   }
 
   benchTest("Measure performance of conversion to H2OFrame on a data frame with wide sparse vectors") {
-    import sqlContext.implicits._
     val numberOfCols = 50 * 1000
     val sparsity = 0.2
     val numberOfRows = 3 * 1000
@@ -112,7 +111,6 @@ class DataFrameConverterBenchSuite extends BenchSuite with SharedH2OTestContext
   }
 
   benchTest("Measure performance of conversion to H2OFrame on a data frame with wide dense vectors") {
-    import sqlContext.implicits._
     val numberOfCols = 10 * 1000
     val numberOfRows = 3 * 1000
     val partitions = 4
@@ -128,7 +126,6 @@ class DataFrameConverterBenchSuite extends BenchSuite with SharedH2OTestContext
   benchTest(
     "Measure performance of conversion to H2OFrame on a matrix 10x11 represented by sparse vectors",
     iterations = 10) {
-    import sqlContext.implicits._
 
     val numberOfRows = 10
     val numberOfCols = 11

core/src/test/scala/ai/h2o/sparkling/ml/utils/FlatArraysOnlySchema.scala renamed to core/src/bench/scala/ai/h2o/sparkling/bench/FlatArraysOnlySchema.scala

@@ -15,12 +15,12 @@
  * limitations under the License.
  */
 
-package ai.h2o.sparkling.ml.utils
+package ai.h2o.sparkling.bench
 
-import org.apache.spark.h2o.utils.TestFrameUtils
+import ai.h2o.sparkling.TestUtils
 import org.apache.spark.sql.types._
 
-case object FlatArraysOnlySchema extends TestFrameUtils.SchemaHolder {
+case object FlatArraysOnlySchema extends TestUtils.SchemaHolder {
   @transient lazy val schema: StructType = StructType(
     Seq(
       StructField("field_GHEYZJXM36Y", ArrayType(LongType)),

core/src/test/scala/ai/h2o/sparkling/ml/utils/FlatSchema.scala renamed to core/src/bench/scala/ai/h2o/sparkling/bench/FlatSchema.scala

@@ -15,12 +15,12 @@
  * limitations under the License.
  */
 
-package ai.h2o.sparkling.ml.utils
+package ai.h2o.sparkling.bench
 
-import org.apache.spark.h2o.utils.TestFrameUtils
+import ai.h2o.sparkling.TestUtils
 import org.apache.spark.sql.types._
 
-case object FlatSchema extends TestFrameUtils.SchemaHolder {
+case object FlatSchema extends TestUtils.SchemaHolder {
   @transient lazy val schema: StructType = StructType(
     Seq(
       StructField("field_9AHKOGTE4", LongType),

core/src/test/scala/ai/h2o/sparkling/ml/utils/StructsOnlySchema.scala renamed to core/src/bench/scala/ai/h2o/sparkling/bench/StructsOnlySchema.scala

@@ -15,12 +15,12 @@
  * limitations under the License.
  */
 
-package ai.h2o.sparkling.ml.utils
+package ai.h2o.sparkling.bench
 
-import org.apache.spark.h2o.utils.TestFrameUtils
+import ai.h2o.sparkling.TestUtils
 import org.apache.spark.sql.types._
 
-case object StructsOnlySchema extends TestFrameUtils.SchemaHolder {
+case object StructsOnlySchema extends TestUtils.SchemaHolder {
   @transient lazy val schema: StructType = {
     StructType(
       Seq(