[SW-2636] A workaround for Scala compiler api breaking change (#2664)

* [SW-2636] A workaround for Scala compiler api breaking change

Author: krasinski

gradle/scala.gradle

@@ -15,11 +15,21 @@ configurations {
     scalaCompilerPlugin
 }
 
+//forcing the scala version but not messing with the incremental compiler because it will not accept it
+configurations
+        .findAll { conf -> !conf.name.contains("scalaCompilerPlugin") && conf.name != "zinc" }
+        .each {conf ->
+            conf.resolutionStrategy {
+                force "org.scala-lang:scala-compiler:${scalaVersion}"
+                force "org.scala-lang:scala-library:${scalaVersion}"
+                force "org.scala-lang:scala-reflect:${scalaVersion}"
+            }
+        }
+
 dependencies {
     scalaCompilerPlugin "org.scalamacros:paradise_${scalaVersion}:2.1.1"
 }
 
-
 // Activate Zinc compiler and configure scalac
 tasks.withType(ScalaCompile) {
     scalaCompileOptions.additionalParameters = [
@@ -66,7 +76,7 @@ task scaladocJar(type: Jar, dependsOn: scaladoc) {
     from scaladoc
 }
 
-task javadocJar (type: Jar, dependsOn: scaladoc) {
+task javadocJar(type: Jar, dependsOn: scaladoc) {
     archiveClassifier = 'javadoc'
     from scaladoc
 }

repl/build.gradle

@@ -10,6 +10,8 @@ dependencies {
   testImplementation("org.apache.spark:spark-repl_${scalaBaseVersion}:${sparkVersion}")
   testImplementation("org.scalatest:scalatest_${scalaBaseVersion}:${scalaTestVersion}")
   testImplementation("junit:junit:4.11")
+  testImplementation(project(":sparkling-water-core"))
+  testImplementation(project(path: ':sparkling-water-core', configuration: 'testArchives'))
 }
 
 defineStandardPublication().call()

repl/src/main/scala/ai/h2o/sparkling/repl/BaseH2OInterpreter.scala

@@ -29,6 +29,7 @@ import scala.Predef.{println => _}
 import scala.annotation.tailrec
 import scala.collection.mutable
 import scala.language.{existentials, implicitConversions, postfixOps}
+import scala.reflect.classTag
 import scala.tools.nsc._
 import scala.tools.nsc.interpreter.{Results => IR, _}
 
@@ -110,7 +111,7 @@ private[repl] abstract class BaseH2OInterpreter(val sparkContext: SparkContext,
   }
 
   private def initializeInterpreter(): Unit = {
-    settings = createSettings()
+    settings = createSettings(classTag[BaseH2OInterpreter].runtimeClass.getClassLoader)
     intp = createInterpreter()
     val spark = SparkSessionUtils.active
     addThunk(intp.beQuietDuring {
@@ -142,7 +143,7 @@ private[repl] abstract class BaseH2OInterpreter(val sparkContext: SparkContext,
   /**
     * Initialize the compiler settings
     */
-  protected def createSettings(): Settings
+  protected def createSettings(classLoader: ClassLoader): Settings
 
   /**
     * Run all thunks after the interpreter has been initialized and throw exception if anything went wrong

repl/src/main/scala/ai/h2o/sparkling/repl/H2OInterpreter.scala

@@ -21,20 +21,19 @@
   */
 package ai.h2o.sparkling.repl
 
-import java.io.File
-import java.net.URI
-
 import org.apache.spark.{SparkConf, SparkContext}
 
+import java.io.File
+import java.net.URI
+import scala.io.Source
 import scala.language.{existentials, implicitConversions, postfixOps}
-import scala.reflect._
 import scala.tools.nsc._
 
 /**
   * H2O Interpreter which is use to interpret scala code
   *
   * @param sparkContext spark context
-  * @param hc H2OContext
+  * @param hc           H2OContext
   * @param sessionId    session ID for interpreter
   */
 class H2OInterpreter(sparkContext: SparkContext, hc: Any, sessionId: Int)
@@ -44,51 +43,47 @@ class H2OInterpreter(sparkContext: SparkContext, hc: Any, sessionId: Int)
     H2OIMain.createInterpreter(sparkContext, settings, responseWriter, sessionId)
   }
 
-  override def createSettings(): Settings = {
-    val settings = new Settings(echo)
-    // prevent each repl line from being run in a new thread
-    settings.Yreplsync.value = true
+  override def createSettings(classLoader: ClassLoader): Settings = {
+    val result = new Settings(echo)
 
     // Check if app.class.path resource on given classloader is set. In case it exists, set it as classpath
     // ( instead of using java class path right away)
     // This solves problem explained here: https://gist.github.com/harrah/404272
-    settings.usejavacp.value = true
-    val loader = classTag[H2OInterpreter].runtimeClass.getClassLoader
-    val method =
-      settings.getClass.getSuperclass.getDeclaredMethod("getClasspath", classOf[String], classOf[ClassLoader])
-    method.setAccessible(true)
-    if (method.invoke(settings, "app", loader).asInstanceOf[Option[String]].isDefined) {
-      settings.usejavacp.value = false
-      settings.embeddedDefaults(loader)
-    }
-
-    val conf = sparkContext.getConf
-    val jars = getJarsForShell(conf)
+    val appClassPath = getClassLoaderAppClassPath(classLoader)
+    val useJavaCpArg = if (appClassPath.isEmpty) Some("-usejavacp") else None
+    val classPathArg =
+      appClassPath
+        .orElse(getReplLocalJars(sparkContext.getConf))
+        .map(classPathValue => List[String]("-classpath", classPathValue))
 
     val interpArguments = List(
+      "-Yrepl-sync", // prevent each repl line from being run in a new thread
       "-Yrepl-class-based", // ensure that lines in REPL are wrapped in the classes instead of objects
       "-Yrepl-outdir",
-      s"${H2OInterpreter.classOutputDirectory.getAbsolutePath}",
-      "-classpath",
-      jars)
+      s"${H2OInterpreter.classOutputDirectory.getAbsolutePath}") ++ useJavaCpArg ++ classPathArg.getOrElse(List.empty)
 
-    settings.processArguments(interpArguments, processAll = true)
+    result.processArguments(interpArguments, processAll = true)
 
-    settings
+    result
   }
 
-  private def getJarsForShell(conf: SparkConf): String = {
+  private def getClassLoaderAppClassPath(runtimeClassLoader: ClassLoader): Option[String] =
+    for {
+      classLoader <- Option(runtimeClassLoader)
+      appClassPathResource <- Option(classLoader.getResource("app.class.path"))
+      appClassPath = Source.fromURL(appClassPathResource)
+    } yield appClassPath.mkString
+
+  private def getReplLocalJars(conf: SparkConf): Option[String] = {
     val localJars = conf.getOption("spark.repl.local.jars")
-    val jarPaths = localJars.getOrElse("").split(",")
-    jarPaths
-      .map { path =>
-        // Remove file:///, file:// or file:/ scheme if exists for each jar
-        if (path.startsWith("file:")) new File(new URI(path)).getPath else path
-      }
-      .mkString(File.pathSeparator)
+    val jarPaths = localJars.map(_.split(",").toSeq)
+    jarPaths.map(_.map { path =>
+      // Remove file:///, file:// or file:/ scheme if exists for each jar
+      if (path.startsWith("file:")) new File(new URI(path)).getPath else path
+    }.mkString(File.pathSeparator))
   }
 }
 
 object H2OInterpreter {
-  def classOutputDirectory = H2OIMain.classOutputDirectory
+  def classOutputDirectory: File = H2OIMain.classOutputDirectory
 }

repl/src/test/scala/ai/h2o/sparkling/repl/H2OInterpreterTestSuite.scala

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.h2o.sparkling.repl
+
+import ai.h2o.sparkling.SharedH2OTestContext
+import org.apache.spark.sql.SparkSession
+import org.junit.runner.RunWith
+import org.scalatest._
+import org.scalatest.junit.JUnitRunner
+
+import java.net.URL
+import java.nio.file.Files
+
+@RunWith(classOf[JUnitRunner])
+class H2OInterpreterTestSuite extends FunSuite with SharedH2OTestContext with Matchers {
+
+  private val replLocalJarsPrefix = "spark-repl-local-jars"
+
+  override def createSparkSession(): SparkSession = {
+    val tempReplJarsDir = tempDirNotEmpty(replLocalJarsPrefix).toUri.toString
+    val conf = defaultSparkConf.set("spark.repl.local.jars", tempReplJarsDir)
+    sparkSession("local[*]", conf)
+  }
+
+  private var objectUnderTest: H2OInterpreter = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    objectUnderTest = new H2OInterpreter(sc, hc, 1)
+  }
+
+  test("should set correct repl related settings") {
+    val result = objectUnderTest.createSettings(classLoaderStub(hasAppClassPath = false))
+    result.Yreplsync.value shouldBe true
+    result.Yreplclassbased.value shouldBe true
+    result.Yreploutdir.value shouldBe H2OInterpreter.classOutputDirectory.getAbsolutePath
+  }
+
+  test("should use spark.repl.local.jars if available and app.class.path is not set") {
+    val result = objectUnderTest.createSettings(classLoaderStub(hasAppClassPath = false))
+    result.usejavacp.value shouldBe true
+    result.classpath.value should include(replLocalJarsPrefix)
+  }
+
+  test("should not set javacp if app.class.path is set") {
+    val tmpFilePrefix = "app-class-path"
+    val result = objectUnderTest.createSettings(
+      classLoaderStub(hasAppClassPath = true, tempDirNotEmpty(tmpFilePrefix).toUri.toURL))
+    result.usejavacp.value shouldBe false
+    result.classpath.value should startWith(tmpFilePrefix)
+  }
+
+  private def classLoaderStub(hasAppClassPath: Boolean, classPathUrl: URL = null) = {
+    new ClassLoader() {
+      override def getResource(name: String): URL =
+        if (name == "app.class.path" && hasAppClassPath) classPathUrl else null
+    }
+  }
+
+  private def tempDirNotEmpty(filePrefix: String) = {
+    val tempDir = Files.createTempDirectory(filePrefix)
+    Files.createTempFile(tempDir, filePrefix, "-file")
+    tempDir
+  }
+}

repl/src/test/scala/ai/h2o/sparkling/repl/PatchUtilsTestSuite.scala

@@ -47,10 +47,6 @@ class PatchUtilsTestSuite extends FunSuite with BeforeAndAfterAll {
   }
 
   test("[SW-386] Test patched OuterScopes") {
-    val regexBeforePatch = getRegexp
-    // Default regexp fails for our class names with intp_id prefix
-    assertMatch(regexBeforePatch, EXAMPLE_CLASS_NAME, FAILED_MATCH)
-
     PatchUtils.PatchManager.patch("SW-386", Thread.currentThread().getContextClassLoader)
 
     val regexAfterPatch = getRegexp