Importing (some of) Standard.Base works from NI runner (#9866)

.vscode/launch.json

@@ -27,6 +27,13 @@
  "command": "npm run dev",
  // "env": {"NODE_OPTIONS": "--inspect"},
  "cwd": "${workspaceFolder}/app/gui2"
+ },
+ {
+ "type": "nativeimage",
+ "request": "launch",
+ "name": "Launch Native Image",
+ "nativeImagePath": "${workspaceFolder}/runner",
+ "args": "--help"
  }
  ]
 }

build.sbt

@@ -18,6 +18,7 @@ import src.main.scala.licenses.{
 import JPMSPlugin.autoImport._
 
 import java.io.File
+import java.nio.file.Paths
 
 // ============================================================================
 // === Global Configuration ===================================================
@@ -283,6 +284,7 @@ lazy val enso = (project in file("."))
  `syntax-definition`,
  `syntax-rust-definition`,
  `text-buffer`,
+ yaml,
  pkg,
  cli,
  `task-progress-notifications`,
@@ -693,6 +695,15 @@ lazy val `syntax-rust-definition` = project
  Compile / javaSource := baseDirectory.value / "generate-java" / "java"
  )
 
+lazy val yaml = (project in file("lib/java/yaml"))
+ .settings(
+ frgaalJavaCompilerSetting,
+ version := "0.1",
+ libraryDependencies ++= Seq(
+ "io.circe" %% "circe-yaml" % circeYamlVersion % "provided"
+ )
+ )
+
 lazy val pkg = (project in file("lib/scala/pkg"))
  .settings(
  Compile / run / mainClass := Some("org.enso.pkg.Main"),
@@ -1021,6 +1032,8 @@ lazy val `project-manager` = (project in file("lib/scala/project-manager"))
  Test / javaOptions ++= testLogProviderOptions
  )
  .settings(
+ NativeImage.smallJdk := None,
+ NativeImage.additionalCp := Seq.empty,
  rebuildNativeImage := NativeImage
  .buildNativeImage(
  "project-manager",
@@ -2248,6 +2261,9 @@ lazy val `runtime-fat-jar` =
  MergeStrategy.concat
  case PathList("module-info.class") =>
  MergeStrategy.preferProject
+ // remove once https://github.com/snakeyaml/snakeyaml/pull/12 gets integrated
+ case PathList("org", "yaml", "snakeyaml", "introspector", xs @ _*) =>
+ MergeStrategy.preferProject
  case PathList(xs @ _*) if xs.last.contains("module-info.class") =>
  MergeStrategy.discard
  case _ => MergeStrategy.first
@@ -2258,6 +2274,7 @@ lazy val `runtime-fat-jar` =
  .dependsOn(`runtime-instrument-repl-debugger`)
  .dependsOn(`runtime-instrument-runtime-server`)
  .dependsOn(`runtime-language-epb`)
+ .dependsOn(yaml)
  .dependsOn(LocalProject("runtime"))
 
 /* Note [Unmanaged Classpath]
@@ -2313,6 +2330,9 @@ lazy val `engine-runner` = project
  MergeStrategy.concat
  case "reference.conf" =>
  MergeStrategy.concat
+ // remove once https://github.com/snakeyaml/snakeyaml/pull/12 gets integrated
+ case PathList("org", "yaml", "snakeyaml", "introspector", xs @ _*) =>
+ MergeStrategy.preferProject
  case PathList(xs @ _*) if xs.last.contains("module-info") =>
  // runner.jar must not be a JPMS module
  MergeStrategy.discard
@@ -2335,6 +2355,69 @@ lazy val `engine-runner` = project
  run / connectInput := true
  )
  .settings(
+ NativeImage.smallJdk := Some(buildSmallJdk.value),
+ NativeImage.additionalCp := {
+ val core = Seq(
+ "runtime.jar",
+ "runner.jar"
+ )
+ core ++ `base-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath())
+ },
+ buildSmallJdk := {
+ val smallJdkDirectory = (target.value / "jdk").getAbsoluteFile()
+ if (smallJdkDirectory.exists()) {
+ IO.delete(smallJdkDirectory)
+ }
+ val JS_MODULES =
+ "org.graalvm.nativeimage,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.base,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.objectfile,org.graalvm.nativeimage.pointsto,com.oracle.graal.graal_enterprise,com.oracle.svm.svm_enterprise,jdk.compiler.graal,jdk.httpserver,java.naming,java.net.http"
+ val DEBUG_MODULES = "jdk.jdwp.agent"
+ val PYTHON_MODULES = "jdk.security.auth,java.naming"
+
+ val javaHome = Option(System.getProperty("java.home")).map(Paths.get(_))
+ val (jlink, modules, libDirs) = javaHome match {
+ case None =>
+ throw new RuntimeException("Missing java.home variable")
+ case Some(jh) =>
+ val exec = jh.resolve("bin").resolve("jlink")
+ val moduleJars = List(
+ "lib/svm/bin/../../graalvm/svm-driver.jar",
+ "lib/svm/bin/../builder/native-image-base.jar",
+ "lib/svm/bin/../builder/objectfile.jar",
+ "lib/svm/bin/../builder/pointsto.jar",
+ "lib/svm/bin/../builder/svm-enterprise.jar",
+ "lib/svm/bin/../builder/svm.jar",
+ "lib/svm/bin/../library-support.jar"
+ )
+ val targetLibDirs = List("graalvm", "svm", "static", "truffle")
+ (
+ exec,
+ moduleJars.map(jar => jh.resolve(jar).toString),
+ targetLibDirs.map(d => jh.resolve("lib").resolve(d))
+ )
+ }
+
+ val exec =
+ s"$jlink --module-path ${modules.mkString(":")} --output $smallJdkDirectory --add-modules $JS_MODULES,$DEBUG_MODULES,$PYTHON_MODULES"
+ val exitCode = scala.sys.process.Process(exec).!
+
+ if (exitCode != 0) {
+ throw new RuntimeException(s"Cannot execute smalljdk.sh")
+ }
+ libDirs.foreach(libDir =>
+ IO.copyDirectory(
+ libDir.toFile,
+ smallJdkDirectory.toPath
+ .resolve("lib")
+ .resolve(libDir.toFile.getName)
+ .toFile
+ )
+ )
+ assert(
+ smallJdkDirectory.exists(),
+ "Directory of small JDK " + smallJdkDirectory + " is not present"
+ )
+ smallJdkDirectory
+ },
  assembly := assembly
  .dependsOn(`runtime-fat-jar` / assembly)
  .value,
@@ -2346,16 +2429,13 @@ lazy val `engine-runner` = project
  additionalOptions = Seq(
  "-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.NoOpLog",
  "-H:IncludeResources=.*Main.enso$",
+ // useful perf & debug switches:
  // "-g",
- //  "-H:+DashboardAll",
- //  "-H:DashboardDump=runner.bgv"
+ // "-H:+SourceLevelDebug",
+ // "-H:-DeleteLocalSymbols",
  "-Dnic=nic"
  ),
  mainClass = Some("org.enso.runner.Main"),
- additionalCp = Seq(
- "runtime.jar",
- "runner.jar"
- ),
  initializeAtRuntime = Seq(
  "org.jline.nativ.JLineLibrary",
  "org.jline.terminal.impl.jna",
@@ -2373,7 +2453,12 @@ lazy val `engine-runner` = project
  "akka.http"
  )
  )
+ .dependsOn(NativeImage.additionalCp)
+ .dependsOn(NativeImage.smallJdk)
  .dependsOn(assembly)
+ .dependsOn(
+ buildEngineDistribution
+ )
  .value,
  buildNativeImage := NativeImage
  .incrementalNativeImageBuild(
@@ -2383,6 +2468,7 @@ lazy val `engine-runner` = project
  .value
  )
  .dependsOn(`version-output`)
+ .dependsOn(yaml)
  .dependsOn(pkg)
  .dependsOn(cli)
  .dependsOn(`library-manager`)
@@ -2392,6 +2478,9 @@ lazy val `engine-runner` = project
  .dependsOn(`logging-service-logback` % Runtime)
  .dependsOn(`polyglot-api`)
 
+lazy val buildSmallJdk =
+ taskKey[File]("Build a minimal JDK used for native image generation")
+
 lazy val launcher = project
  .in(file("engine/launcher"))
  .configs(Test)
@@ -2406,6 +2495,8 @@ lazy val launcher = project
  )
  )
  .settings(
+ NativeImage.smallJdk := None,
+ NativeImage.additionalCp := Seq.empty,
  rebuildNativeImage := NativeImage
  .buildNativeImage(
  "enso",

docs/infrastructure/native-image.md

@@ -115,7 +115,7 @@ Note that for convenience, you can run the launcher/engine runner via
 `bin/enso`, e.g.
 
 ```bash
-env JAVA_OPTS="-agentlib:native-image-agent=config-merge-dir=./engine/runner-native/src/main/resources/META-INF/native-image/org/enso/runner" ./built-distribution/enso-engine-0.0.0-dev-linux-amd64/enso-0.0.0-dev/bin/enso --run tmp.enso
+env JAVA_OPTS="-agentlib:native-image-agent=config-merge-dir=./engine/runner/src/main/resources/META-INF/native-image/org/enso/runner" ./built-distribution/enso-engine-0.0.0-dev-linux-amd64/enso-0.0.0-dev/bin/enso --run tmp.enso
 ```
 
 The command may need to be re-run with different arguments to ensure that all
@@ -210,7 +210,7 @@ sbt> engine-runner/buildNativeImage
 and execute the binary on a sample factorial test program
 
 ```bash
-> runner --run engine/runner-native/src/test/resources/Factorial.enso 6
+> runner --run engine/runner/src/test/resources/Factorial.enso 6
 ```
 
 The task that generates the Native Image, along with all the necessary

engine/runner/src/main/java/org/enso/runner/ContextFactory.java

@@ -149,7 +149,6 @@ PolyglotContext build() {
  .options(options)
  .option(RuntimeOptions.ENABLE_AUTO_PARALLELISM, Boolean.toString(enableAutoParallelism))
  .option(RuntimeOptions.WARNINGS_LIMIT, Integer.toString(warningsLimit))
- .option("js.foreign-object-prototype", "true")
  .out(out)
  .in(in)
  .serverTransport(
@@ -181,17 +180,13 @@ PolyglotContext build() {
  }
  if (ENGINE_HAS_JAVA) {
  var javaHome = System.getProperty("java.home");
- if (javaHome == null) {
- javaHome = System.getenv("JAVA_HOME");
- }
- if (javaHome == null) {
- throw new IllegalStateException("Specify JAVA_HOME environment property");
+ if (javaHome != null) {
+ builder.option("java.JavaHome", javaHome);
  }
  builder
  .option("java.ExposeNativeJavaVM", "true")
  .option("java.Polyglot", "true")
  .option("java.UseBindingsLoader", "true")
- .option("java.JavaHome", javaHome)
  .allowCreateThread(true);
  }
  return new PolyglotContext(builder.build());
@@ -205,6 +200,13 @@ PolyglotContext build() {
 
  static {
  var modules = ModuleLayer.boot().modules().stream();
- ENGINE_HAS_JAVA = modules.anyMatch(m -> "org.graalvm.espresso".equals(m.getName()));
+ var found = modules.anyMatch(m -> "org.graalvm.espresso".equals(m.getName()));
+ if (!found) {
+ var url =
+ ContextFactory.class.getResource(
+ "/META-INF/native-image/com.oracle.truffle.espresso/native-image.properties");
+ found = url != null;
+ }
+ ENGINE_HAS_JAVA = found;
  }
 }