From 64fd1ac3a0292db5cee65961c181fa4b00ff1c53 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 30 May 2025 14:38:51 +0200 Subject: [PATCH 01/26] Sending messages between two JVMs with the help of Persistance.Pool --- build.sbt | 4 + .../src/main/java/module-info.java | 1 + .../java/org/enso/os/environment/jni/JVM.java | 119 ++++++++++++++---- .../org/enso/os/environment/jni/JVMPeer.java | 106 ++++++++++++++++ 4 files changed, 204 insertions(+), 26 deletions(-) create mode 100644 lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java diff --git a/build.sbt b/build.sbt index 0be12477e9f4..d8698cc517aa 100644 --- a/build.sbt +++ b/build.sbt @@ -4313,6 +4313,7 @@ lazy val `os-environment` = ), Compile / internalModuleDependencies ++= Seq( (`engine-common` / Compile / exportedModule).value, + (`persistance` / Compile / exportedModule).value, (`logging-utils` / Compile / exportedModule).value, (`logging-config` / Compile / exportedModule).value ), @@ -4331,6 +4332,7 @@ lazy val `os-environment` = additionalOptions = Seq( "-ea", "--features=org.enso.os.environment.TestCollectorFeature", + "-H:+ForeignAPISupport", "-R:-InstallSegfaultHandler" ) ) @@ -4352,6 +4354,8 @@ lazy val `os-environment` = .value, Test / fork := true ) + .dependsOn(`persistance`) + .dependsOn(`persistance-dsl` % "provided") .dependsOn(`engine-common`) lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) diff --git a/lib/java/os-environment/src/main/java/module-info.java b/lib/java/os-environment/src/main/java/module-info.java index b64339edf424..f55f96b50114 100644 --- a/lib/java/os-environment/src/main/java/module-info.java +++ b/lib/java/os-environment/src/main/java/module-info.java @@ -3,6 +3,7 @@ * Image. */ module org.enso.os.environment { + requires org.enso.persistance; requires org.enso.engine.common; requires org.graalvm.nativeimage; requires org.slf4j; diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java index 822fc0edccd1..187ee7e8845b 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java @@ -1,9 +1,14 @@ package org.enso.os.environment.jni; import java.io.File; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.enso.common.Platform; +import org.enso.persist.Persistance; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.UnmanagedMemory; import org.graalvm.nativeimage.c.struct.SizeOf; @@ -16,7 +21,11 @@ public final class JVM { private final String[] options; private JNI.JNIEnv env = WordFactory.nullPointer(); - JVM(JNIBoot.JNICreateJavaVMPointer factory, String[] options) { + /** persistance pool associated with this JVM object */ + private final Persistance.Pool pool; + + JVM(Persistance.Pool pool, JNIBoot.JNICreateJavaVMPointer factory, String[] options) { + this.pool = pool; this.createJvmFn = factory; this.options = options; } @@ -51,42 +60,100 @@ public static JVM create(File javaHome, String... options) { } } jvmArgs.addAll(Arrays.asList(options)); - return new JVM(createJvmFn, jvmArgs.toArray(new String[0])); + return new JVM(JVMPeer.POOL, createJvmFn, jvmArgs.toArray(new String[0])); } /** - * Executes main method of provided class + * Executes a message in the other JVM. The message is any subclass of {@link Message} + * registered for persistance via {@link Persistable @Persistable} annotation into the {@link + * Persistance.Pool pool associated with this JVM}. The result (which is of type {@code R}) also + * has to be registered for serde. * - * @param classNameWithSlashes class (with `/` as separators) to search main method in - * @param args arguments to pass to the main method + * @param msg the message that gets serialized, transfered into the other JVM, deserialized on the + * other side and {@link Message#evaluate() evaluated} there + * @param the type of result we expect the message to return + * @return the value gets computed via {@link Message#evaluate()} in the other JVM and then it + * gets serialized and transfered back to us. Deserialized and the value is then returned from + * this method */ - public void executeMain(String classNameWithSlashes, String... args) { + public final R execute(Message msg) { + try (var arena = Arena.ofConfined()) { + var bytes = pool.write(msg, null); + var memory = arena.allocate(Math.max(bytes.length, 4096)); + memory.copyFrom(MemorySegment.ofArray(bytes)); + long len = executeMessageBytes("org/enso/os/environment/jni/JVMPeer", "handle", memory); + assert len >= 0; + var reply = memory.asByteBuffer(); + reply.position(0); + reply.limit((int) len); + var result = pool.read(reply, null); + return result.get(msg.replyType); + } catch (IOException ex) { + throw new IllegalStateException(ex); + } + } + + private long executeMessageBytes( + String classNameWithSlashes, String method, MemorySegment segment) { var e = env(); try (var className = CTypeConversion.toCString(classNameWithSlashes); - var mainName = CTypeConversion.toCString("main"); - var stringName = CTypeConversion.toCString("java/lang/String"); - var mainSig = CTypeConversion.toCString("([Ljava/lang/String;)V"); ) { + var methodName = CTypeConversion.toCString(method); + var methodSig = CTypeConversion.toCString("(JJ)J"); ) { var fn = e.getFunctions(); - var mainClazz = fn.getFindClass().call(e, className.get()); - assert mainClazz.isNonNull() : "Class not found " + classNameWithSlashes; - var mainMethod = fn.getGetStaticMethodID().call(e, mainClazz, mainName.get(), mainSig.get()); - assert mainMethod.isNonNull() : "main method found in " + classNameWithSlashes; - var stringClazz = fn.getFindClass().call(e, stringName.get()); - var argsCopy = - fn.getNewObjectArray().call(e, args.length, stringClazz, WordFactory.nullPointer()); - - for (var i = 0; i < args.length; i++) { - try (var ithArg = CTypeConversion.toCString(args[i]); ) { - var str = fn.getNewStringUTF().call(e, ithArg.get()); - fn.getSetObjectArrayElement().call(e, argsCopy, i, str); - } - } - var arg = StackValue.get(JNI.JValue.class); - arg.setJObject(argsCopy); - fn.getCallStaticVoidMethodA().call(e, mainClazz, mainMethod, arg); + var clazz = fn.getFindClass().call(e, className.get()); + assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; + var mainMethod = fn.getGetStaticMethodID().call(e, clazz, methodName.get(), methodSig.get()); + assert mainMethod.isNonNull() : "method not found in " + classNameWithSlashes; + var address = segment.address(); + assert address > 0 : "We need an address"; + var arg = StackValue.get(2, JNI.JValue.class); + arg.addressOf(0).setLong(address); + arg.addressOf(1).setLong(segment.byteSize()); + var replySize = fn.getCallStaticLongMethodA().call(e, clazz, mainMethod, arg); + return replySize; } } + /** + * Subclasses of message denote a computational task to be performed in the "other JVM". + * + * @param type of the return value + */ + public abstract static class Message { + private final Class replyType; + + /** + * Constructor for subclasses. Use it as {@code super(Integer.class)} to specify the reply type + * of the exception which is then returned from the {@link + * #execute(org.enso.os.environment.jni.JVM.Message)} method. + * + * @param replyType the type of the reply + */ + protected Message(Class replyType) { + this.replyType = replyType; + } + + /** + * Evaluates the exception. Invoked in the other JVM. + * + * @return the result of the evaluation or {@code null} + * @throws Throwable the computation may yield exceptions or errors which are then transferred + * back to the callee JVM + */ + protected abstract R evaluate() throws Throwable; + } + + /** + * Executes main method of provided class + * + * @param classNameWithSlashes class (with `/` as separators) to search main method in + * @param args arguments to pass to the main method + */ + public final void executeMain(String classNameWithSlashes, String... args) { + var msg = new JVMPeer.ExecuteMainClass(classNameWithSlashes, List.of(args)); + execute(msg); + } + /** * Initialize or just obtain environment associated with this JVM. * diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java new file mode 100644 index 000000000000..fad0ec032c95 --- /dev/null +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -0,0 +1,106 @@ +package org.enso.os.environment.jni; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.enso.persist.Persistable; +import org.enso.persist.Persistance; + +final class JVMPeer { + static final Persistance.Pool POOL = Persistables.POOL; + + private JVMPeer() {} + + static long handle(long address, long size) { + try { + var seg = MemorySegment.ofAddress(address).reinterpret(size); + var buf = seg.asByteBuffer(); + var ref = POOL.read(buf, null); + var msg = ref.get(JVM.Message.class); + var res = msg.evaluate(); + var bytes = Persistables.POOL.write(res, null); + seg.copyFrom(MemorySegment.ofArray(bytes)); + return bytes.length; + } catch (Throwable t) { + // TBD: proper handling of exceptions is needed + t.printStackTrace(); + return -1; + } + } + + @Persistable(id = 432001) + public static final class ExecuteMainClass extends JVM.Message { + private final String mainClassWithSlashes; + private final List args; + + public ExecuteMainClass(String mainClassWithSlashes, List args) { + super(ExecuteMainClass.class); + this.mainClassWithSlashes = mainClassWithSlashes; + this.args = args; + } + + public String mainClassWithSlashes() { + return mainClassWithSlashes; + } + + public List args() { + return Collections.unmodifiableList(args); + } + + @Override + protected final ExecuteMainClass evaluate() throws Exception { + var clazz = Class.forName(mainClassWithSlashes.replace('/', '.')); + var method = clazz.getDeclaredMethod("main", String[].class); + method.setAccessible(true); + method.invoke(null, (Object) args.toArray(new String[args.size()])); + return this; + } + } + + @Persistable(id = 432002) + public static final class PersistList extends Persistance { + public PersistList() { + super(List.class, true, 432002); + } + + @Override + protected void writeObject(List obj, Persistance.Output out) throws IOException { + out.writeInt(obj.size()); + for (Object o : obj) { + out.writeObject(o); + } + } + + @Override + @SuppressWarnings("unchecked") + protected List readObject(Persistance.Input in) throws IOException, ClassNotFoundException { + int size = in.readInt(); + var lst = new ArrayList(size); + for (int i = 0; i < size; i++) { + var obj = in.readObject(); + lst.add(obj); + } + return lst; + } + } + + @Persistable(id = 4437) + public static final class PersistString extends Persistance { + public PersistString() { + super(String.class, true, 4437); + } + + @Override + protected void writeObject(String obj, Persistance.Output out) throws IOException { + out.writeUTF(obj); + } + + @Override + protected String readObject(Persistance.Input in) throws IOException, ClassNotFoundException { + var obj = in.readUTF(); + return obj; + } + } +} From 2e3911c6fd3818c2f3557b77e5362040430651fa Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 30 May 2025 15:25:02 +0200 Subject: [PATCH 02/26] ExecuteMainClass message doesn't have a return value --- .../main/java/org/enso/os/environment/jni/JVMPeer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index fad0ec032c95..78abb0145272 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -31,12 +31,12 @@ static long handle(long address, long size) { } @Persistable(id = 432001) - public static final class ExecuteMainClass extends JVM.Message { + public static final class ExecuteMainClass extends JVM.Message { private final String mainClassWithSlashes; private final List args; public ExecuteMainClass(String mainClassWithSlashes, List args) { - super(ExecuteMainClass.class); + super(Void.class); this.mainClassWithSlashes = mainClassWithSlashes; this.args = args; } @@ -50,12 +50,12 @@ public List args() { } @Override - protected final ExecuteMainClass evaluate() throws Exception { + protected final Void evaluate() throws Exception { var clazz = Class.forName(mainClassWithSlashes.replace('/', '.')); var method = clazz.getDeclaredMethod("main", String[].class); method.setAccessible(true); method.invoke(null, (Object) args.toArray(new String[args.size()])); - return this; + return null; } } From 5b42e1d7512c6ef4ae8e1b04a006b9761ddf85ef Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 08:00:13 +0200 Subject: [PATCH 03/26] Exchange messages between JVMs. Send ReportResult back. --- .../java/org/enso/os/environment/jni/JVM.java | 31 +++++++++- .../org/enso/os/environment/jni/TestMain.java | 62 ++++++++++++++----- .../os/environment/jni/LoadClassTest.java | 36 +++-------- .../persist/impl/PersistableProcessor.java | 6 ++ 4 files changed, 90 insertions(+), 45 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java index 187ee7e8845b..5d7b13c98ffe 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java @@ -7,11 +7,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import org.enso.common.Platform; import org.enso.persist.Persistance; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.UnmanagedMemory; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.word.WordFactory; @@ -77,11 +83,19 @@ public static JVM create(File javaHome, String... options) { * this method */ public final R execute(Message msg) { + return executeImpl( + pool, + msg, + (memory) -> executeMessageBytes("org/enso/os/environment/jni/JVMPeer", "handle", memory)); + } + + static R executeImpl( + Persistance.Pool pool, Message msg, Function send) { try (var arena = Arena.ofConfined()) { var bytes = pool.write(msg, null); var memory = arena.allocate(Math.max(bytes.length, 4096)); memory.copyFrom(MemorySegment.ofArray(bytes)); - long len = executeMessageBytes("org/enso/os/environment/jni/JVMPeer", "handle", memory); + long len = send.apply(memory); assert len >= 0; var reply = memory.asByteBuffer(); reply.position(0); @@ -114,6 +128,21 @@ private long executeMessageBytes( } } + @CEntryPoint + private static long acceptRequestFromHotSpotJvm( + IsolateThread threadId, CCharPointer data, long size) { + var len = JVMPeer.handle(data.rawValue(), size); + return len; + } + + static final CEntryPointLiteral CALLBACK_FN = + CEntryPointLiteral.create( + JVM.class, + "acceptRequestFromHotSpotJvm", + IsolateThread.class, + CCharPointer.class, + long.class); + /** * Subclasses of message denote a computational task to be performed in the "other JVM". * diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index a2bbd58ba0f9..3c25377ca72e 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -1,14 +1,18 @@ package org.enso.os.environment.jni; -import java.lang.foreign.Arena; import java.lang.foreign.FunctionDescriptor; import java.lang.foreign.Linker; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.lang.invoke.MethodHandle; import java.math.BigInteger; +import java.util.HashMap; +import java.util.Map; +import org.enso.persist.Persistable; final class TestMain { + static final Map CORRECT_RESULTS = new HashMap<>(); + private TestMain() {} static void main(String... args) throws Throwable { @@ -16,10 +20,7 @@ static void main(String... args) throws Throwable { var fnCallbackAddress = MemorySegment.ofAddress(Long.parseLong(args[1])); var fnDescriptor = FunctionDescriptor.of( - ValueLayout.JAVA_BOOLEAN, - ValueLayout.JAVA_LONG, - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS); + ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG); var fnHandle = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); var n = Integer.parseInt(args[2]); @@ -40,16 +41,47 @@ static BigInteger factorial(long n) { private static void reportResultToSvmIsolate( long jvmIsolate, MethodHandle fn, long key, String value) throws Throwable { - try (var arena = Arena.ofConfined()) { - var valueStr = arena.allocateFrom(value); - Object result = fn.invoke(jvmIsolate, key, valueStr); - if (!Boolean.TRUE.equals(result)) { - var ex = - new IllegalStateException( - "Not correct result for " + key + " and value " + value + " result " + result); - ex.printStackTrace(); - throw ex; - } + ReportResult rr = new ReportResult(key, value); + JVM.executeImpl( + JVMPeer.POOL, + rr, + (seg) -> { + Object res = -1L; + try { + var isoRef = MemorySegment.ofAddress(jvmIsolate); + res = fn.invoke(isoRef, seg, seg.byteSize()); + } catch (Throwable ex) { + ex.printStackTrace(); + } + return (long) res; + }); + } + + @Persistable(id = 430606) + public static final class ReportResult extends JVM.Message { + private long key; + private String value; + + public ReportResult(long key, String value) { + super(Void.class); + this.key = key; + this.value = value; + } + + public long key() { + return key; + } + + public String value() { + return value; + } + + @Override + protected Void evaluate() throws Throwable { + var vm = System.getProperty("java.vm.name"); + assert "Substrate VM".equals(vm) : "Running in SVM again: " + vm; + CORRECT_RESULTS.put(key, value); + return null; } } } diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index 7cd73818b2cf..10e9248a73ba 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -4,17 +4,10 @@ import static org.junit.Assert.assertTrue; import java.io.File; -import java.util.HashMap; -import java.util.Map; import java.util.Random; import org.enso.os.environment.jni.JNI.JValue; import org.graalvm.nativeimage.CurrentIsolate; -import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; -import org.graalvm.nativeimage.c.function.CEntryPoint; -import org.graalvm.nativeimage.c.function.CEntryPointLiteral; -import org.graalvm.nativeimage.c.function.CFunctionPointer; -import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.junit.Test; @@ -117,7 +110,7 @@ public void setSystemProperty() { @Test public void executeMainClass() throws Exception { var jvmIsolate = CurrentIsolate.getCurrentThread().rawValue(); - var callbackFn = CALLBACK_FN.getFunctionPointer().rawValue(); + var callbackFn = JVM.CALLBACK_FN.getFunctionPointer().rawValue(); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { @@ -125,26 +118,11 @@ public void executeMainClass() throws Exception { var mainClass = "org/enso/os/environment/jni/TestMain"; jvm().executeMain(mainClass, "" + jvmIsolate, "" + callbackFn, "" + n); } - assertEquals("Five results found: " + CORRECT_RESULTS, 5, CORRECT_RESULTS.size()); - } - - private static final Map CORRECT_RESULTS = new HashMap<>(); - - @CEntryPoint - private static boolean acceptResultFromHotSpotJvm( - IsolateThread threadId, long n, CCharPointer resultStr) { - var result = CTypeConversion.toJavaString(resultStr); - var ownResult = TestMain.factorial(n).toString(); - assertEquals("fac(" + n + ") is correct in both JVMs", ownResult, result); - CORRECT_RESULTS.put(n, result); - return ownResult.equals(result); + assertEquals( + "Five results found: " + TestMain.CORRECT_RESULTS, 5, TestMain.CORRECT_RESULTS.size()); + for (var e : TestMain.CORRECT_RESULTS.entrySet()) { + var expecting = TestMain.factorial(e.getKey()); + assertEquals("fac(" + e.getKey() + ") should be", expecting.toString(), e.getValue()); + } } - - private static final CEntryPointLiteral CALLBACK_FN = - CEntryPointLiteral.create( - LoadClassTest.class, - "acceptResultFromHotSpotJvm", - IsolateThread.class, - long.class, - CCharPointer.class); } diff --git a/lib/java/persistance-dsl/src/main/java/org/enso/persist/impl/PersistableProcessor.java b/lib/java/persistance-dsl/src/main/java/org/enso/persist/impl/PersistableProcessor.java index a729f61257d6..7f763245f40b 100644 --- a/lib/java/persistance-dsl/src/main/java/org/enso/persist/impl/PersistableProcessor.java +++ b/lib/java/persistance-dsl/src/main/java/org/enso/persist/impl/PersistableProcessor.java @@ -274,6 +274,9 @@ && isVisibleFrom(e, orig)) case INT -> w.append(" var ") .append(v.getSimpleName()) .append(" = in.readInt();\n"); + case LONG -> w.append(" var ") + .append(v.getSimpleName()) + .append(" = in.readLong();\n"); default -> processingEnv .getMessager() .printMessage(Kind.ERROR, "Unsupported primitive type: " + v.asType().getKind()); @@ -329,6 +332,9 @@ && isVisibleFrom(e, orig)) case INT -> w.append(" out.writeInt(obj.") .append(v.getSimpleName()) .append("());\n"); + case LONG -> w.append(" out.writeLong(obj.") + .append(v.getSimpleName()) + .append("());\n"); default -> processingEnv .getMessager() .printMessage(Kind.ERROR, "Unsupported primitive type: " + v.asType().getKind()); From 9dc5646d072a655d5e27220b0f2b4e5e4e325590 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 08:54:50 +0200 Subject: [PATCH 04/26] Introducing channel between two JVMs --- .../java/org/enso/os/environment/jni/JVM.java | 111 ++++++++++-------- .../org/enso/os/environment/jni/JVMPeer.java | 5 +- .../org/enso/os/environment/jni/TestMain.java | 3 +- 3 files changed, 67 insertions(+), 52 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java index 5d7b13c98ffe..f96c50d5228a 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java @@ -27,11 +27,7 @@ public final class JVM { private final String[] options; private JNI.JNIEnv env = WordFactory.nullPointer(); - /** persistance pool associated with this JVM object */ - private final Persistance.Pool pool; - - JVM(Persistance.Pool pool, JNIBoot.JNICreateJavaVMPointer factory, String[] options) { - this.pool = pool; + JVM(JNIBoot.JNICreateJavaVMPointer factory, String[] options) { this.createJvmFn = factory; this.options = options; } @@ -66,27 +62,7 @@ public static JVM create(File javaHome, String... options) { } } jvmArgs.addAll(Arrays.asList(options)); - return new JVM(JVMPeer.POOL, createJvmFn, jvmArgs.toArray(new String[0])); - } - - /** - * Executes a message in the other JVM. The message is any subclass of {@link Message} - * registered for persistance via {@link Persistable @Persistable} annotation into the {@link - * Persistance.Pool pool associated with this JVM}. The result (which is of type {@code R}) also - * has to be registered for serde. - * - * @param msg the message that gets serialized, transfered into the other JVM, deserialized on the - * other side and {@link Message#evaluate() evaluated} there - * @param the type of result we expect the message to return - * @return the value gets computed via {@link Message#evaluate()} in the other JVM and then it - * gets serialized and transfered back to us. Deserialized and the value is then returned from - * this method - */ - public final R execute(Message msg) { - return executeImpl( - pool, - msg, - (memory) -> executeMessageBytes("org/enso/os/environment/jni/JVMPeer", "handle", memory)); + return new JVM(createJvmFn, jvmArgs.toArray(new String[0])); } static R executeImpl( @@ -107,27 +83,6 @@ static R executeImpl( } } - private long executeMessageBytes( - String classNameWithSlashes, String method, MemorySegment segment) { - var e = env(); - try (var className = CTypeConversion.toCString(classNameWithSlashes); - var methodName = CTypeConversion.toCString(method); - var methodSig = CTypeConversion.toCString("(JJ)J"); ) { - var fn = e.getFunctions(); - var clazz = fn.getFindClass().call(e, className.get()); - assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; - var mainMethod = fn.getGetStaticMethodID().call(e, clazz, methodName.get(), methodSig.get()); - assert mainMethod.isNonNull() : "method not found in " + classNameWithSlashes; - var address = segment.address(); - assert address > 0 : "We need an address"; - var arg = StackValue.get(2, JNI.JValue.class); - arg.addressOf(0).setLong(address); - arg.addressOf(1).setLong(segment.byteSize()); - var replySize = fn.getCallStaticLongMethodA().call(e, clazz, mainMethod, arg); - return replySize; - } - } - @CEntryPoint private static long acceptRequestFromHotSpotJvm( IsolateThread threadId, CCharPointer data, long size) { @@ -143,6 +98,62 @@ private static long acceptRequestFromHotSpotJvm( CCharPointer.class, long.class); + /** Channel connects two JVMs. */ + public static final class Channel { + /** persistance pool associated with this JVM object */ + private final Persistance.Pool pool; + + /** JNI env */ + private final JNI.JNIEnv env; + + private Channel(Persistance.Pool pool, JNI.JNIEnv env) { + this.pool = pool; + this.env = env; + } + + /** + * Executes a message in the other JVM. The message is any subclass of {@link Message} + * registered for persistance via {@link Persistable @Persistable} annotation into the {@link + * Persistance.Pool pool associated with this JVM}. The result (which is of type {@code R}) also + * has to be registered for serde. + * + * @param msg the message that gets serialized, transfered into the other JVM, deserialized on + * the other side and {@link Message#evaluate() evaluated} there + * @param the type of result we expect the message to return + * @return the value gets computed via {@link Message#evaluate()} in the other JVM and then it + * gets serialized and transfered back to us. Deserialized and the value is then returned + * from this method + */ + public final R execute(Message msg) { + return executeImpl( + pool, + msg, + (memory) -> + executeMessageBytes(env, "org/enso/os/environment/jni/JVMPeer", "handle", memory)); + } + + private static long executeMessageBytes( + JNI.JNIEnv e, String classNameWithSlashes, String method, MemorySegment segment) { + try (var className = CTypeConversion.toCString(classNameWithSlashes); + var methodName = CTypeConversion.toCString(method); + var methodSig = CTypeConversion.toCString("(JJ)J"); ) { + var fn = e.getFunctions(); + var clazz = fn.getFindClass().call(e, className.get()); + assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; + var mainMethod = + fn.getGetStaticMethodID().call(e, clazz, methodName.get(), methodSig.get()); + assert mainMethod.isNonNull() : "method not found in " + classNameWithSlashes; + var address = segment.address(); + assert address > 0 : "We need an address"; + var arg = StackValue.get(2, JNI.JValue.class); + arg.addressOf(0).setLong(address); + arg.addressOf(1).setLong(segment.byteSize()); + var replySize = fn.getCallStaticLongMethodA().call(e, clazz, mainMethod, arg); + return replySize; + } + } + } + /** * Subclasses of message denote a computational task to be performed in the "other JVM". * @@ -165,11 +176,12 @@ protected Message(Class replyType) { /** * Evaluates the exception. Invoked in the other JVM. * + * @param channel allows sending messages to the other JVM * @return the result of the evaluation or {@code null} * @throws Throwable the computation may yield exceptions or errors which are then transferred * back to the callee JVM */ - protected abstract R evaluate() throws Throwable; + protected abstract R evaluate(Channel channel) throws Throwable; } /** @@ -180,7 +192,8 @@ protected Message(Class replyType) { */ public final void executeMain(String classNameWithSlashes, String... args) { var msg = new JVMPeer.ExecuteMainClass(classNameWithSlashes, List.of(args)); - execute(msg); + var channel = new Channel(JVMPeer.POOL, env()); + channel.execute(msg); } /** diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index 78abb0145272..29ba065b9fba 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.enso.os.environment.jni.JVM.Channel; import org.enso.persist.Persistable; import org.enso.persist.Persistance; @@ -19,7 +20,7 @@ static long handle(long address, long size) { var buf = seg.asByteBuffer(); var ref = POOL.read(buf, null); var msg = ref.get(JVM.Message.class); - var res = msg.evaluate(); + var res = msg.evaluate(null); var bytes = Persistables.POOL.write(res, null); seg.copyFrom(MemorySegment.ofArray(bytes)); return bytes.length; @@ -50,7 +51,7 @@ public List args() { } @Override - protected final Void evaluate() throws Exception { + protected final Void evaluate(Channel notNeeded) throws Exception { var clazz = Class.forName(mainClassWithSlashes.replace('/', '.')); var method = clazz.getDeclaredMethod("main", String[].class); method.setAccessible(true); diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 3c25377ca72e..1ed9901b4686 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -8,6 +8,7 @@ import java.math.BigInteger; import java.util.HashMap; import java.util.Map; +import org.enso.os.environment.jni.JVM.Channel; import org.enso.persist.Persistable; final class TestMain { @@ -77,7 +78,7 @@ public String value() { } @Override - protected Void evaluate() throws Throwable { + protected Void evaluate(Channel otherVM) throws Throwable { var vm = System.getProperty("java.vm.name"); assert "Substrate VM".equals(vm) : "Running in SVM again: " + vm; CORRECT_RESULTS.put(key, value); From 4d69228ef4617bb3ad8c3071c449bc9837b80a06 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 11:06:07 +0200 Subject: [PATCH 05/26] Using RequestFactorial message --- .../java/org/enso/os/environment/jni/JVM.java | 87 ++++++++++++++----- .../org/enso/os/environment/jni/JVMPeer.java | 24 +++-- .../org/enso/os/environment/jni/TestMain.java | 29 ++++++- .../os/environment/jni/LoadClassTest.java | 28 +++++- 4 files changed, 131 insertions(+), 37 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java index f96c50d5228a..e49617ab9f88 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java @@ -3,13 +3,18 @@ import java.io.File; import java.io.IOException; import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Function; import org.enso.common.Platform; import org.enso.persist.Persistance; +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.ImageInfo; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.UnmanagedMemory; @@ -85,8 +90,9 @@ static R executeImpl( @CEntryPoint private static long acceptRequestFromHotSpotJvm( - IsolateThread threadId, CCharPointer data, long size) { - var len = JVMPeer.handle(data.rawValue(), size); + IsolateThread threadId, CCharPointer data, long size) throws Throwable { + // TBD: recursive calls will need to find a proper channel + var len = JVMPeer.handleWithChannel(null, data.rawValue(), size); return len; } @@ -100,15 +106,28 @@ private static long acceptRequestFromHotSpotJvm( /** Channel connects two JVMs. */ public static final class Channel { - /** persistance pool associated with this JVM object */ + /** persistance pool associated with this channel object */ private final Persistance.Pool pool; - /** JNI env */ private final JNI.JNIEnv env; + private final long isolate; + private final long callbackFn; - private Channel(Persistance.Pool pool, JNI.JNIEnv env) { + /* private */ Channel(Persistance.Pool pool, JNI.JNIEnv env) { this.pool = pool; this.env = env; + this.isolate = -1; + this.callbackFn = -1; + } + + /* private */ Channel(Persistance.Pool pool, long isolate, long callbackFn) { + if (ImageInfo.inImageCode()) { + throw new IllegalStateException("Only usable in HotSpot"); + } + this.pool = pool; + this.env = null; + this.isolate = isolate; + this.callbackFn = callbackFn; } /** @@ -125,30 +144,52 @@ private Channel(Persistance.Pool pool, JNI.JNIEnv env) { * from this method */ public final R execute(Message msg) { - return executeImpl( - pool, - msg, - (memory) -> - executeMessageBytes(env, "org/enso/os/environment/jni/JVMPeer", "handle", memory)); + if (this.isolate == -1) { + return executeImpl(pool, msg, (memory) -> toHotSpotMessage(env, memory)); + } else { + var fnCallbackAddress = MemorySegment.ofAddress(callbackFn); + var fnDescriptor = + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG); + var fnHandle = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); + return executeImpl( + pool, + msg, + (seg) -> { + Object res = -1L; + try { + var isoRef = MemorySegment.ofAddress(isolate); + res = fnHandle.invoke(isoRef, seg, seg.byteSize()); + } catch (Throwable ex) { + ex.printStackTrace(); + } + return (long) res; + }); + } } - private static long executeMessageBytes( - JNI.JNIEnv e, String classNameWithSlashes, String method, MemorySegment segment) { - try (var className = CTypeConversion.toCString(classNameWithSlashes); - var methodName = CTypeConversion.toCString(method); - var methodSig = CTypeConversion.toCString("(JJ)J"); ) { + private static long toHotSpotMessage(JNI.JNIEnv e, MemorySegment segment) { + var classNameWithSlashes = "org/enso/os/environment/jni/JVMPeer"; + var methodName = "handle"; + try (var classInC = CTypeConversion.toCString(classNameWithSlashes); + var methodInC = CTypeConversion.toCString(methodName); + var signatureInC = CTypeConversion.toCString("(JJJJ)J"); ) { var fn = e.getFunctions(); - var clazz = fn.getFindClass().call(e, className.get()); + var clazz = fn.getFindClass().call(e, classInC.get()); assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; - var mainMethod = - fn.getGetStaticMethodID().call(e, clazz, methodName.get(), methodSig.get()); - assert mainMethod.isNonNull() : "method not found in " + classNameWithSlashes; + var method = fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); + assert method.isNonNull() : "method not found in " + classNameWithSlashes; var address = segment.address(); assert address > 0 : "We need an address"; - var arg = StackValue.get(2, JNI.JValue.class); - arg.addressOf(0).setLong(address); - arg.addressOf(1).setLong(segment.byteSize()); - var replySize = fn.getCallStaticLongMethodA().call(e, clazz, mainMethod, arg); + var arg = StackValue.get(4, JNI.JValue.class); + arg.addressOf(0).setLong(CurrentIsolate.getCurrentThread().rawValue()); + arg.addressOf(1).setLong(CALLBACK_FN.getFunctionPointer().rawValue()); + arg.addressOf(2).setLong(address); + arg.addressOf(3).setLong(segment.byteSize()); + var replySize = fn.getCallStaticLongMethodA().call(e, clazz, method, arg); return replySize; } } diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index 29ba065b9fba..aa9904fa77a4 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -14,16 +14,11 @@ final class JVMPeer { private JVMPeer() {} - static long handle(long address, long size) { + /** called via JNI from SVM to HotSpot */ + static long handle(long threadId, long callbackFn, long address, long size) { try { - var seg = MemorySegment.ofAddress(address).reinterpret(size); - var buf = seg.asByteBuffer(); - var ref = POOL.read(buf, null); - var msg = ref.get(JVM.Message.class); - var res = msg.evaluate(null); - var bytes = Persistables.POOL.write(res, null); - seg.copyFrom(MemorySegment.ofArray(bytes)); - return bytes.length; + var channel = new JVM.Channel(JVMPeer.POOL, threadId, callbackFn); + return handleWithChannel(channel, address, size); } catch (Throwable t) { // TBD: proper handling of exceptions is needed t.printStackTrace(); @@ -31,6 +26,17 @@ static long handle(long address, long size) { } } + static long handleWithChannel(Channel channel, long address, long size) throws Throwable { + var seg = MemorySegment.ofAddress(address).reinterpret(size); + var buf = seg.asByteBuffer(); + var ref = POOL.read(buf, null); + var msg = ref.get(JVM.Message.class); + var res = msg.evaluate(channel); + var bytes = Persistables.POOL.write(res, null); + seg.copyFrom(MemorySegment.ofArray(bytes)); + return bytes.length; + } + @Persistable(id = 432001) public static final class ExecuteMainClass extends JVM.Message { private final String mainClassWithSlashes; diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 1ed9901b4686..42f541c4e628 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -58,12 +58,33 @@ private static void reportResultToSvmIsolate( }); } + @Persistable(id = 430607) + static final class RequestFactorial extends JVM.Message { + private long n; + + RequestFactorial(long n) { + super(Void.class); + this.n = n; + } + + @Override + protected Void evaluate(Channel channel) throws Throwable { + var res = factorial(n).toString(); + channel.execute(new ReportResult(n, res)); + return null; + } + + long n() { + return n; + } + } + @Persistable(id = 430606) - public static final class ReportResult extends JVM.Message { - private long key; - private String value; + static final class ReportResult extends JVM.Message { + private final long key; + private final String value; - public ReportResult(long key, String value) { + ReportResult(long key, String value) { super(Void.class); this.key = key; this.value = value; diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index 10e9248a73ba..fd31e75fe120 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -12,6 +12,11 @@ import org.junit.Test; public class LoadClassTest { + // TBD: Make the number bigger again + // - which currently exceeds the maximum size of a message + // - so fix it + private static final int MAX = 5; // 5000; + private static final int MIN = 1; // 1000; private static final String PATH = System.getProperty("java.home"); // set from TestCollectorFeature public static String MODULE_PATH; @@ -111,10 +116,12 @@ public void setSystemProperty() { public void executeMainClass() throws Exception { var jvmIsolate = CurrentIsolate.getCurrentThread().rawValue(); var callbackFn = JVM.CALLBACK_FN.getFunctionPointer().rawValue(); + TestMain.CORRECT_RESULTS.clear(); + assertEquals("Results are empty", 0, TestMain.CORRECT_RESULTS.size()); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { - n += gen.nextLong(1000, 5000); + n += gen.nextLong(MIN, MAX); var mainClass = "org/enso/os/environment/jni/TestMain"; jvm().executeMain(mainClass, "" + jvmIsolate, "" + callbackFn, "" + n); } @@ -125,4 +132,23 @@ public void executeMainClass() throws Exception { assertEquals("fac(" + e.getKey() + ") should be", expecting.toString(), e.getValue()); } } + + @Test + public void computeFactoriaViaMessages() throws Exception { + TestMain.CORRECT_RESULTS.clear(); + assertEquals("Results are empty", 0, TestMain.CORRECT_RESULTS.size()); + var channel = new JVM.Channel(JVMPeer.POOL, env()); + var gen = new Random(); + var n = 0L; + for (var i = 0; i < 5; i++) { + n += gen.nextLong(MIN, MAX); + channel.execute(new TestMain.RequestFactorial(n)); + } + assertEquals( + "Five results found: " + TestMain.CORRECT_RESULTS, 5, TestMain.CORRECT_RESULTS.size()); + for (var e : TestMain.CORRECT_RESULTS.entrySet()) { + var expecting = TestMain.factorial(e.getKey()); + assertEquals("fac(" + e.getKey() + ") should be", expecting.toString(), e.getValue()); + } + } } From 1e753ebb96df4e7d3847332c947d5381cfad5248 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 13:13:39 +0200 Subject: [PATCH 06/26] Message with a result --- .../org/enso/os/environment/jni/TestMain.java | 20 +++++++++++++++++++ .../os/environment/jni/LoadClassTest.java | 15 +++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 42f541c4e628..7f55348ac90b 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -79,6 +79,26 @@ long n() { } } + @Persistable(id = 430608) + static final class ComputeFactorial extends JVM.Message { + private long n; + + ComputeFactorial(long n) { + super(String.class); + this.n = n; + } + + @Override + protected String evaluate(Channel channel) throws Throwable { + var res = factorial(n).toString(); + return res; + } + + long n() { + return n; + } + } + @Persistable(id = 430606) static final class ReportResult extends JVM.Message { private final long key; diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index fd31e75fe120..d6bb393befe1 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -134,7 +134,7 @@ public void executeMainClass() throws Exception { } @Test - public void computeFactoriaViaMessages() throws Exception { + public void computeFactorialViaMessages() throws Exception { TestMain.CORRECT_RESULTS.clear(); assertEquals("Results are empty", 0, TestMain.CORRECT_RESULTS.size()); var channel = new JVM.Channel(JVMPeer.POOL, env()); @@ -151,4 +151,17 @@ public void computeFactoriaViaMessages() throws Exception { assertEquals("fac(" + e.getKey() + ") should be", expecting.toString(), e.getValue()); } } + + @Test + public void computeFactorialViaSingleMessage() throws Exception { + var channel = new JVM.Channel(JVMPeer.POOL, env()); + var gen = new Random(); + var n = 0L; + for (var i = 0; i < 5; i++) { + n += gen.nextLong(MIN, MAX); + var res = channel.execute(new TestMain.ComputeFactorial(n)); + var expecting = TestMain.factorial(n); + assertEquals("fac(" + n + ") should be", expecting.toString(), res); + } + } } From d7d547bcb890c3b646cd01fb96098a7483f39554 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 13:19:50 +0200 Subject: [PATCH 07/26] Return BigInteger result --- .../org/enso/os/environment/jni/JVMPeer.java | 24 +++++++++++++++++++ .../org/enso/os/environment/jni/TestMain.java | 8 +++---- .../os/environment/jni/LoadClassTest.java | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index aa9904fa77a4..4a41d5fe2742 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -110,4 +111,27 @@ protected String readObject(Persistance.Input in) throws IOException, ClassNotFo return obj; } } + + @Persistable(id = 4438) + public static final class PersistBigInt extends Persistance { + public PersistBigInt() { + super(BigInteger.class, true, 4438); + } + + @Override + protected void writeObject(BigInteger obj, Persistance.Output out) throws IOException { + var arr = obj.toByteArray(); + out.writeInt(arr.length); + out.write(arr); + } + + @Override + protected BigInteger readObject(Persistance.Input in) + throws IOException, ClassNotFoundException { + var size = in.readInt(); + var arr = new byte[size]; + in.readFully(arr); + return new BigInteger(arr); + } + } } diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 7f55348ac90b..077624aafc4a 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -80,17 +80,17 @@ long n() { } @Persistable(id = 430608) - static final class ComputeFactorial extends JVM.Message { + static final class ComputeFactorial extends JVM.Message { private long n; ComputeFactorial(long n) { - super(String.class); + super(BigInteger.class); this.n = n; } @Override - protected String evaluate(Channel channel) throws Throwable { - var res = factorial(n).toString(); + protected BigInteger evaluate(Channel channel) throws Throwable { + var res = factorial(n); return res; } diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index d6bb393befe1..c62f5b78cd02 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -161,7 +161,7 @@ public void computeFactorialViaSingleMessage() throws Exception { n += gen.nextLong(MIN, MAX); var res = channel.execute(new TestMain.ComputeFactorial(n)); var expecting = TestMain.factorial(n); - assertEquals("fac(" + n + ") should be", expecting.toString(), res); + assertEquals("fac(" + n + ") should be", expecting, res); } } } From 879a09204fe0ddff153553da236c63a79fb0992c Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 16:43:59 +0200 Subject: [PATCH 08/26] Remove runtime dependency on Lookup --- build.sbt | 35 +++++++++---------- distribution/engine/THIRD-PARTY/NOTICE | 5 --- .../NOTICES | 1 - .../src/main/java/module-info.java | 1 - .../src/main/java/module-info.java | 1 - .../server/RuntimeServerInstrument.java | 7 ++-- .../copyright-keep | 1 - .../files-ignore | 2 -- tools/legal-review/engine/report-state | 4 +-- 9 files changed, 22 insertions(+), 35 deletions(-) delete mode 100644 distribution/engine/THIRD-PARTY/org.netbeans.api.org-openide-util-lookup-RELEASE180/NOTICES delete mode 100644 tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/copyright-keep delete mode 100644 tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/files-ignore diff --git a/build.sbt b/build.sbt index d8698cc517aa..d58075e28e3c 100644 --- a/build.sbt +++ b/build.sbt @@ -2141,13 +2141,10 @@ lazy val `persistance` = (project in file("lib/java/persistance")) Compile / javacOptions := ((Compile / javacOptions).value), inConfig(Compile)(truffleRunOptionsSettings), libraryDependencies ++= slf4jApi ++ Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, - "junit" % "junit" % junitVersion % Test, - "com.github.sbt" % "junit-interface" % junitIfVersion % Test + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test ), - Compile / moduleDependencies ++= slf4jApi ++ Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion - ) + Compile / moduleDependencies ++= slf4jApi ) .dependsOn(`persistance-dsl` % Test) @@ -3356,12 +3353,12 @@ lazy val `runtime-parser` = commands += WithDebugCommand.withDebug, fork := true, libraryDependencies ++= Seq( - "junit" % "junit" % junitVersion % Test, - "com.github.sbt" % "junit-interface" % junitIfVersion % Test, - "org.scalatest" %% "scalatest" % scalatestVersion % Test + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided" ), Compile / moduleDependencies ++= Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion ), // Java compiler is not able to correctly find all the annotation processors, because // one of them is on module-path. To overcome this, we explicitly list all of them here. @@ -3643,9 +3640,10 @@ lazy val `runtime-instrument-common` = "ENSO_TEST_DISABLE_IR_CACHE" -> "false" ), libraryDependencies ++= Seq( - "junit" % "junit" % junitVersion % Test, - "com.github.sbt" % "junit-interface" % junitIfVersion % Test, - "org.scalatest" %% "scalatest" % scalatestVersion % Test + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test, + "org.scalatest" %% "scalatest" % scalatestVersion % Test, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % Test ), javaModuleName := "org.enso.runtime.instrument.common", Compile / moduleDependencies ++= slf4jApi ++ Seq( @@ -3739,12 +3737,11 @@ lazy val `runtime-instrument-runtime-server` = Compile / forceModuleInfoCompilation := true, instrumentationSettings, Compile / moduleDependencies ++= Seq( - "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion, - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, - "org.graalvm.sdk" % "word" % graalMavenPackagesVersion, - "org.graalvm.sdk" % "nativeimage" % graalMavenPackagesVersion, - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion, + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "org.graalvm.sdk" % "word" % graalMavenPackagesVersion, + "org.graalvm.sdk" % "nativeimage" % graalMavenPackagesVersion ), Compile / internalModuleDependencies := Seq( (`runtime-instrument-common` / Compile / exportedModule).value, diff --git a/distribution/engine/THIRD-PARTY/NOTICE b/distribution/engine/THIRD-PARTY/NOTICE index 7fe41ad75bc1..1487b6890fe7 100644 --- a/distribution/engine/THIRD-PARTY/NOTICE +++ b/distribution/engine/THIRD-PARTY/NOTICE @@ -556,11 +556,6 @@ The license file can be found at `licenses/APACHE2.0`. Copyright notices related to this dependency can be found in the directory `org.netbeans.api.org-netbeans-modules-sampler-RELEASE180`. -'org-openide-util-lookup', licensed under the The Apache Software License, Version 2.0, is distributed with the engine. -The license file can be found at `licenses/APACHE2.0`. -Copyright notices related to this dependency can be found in the directory `org.netbeans.api.org-openide-util-lookup-RELEASE180`. - - 'reactive-streams', licensed under the CC0, is distributed with the engine. The license file can be found at `licenses/CC0`. Copyright notices related to this dependency can be found in the directory `org.reactivestreams.reactive-streams-1.0.3`. diff --git a/distribution/engine/THIRD-PARTY/org.netbeans.api.org-openide-util-lookup-RELEASE180/NOTICES b/distribution/engine/THIRD-PARTY/org.netbeans.api.org-openide-util-lookup-RELEASE180/NOTICES deleted file mode 100644 index dc34027f6f18..000000000000 --- a/distribution/engine/THIRD-PARTY/org.netbeans.api.org-openide-util-lookup-RELEASE180/NOTICES +++ /dev/null @@ -1 +0,0 @@ -regarding copyright ownership. The ASF licenses this file diff --git a/engine/runtime-compiler/src/main/java/module-info.java b/engine/runtime-compiler/src/main/java/module-info.java index d88961c29a23..f1b81e1d0020 100644 --- a/engine/runtime-compiler/src/main/java/module-info.java +++ b/engine/runtime-compiler/src/main/java/module-info.java @@ -11,7 +11,6 @@ requires org.enso.syntax; requires org.enso.scala.wrapper; - requires org.openide.util.lookup.RELEASE180; requires org.slf4j; exports org.enso.compiler; diff --git a/engine/runtime-instrument-runtime-server/src/main/java/module-info.java b/engine/runtime-instrument-runtime-server/src/main/java/module-info.java index dc8eedcb9c66..ed46a77bf65e 100644 --- a/engine/runtime-instrument-runtime-server/src/main/java/module-info.java +++ b/engine/runtime-instrument-runtime-server/src/main/java/module-info.java @@ -5,7 +5,6 @@ requires org.enso.engine.common; requires org.enso.distribution; requires org.enso.lockmanager; - requires org.openide.util.lookup.RELEASE180; requires org.graalvm.polyglot; requires org.graalvm.truffle; diff --git a/engine/runtime-instrument-runtime-server/src/main/java/org/enso/interpreter/instrument/runtime/server/RuntimeServerInstrument.java b/engine/runtime-instrument-runtime-server/src/main/java/org/enso/interpreter/instrument/runtime/server/RuntimeServerInstrument.java index 09488901fc42..805518a47860 100644 --- a/engine/runtime-instrument-runtime-server/src/main/java/org/enso/interpreter/instrument/runtime/server/RuntimeServerInstrument.java +++ b/engine/runtime-instrument-runtime-server/src/main/java/org/enso/interpreter/instrument/runtime/server/RuntimeServerInstrument.java @@ -10,6 +10,7 @@ import java.net.URI; import java.util.Arrays; import java.util.Optional; +import java.util.ServiceLoader; import org.enso.distribution.locking.LockManager; import org.enso.interpreter.instrument.Handler; import org.enso.interpreter.instrument.HandlerFactory; @@ -25,7 +26,6 @@ import org.graalvm.options.OptionDescriptors; import org.graalvm.polyglot.io.MessageEndpoint; import org.graalvm.polyglot.io.MessageTransport; -import org.openide.util.Lookup; /** * An instrument exposing a server for other services to connect to, in order to control the current @@ -112,8 +112,9 @@ protected void onCreate(Env env) { if (TruffleOptions.AOT) { this.handler = HandlerFactoryImpl.create(); } else { - var loadedHandler = Lookup.getDefault().lookup(HandlerFactory.class); - this.handler = loadedHandler != null ? loadedHandler.create() : HandlerFactoryImpl.create(); + var loadedHandler = ServiceLoader.load(HandlerFactory.class).findFirst(); + this.handler = + loadedHandler.isPresent() ? loadedHandler.get().create() : HandlerFactoryImpl.create(); } try { diff --git a/tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/copyright-keep b/tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/copyright-keep deleted file mode 100644 index dc34027f6f18..000000000000 --- a/tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/copyright-keep +++ /dev/null @@ -1 +0,0 @@ -regarding copyright ownership. The ASF licenses this file diff --git a/tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/files-ignore b/tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/files-ignore deleted file mode 100644 index 26e9c87b6125..000000000000 --- a/tools/legal-review/engine/org.netbeans.api.org-openide-util-lookup-RELEASE180/files-ignore +++ /dev/null @@ -1,2 +0,0 @@ -META-INF/NOTICE -META-INF/LICENSE diff --git a/tools/legal-review/engine/report-state b/tools/legal-review/engine/report-state index 79a7ed410b7f..dabd2db7753e 100644 --- a/tools/legal-review/engine/report-state +++ b/tools/legal-review/engine/report-state @@ -1,3 +1,3 @@ -3924E2C2018608E709D696E47452443D9AD744270998BA744F60F98B89D9616C -F7A98FA4ED2312A599D8D9BFF7C557FC99E93A66E33143212B8EB33B8B873AFB +AD86AB5EE5361D9FA7CC00CDD6193274A81ABC7EA15DA1D6806CB29602937659 +BC8B409BF037E97CB507E861A7FA58A8D67B7BFD71A6EDC84A31CC86FA24EF1F 0 From a23cdd5948ac3fe24727d3432d51f5af0f0434ea Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 17:10:27 +0200 Subject: [PATCH 09/26] Back and forth computation of factorial --- .../org/enso/os/environment/jni/JVMPeer.java | 17 ++++++++ .../org/enso/os/environment/jni/TestMain.java | 29 ++++++++++++++ .../os/environment/jni/LoadClassTest.java | 39 +++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index 4a41d5fe2742..d85232524f67 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -134,4 +134,21 @@ protected BigInteger readObject(Persistance.Input in) return new BigInteger(arr); } } + + @Persistable(id = 4439) + public static final class PersistLong extends Persistance { + public PersistLong() { + super(Long.class, true, 4439); + } + + @Override + protected void writeObject(Long obj, Persistance.Output out) throws IOException { + out.writeLong(obj); + } + + @Override + protected Long readObject(Persistance.Input in) throws IOException { + return in.readLong(); + } + } } diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 077624aafc4a..820a2c295ff4 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -126,4 +126,33 @@ protected Void evaluate(Channel otherVM) throws Throwable { return null; } } + + @Persistable(id = 430609) + static final class CountDownAndReturn extends JVM.Message { + private final long value; + private final long acc; + + CountDownAndReturn(long value, long acc) { + super(Long.class); + this.value = value; + this.acc = acc; + } + + long value() { + return value; + } + + long acc() { + return acc; + } + + @Override + protected Long evaluate(Channel otherVM) throws Throwable { + if (value <= 1) { + return acc; + } else { + return otherVM.execute(new CountDownAndReturn(value - 1, acc * value)); + } + } + } } diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index c62f5b78cd02..c49f5e8e1a6e 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -9,6 +9,7 @@ import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.junit.Ignore; import org.junit.Test; public class LoadClassTest { @@ -164,4 +165,42 @@ public void computeFactorialViaSingleMessage() throws Exception { assertEquals("fac(" + n + ") should be", expecting, res); } } + + @Test + public void backAndForthFactorialOne() throws Exception { + var channel = new JVM.Channel(JVMPeer.POOL, env()); + var fac = channel.execute(new TestMain.CountDownAndReturn(1, 1)); + assertEquals(1, fac.longValue()); + } + + @Test + public void backAndForthFactorialTwo() throws Exception { + var channel = new JVM.Channel(JVMPeer.POOL, env()); + var fac = channel.execute(new TestMain.CountDownAndReturn(2, 1)); + assertEquals(2, fac.longValue()); + } + + @Ignore + @Test + public void backAndForthFactorialThree() throws Exception { + var channel = new JVM.Channel(JVMPeer.POOL, env()); + var fac = channel.execute(new TestMain.CountDownAndReturn(3, 1)); + assertEquals(6, fac.longValue()); + } + + @Ignore + @Test + public void backAndForthFactorialFour() throws Exception { + var channel = new JVM.Channel(JVMPeer.POOL, env()); + var fac = channel.execute(new TestMain.CountDownAndReturn(4, 1)); + assertEquals(24, fac.longValue()); + } + + @Ignore + @Test + public void backAndForthFactorialFive() throws Exception { + var channel = new JVM.Channel(JVMPeer.POOL, env()); + var fac = channel.execute(new TestMain.CountDownAndReturn(5, 1)); + assertEquals(120, fac.longValue()); + } } From eeab6a9788e5abcad3ab065e18e658bbe41c91fc Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 2 Jun 2025 17:14:39 +0200 Subject: [PATCH 10/26] Enabling Foreign API support --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index d58075e28e3c..e993e90f07a6 100644 --- a/build.sbt +++ b/build.sbt @@ -4003,6 +4003,7 @@ lazy val `engine-runner` = project "-H:+AddAllCharsets", "-H:+IncludeAllLocales", "-H:+RunReachabilityHandlersConcurrently", + "-H:+ForeignAPISupport", "-R:-InstallSegfaultHandler", // Workaround a problem with build-/runtime-initialization conflict // by disabling this service provider From cf02db1781087f0c9f462184fa64dba4d00df3c6 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 07:14:29 +0200 Subject: [PATCH 11/26] Moving Channel and Message into their own class --- .../org/enso/os/environment/jni/Channel.java | 143 ++++++++++++++++++ .../java/org/enso/os/environment/jni/JVM.java | 130 +--------------- .../org/enso/os/environment/jni/JVMPeer.java | 7 +- .../org/enso/os/environment/jni/TestMain.java | 9 +- .../os/environment/jni/LoadClassTest.java | 14 +- 5 files changed, 159 insertions(+), 144 deletions(-) create mode 100644 lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java new file mode 100644 index 000000000000..31365a8487b3 --- /dev/null +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -0,0 +1,143 @@ +package org.enso.os.environment.jni; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import org.enso.persist.Persistance; +import org.graalvm.nativeimage.CurrentIsolate; +import org.graalvm.nativeimage.ImageInfo; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.type.CTypeConversion; + +/** Channel connects two {@link JVM} instances. */ +public final class Channel { + + /** persistance pool associated with this channel object */ + private final Persistance.Pool pool; + + private final JNI.JNIEnv env; + private final long isolate; + private final long callbackFn; + + /* private */ + Channel(Persistance.Pool pool, JNI.JNIEnv env) { + this.pool = pool; + this.env = env; + this.isolate = -1; + this.callbackFn = -1; + } + + /* private */ + Channel(Persistance.Pool pool, long isolate, long callbackFn) { + if (ImageInfo.inImageCode()) { + throw new IllegalStateException("Only usable in HotSpot"); + } + this.pool = pool; + this.env = null; + this.isolate = isolate; + this.callbackFn = callbackFn; + } + + /** + * Executes a message in the other JVM. The message is any subclass of {@link Message} + * registered for persistance via {@link Persistable @Persistable} annotation into the {@link + * Persistance.Pool pool associated with this JVM}. The result (which is of type {@code R}) also + * has to be registered for serde. + * + * @param msg the message that gets serialized, transfered into the other JVM, deserialized on the + * other side and {@link Message#evaluate() evaluated} there + * @param the type of result we expect the message to return + * @return the value gets computed via {@link Message#evaluate()} in the other JVM and then it + * gets serialized and transfered back to us. Deserialized and the value is then returned from + * this method + */ + public final R execute(Message msg) { + if (this.isolate == -1) { + return JVM.executeImpl(pool, msg, memory -> toHotSpotMessage(env, memory)); + } else { + java.lang.foreign.MemorySegment fnCallbackAddress = MemorySegment.ofAddress(callbackFn); + java.lang.foreign.FunctionDescriptor fnDescriptor = + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG); + java.lang.invoke.MethodHandle fnHandle = + Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); + return JVM.executeImpl( + pool, + msg, + seg -> { + Object res = -1L; + try { + java.lang.foreign.MemorySegment isoRef = MemorySegment.ofAddress(isolate); + res = fnHandle.invoke(isoRef, seg, seg.byteSize()); + } catch (Throwable ex) { + ex.printStackTrace(); + } + return (long) res; + }); + } + } + + private static long toHotSpotMessage(JNI.JNIEnv e, MemorySegment segment) { + java.lang.String classNameWithSlashes = "org/enso/os/environment/jni/JVMPeer"; + java.lang.String methodName = "handle"; + try (org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder classInC = + CTypeConversion.toCString(classNameWithSlashes); + org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder methodInC = + CTypeConversion.toCString(methodName); + org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder signatureInC = + CTypeConversion.toCString("(JJJJ)J")) { + org.enso.os.environment.jni.JNINativeInterface fn = e.getFunctions(); + org.enso.os.environment.jni.JNI.JClass clazz = fn.getFindClass().call(e, classInC.get()); + assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; + org.enso.os.environment.jni.JNI.JMethodID method = + fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); + assert method.isNonNull() : "method not found in " + classNameWithSlashes; + long address = segment.address(); + assert address > 0 : "We need an address"; + org.enso.os.environment.jni.JNI.JValue arg = StackValue.get(4, JNI.JValue.class); + arg.addressOf(0).setLong(CurrentIsolate.getCurrentThread().rawValue()); + arg.addressOf(1).setLong(JVM.CALLBACK_FN.getFunctionPointer().rawValue()); + arg.addressOf(2).setLong(address); + arg.addressOf(3).setLong(segment.byteSize()); + long replySize = fn.getCallStaticLongMethodA().call(e, clazz, method, arg); + return replySize; + } + } + + /** + * Subclasses of message denote a computational task to be performed in the "other {@link JVM}". + * + * @param type of the return value + */ + public abstract static class Message { + + final Class replyType; + + /** + * Constructor for subclasses. Use it as {@code super(Integer.class)} to specify the reply type + * of the exception which is then returned from the {@link #evaluate} method. + * + * @param replyType the type of the reply + */ + protected Message(Class replyType) { + this.replyType = replyType; + } + + /** + * Handles evaluation of the exception. Use {@link Channel#execute} to pass this messages to the + * other {@link JVM}. After all the serde and transfer to the other {@link JVM} this method is + * executed to perform its operation. Then the result is passed back via serde again and + * returned from the {@link Channel#execute} method. + * + * @param channel allows sending messages to the other JVM + * @return the result of the evaluation or {@code null} + * @throws Throwable the computation may yield exceptions or errors which are then transferred + * back to the callee JVM + */ + protected abstract R evaluate(Channel channel) throws Throwable; + } +} diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java index e49617ab9f88..a9621859d948 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java @@ -3,18 +3,13 @@ import java.io.File; import java.io.IOException; import java.lang.foreign.Arena; -import java.lang.foreign.FunctionDescriptor; -import java.lang.foreign.Linker; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Function; import org.enso.common.Platform; import org.enso.persist.Persistance; -import org.graalvm.nativeimage.CurrentIsolate; -import org.graalvm.nativeimage.ImageInfo; import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.UnmanagedMemory; @@ -71,7 +66,7 @@ public static JVM create(File javaHome, String... options) { } static R executeImpl( - Persistance.Pool pool, Message msg, Function send) { + Persistance.Pool pool, Channel.Message msg, Function send) { try (var arena = Arena.ofConfined()) { var bytes = pool.write(msg, null); var memory = arena.allocate(Math.max(bytes.length, 4096)); @@ -104,127 +99,6 @@ private static long acceptRequestFromHotSpotJvm( CCharPointer.class, long.class); - /** Channel connects two JVMs. */ - public static final class Channel { - /** persistance pool associated with this channel object */ - private final Persistance.Pool pool; - - private final JNI.JNIEnv env; - private final long isolate; - private final long callbackFn; - - /* private */ Channel(Persistance.Pool pool, JNI.JNIEnv env) { - this.pool = pool; - this.env = env; - this.isolate = -1; - this.callbackFn = -1; - } - - /* private */ Channel(Persistance.Pool pool, long isolate, long callbackFn) { - if (ImageInfo.inImageCode()) { - throw new IllegalStateException("Only usable in HotSpot"); - } - this.pool = pool; - this.env = null; - this.isolate = isolate; - this.callbackFn = callbackFn; - } - - /** - * Executes a message in the other JVM. The message is any subclass of {@link Message} - * registered for persistance via {@link Persistable @Persistable} annotation into the {@link - * Persistance.Pool pool associated with this JVM}. The result (which is of type {@code R}) also - * has to be registered for serde. - * - * @param msg the message that gets serialized, transfered into the other JVM, deserialized on - * the other side and {@link Message#evaluate() evaluated} there - * @param the type of result we expect the message to return - * @return the value gets computed via {@link Message#evaluate()} in the other JVM and then it - * gets serialized and transfered back to us. Deserialized and the value is then returned - * from this method - */ - public final R execute(Message msg) { - if (this.isolate == -1) { - return executeImpl(pool, msg, (memory) -> toHotSpotMessage(env, memory)); - } else { - var fnCallbackAddress = MemorySegment.ofAddress(callbackFn); - var fnDescriptor = - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG); - var fnHandle = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); - return executeImpl( - pool, - msg, - (seg) -> { - Object res = -1L; - try { - var isoRef = MemorySegment.ofAddress(isolate); - res = fnHandle.invoke(isoRef, seg, seg.byteSize()); - } catch (Throwable ex) { - ex.printStackTrace(); - } - return (long) res; - }); - } - } - - private static long toHotSpotMessage(JNI.JNIEnv e, MemorySegment segment) { - var classNameWithSlashes = "org/enso/os/environment/jni/JVMPeer"; - var methodName = "handle"; - try (var classInC = CTypeConversion.toCString(classNameWithSlashes); - var methodInC = CTypeConversion.toCString(methodName); - var signatureInC = CTypeConversion.toCString("(JJJJ)J"); ) { - var fn = e.getFunctions(); - var clazz = fn.getFindClass().call(e, classInC.get()); - assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; - var method = fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); - assert method.isNonNull() : "method not found in " + classNameWithSlashes; - var address = segment.address(); - assert address > 0 : "We need an address"; - var arg = StackValue.get(4, JNI.JValue.class); - arg.addressOf(0).setLong(CurrentIsolate.getCurrentThread().rawValue()); - arg.addressOf(1).setLong(CALLBACK_FN.getFunctionPointer().rawValue()); - arg.addressOf(2).setLong(address); - arg.addressOf(3).setLong(segment.byteSize()); - var replySize = fn.getCallStaticLongMethodA().call(e, clazz, method, arg); - return replySize; - } - } - } - - /** - * Subclasses of message denote a computational task to be performed in the "other JVM". - * - * @param type of the return value - */ - public abstract static class Message { - private final Class replyType; - - /** - * Constructor for subclasses. Use it as {@code super(Integer.class)} to specify the reply type - * of the exception which is then returned from the {@link - * #execute(org.enso.os.environment.jni.JVM.Message)} method. - * - * @param replyType the type of the reply - */ - protected Message(Class replyType) { - this.replyType = replyType; - } - - /** - * Evaluates the exception. Invoked in the other JVM. - * - * @param channel allows sending messages to the other JVM - * @return the result of the evaluation or {@code null} - * @throws Throwable the computation may yield exceptions or errors which are then transferred - * back to the callee JVM - */ - protected abstract R evaluate(Channel channel) throws Throwable; - } - /** * Executes main method of provided class * @@ -233,7 +107,7 @@ protected Message(Class replyType) { */ public final void executeMain(String classNameWithSlashes, String... args) { var msg = new JVMPeer.ExecuteMainClass(classNameWithSlashes, List.of(args)); - var channel = new Channel(JVMPeer.POOL, env()); + org.enso.os.environment.jni.Channel channel = new Channel(JVMPeer.POOL, env()); channel.execute(msg); } diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index d85232524f67..d00e8bb3398e 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -6,7 +6,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.enso.os.environment.jni.JVM.Channel; import org.enso.persist.Persistable; import org.enso.persist.Persistance; @@ -18,7 +17,7 @@ private JVMPeer() {} /** called via JNI from SVM to HotSpot */ static long handle(long threadId, long callbackFn, long address, long size) { try { - var channel = new JVM.Channel(JVMPeer.POOL, threadId, callbackFn); + var channel = new Channel(JVMPeer.POOL, threadId, callbackFn); return handleWithChannel(channel, address, size); } catch (Throwable t) { // TBD: proper handling of exceptions is needed @@ -31,7 +30,7 @@ static long handleWithChannel(Channel channel, long address, long size) throws T var seg = MemorySegment.ofAddress(address).reinterpret(size); var buf = seg.asByteBuffer(); var ref = POOL.read(buf, null); - var msg = ref.get(JVM.Message.class); + var msg = ref.get(Channel.Message.class); var res = msg.evaluate(channel); var bytes = Persistables.POOL.write(res, null); seg.copyFrom(MemorySegment.ofArray(bytes)); @@ -39,7 +38,7 @@ static long handleWithChannel(Channel channel, long address, long size) throws T } @Persistable(id = 432001) - public static final class ExecuteMainClass extends JVM.Message { + public static final class ExecuteMainClass extends Channel.Message { private final String mainClassWithSlashes; private final List args; diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 820a2c295ff4..ac48486c705e 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -8,7 +8,6 @@ import java.math.BigInteger; import java.util.HashMap; import java.util.Map; -import org.enso.os.environment.jni.JVM.Channel; import org.enso.persist.Persistable; final class TestMain { @@ -59,7 +58,7 @@ private static void reportResultToSvmIsolate( } @Persistable(id = 430607) - static final class RequestFactorial extends JVM.Message { + static final class RequestFactorial extends Channel.Message { private long n; RequestFactorial(long n) { @@ -80,7 +79,7 @@ long n() { } @Persistable(id = 430608) - static final class ComputeFactorial extends JVM.Message { + static final class ComputeFactorial extends Channel.Message { private long n; ComputeFactorial(long n) { @@ -100,7 +99,7 @@ long n() { } @Persistable(id = 430606) - static final class ReportResult extends JVM.Message { + static final class ReportResult extends Channel.Message { private final long key; private final String value; @@ -128,7 +127,7 @@ protected Void evaluate(Channel otherVM) throws Throwable { } @Persistable(id = 430609) - static final class CountDownAndReturn extends JVM.Message { + static final class CountDownAndReturn extends Channel.Message { private final long value; private final long acc; diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index c49f5e8e1a6e..c83b3adedae6 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -138,7 +138,7 @@ public void executeMainClass() throws Exception { public void computeFactorialViaMessages() throws Exception { TestMain.CORRECT_RESULTS.clear(); assertEquals("Results are empty", 0, TestMain.CORRECT_RESULTS.size()); - var channel = new JVM.Channel(JVMPeer.POOL, env()); + var channel = new Channel(JVMPeer.POOL, env()); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { @@ -155,7 +155,7 @@ public void computeFactorialViaMessages() throws Exception { @Test public void computeFactorialViaSingleMessage() throws Exception { - var channel = new JVM.Channel(JVMPeer.POOL, env()); + var channel = new Channel(JVMPeer.POOL, env()); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { @@ -168,14 +168,14 @@ public void computeFactorialViaSingleMessage() throws Exception { @Test public void backAndForthFactorialOne() throws Exception { - var channel = new JVM.Channel(JVMPeer.POOL, env()); + var channel = new Channel(JVMPeer.POOL, env()); var fac = channel.execute(new TestMain.CountDownAndReturn(1, 1)); assertEquals(1, fac.longValue()); } @Test public void backAndForthFactorialTwo() throws Exception { - var channel = new JVM.Channel(JVMPeer.POOL, env()); + var channel = new Channel(JVMPeer.POOL, env()); var fac = channel.execute(new TestMain.CountDownAndReturn(2, 1)); assertEquals(2, fac.longValue()); } @@ -183,7 +183,7 @@ public void backAndForthFactorialTwo() throws Exception { @Ignore @Test public void backAndForthFactorialThree() throws Exception { - var channel = new JVM.Channel(JVMPeer.POOL, env()); + var channel = new Channel(JVMPeer.POOL, env()); var fac = channel.execute(new TestMain.CountDownAndReturn(3, 1)); assertEquals(6, fac.longValue()); } @@ -191,7 +191,7 @@ public void backAndForthFactorialThree() throws Exception { @Ignore @Test public void backAndForthFactorialFour() throws Exception { - var channel = new JVM.Channel(JVMPeer.POOL, env()); + var channel = new Channel(JVMPeer.POOL, env()); var fac = channel.execute(new TestMain.CountDownAndReturn(4, 1)); assertEquals(24, fac.longValue()); } @@ -199,7 +199,7 @@ public void backAndForthFactorialFour() throws Exception { @Ignore @Test public void backAndForthFactorialFive() throws Exception { - var channel = new JVM.Channel(JVMPeer.POOL, env()); + var channel = new Channel(JVMPeer.POOL, env()); var fac = channel.execute(new TestMain.CountDownAndReturn(5, 1)); assertEquals(120, fac.longValue()); } From 32aa5b06af1e53fd05709c49305d56c860af0880 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 07:23:13 +0200 Subject: [PATCH 12/26] Handling executeMain the old way to avoid access checks associated with reflection --- .../java/org/enso/os/environment/jni/JVM.java | 28 ++++++++++++++--- .../org/enso/os/environment/jni/JVMPeer.java | 30 ------------------- 2 files changed, 24 insertions(+), 34 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java index a9621859d948..825214364f6f 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java @@ -6,7 +6,6 @@ import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.function.Function; import org.enso.common.Platform; import org.enso.persist.Persistance; @@ -106,9 +105,30 @@ private static long acceptRequestFromHotSpotJvm( * @param args arguments to pass to the main method */ public final void executeMain(String classNameWithSlashes, String... args) { - var msg = new JVMPeer.ExecuteMainClass(classNameWithSlashes, List.of(args)); - org.enso.os.environment.jni.Channel channel = new Channel(JVMPeer.POOL, env()); - channel.execute(msg); + var e = env(); + try (var className = CTypeConversion.toCString(classNameWithSlashes); + var mainName = CTypeConversion.toCString("main"); + var stringName = CTypeConversion.toCString("java/lang/String"); + var mainSig = CTypeConversion.toCString("([Ljava/lang/String;)V"); ) { + var fn = e.getFunctions(); + var mainClazz = fn.getFindClass().call(e, className.get()); + assert mainClazz.isNonNull() : "Class not found " + classNameWithSlashes; + var mainMethod = fn.getGetStaticMethodID().call(e, mainClazz, mainName.get(), mainSig.get()); + assert mainMethod.isNonNull() : "main method found in " + classNameWithSlashes; + var stringClazz = fn.getFindClass().call(e, stringName.get()); + var argsCopy = + fn.getNewObjectArray().call(e, args.length, stringClazz, WordFactory.nullPointer()); + + for (var i = 0; i < args.length; i++) { + try (var ithArg = CTypeConversion.toCString(args[i]); ) { + var str = fn.getNewStringUTF().call(e, ithArg.get()); + fn.getSetObjectArrayElement().call(e, argsCopy, i, str); + } + } + var arg = StackValue.get(JNI.JValue.class); + arg.setJObject(argsCopy); + fn.getCallStaticVoidMethodA().call(e, mainClazz, mainMethod, arg); + } } /** diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index d00e8bb3398e..f42ab95aceba 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -4,7 +4,6 @@ import java.lang.foreign.MemorySegment; import java.math.BigInteger; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import org.enso.persist.Persistable; import org.enso.persist.Persistance; @@ -37,35 +36,6 @@ static long handleWithChannel(Channel channel, long address, long size) throws T return bytes.length; } - @Persistable(id = 432001) - public static final class ExecuteMainClass extends Channel.Message { - private final String mainClassWithSlashes; - private final List args; - - public ExecuteMainClass(String mainClassWithSlashes, List args) { - super(Void.class); - this.mainClassWithSlashes = mainClassWithSlashes; - this.args = args; - } - - public String mainClassWithSlashes() { - return mainClassWithSlashes; - } - - public List args() { - return Collections.unmodifiableList(args); - } - - @Override - protected final Void evaluate(Channel notNeeded) throws Exception { - var clazz = Class.forName(mainClassWithSlashes.replace('/', '.')); - var method = clazz.getDeclaredMethod("main", String[].class); - method.setAccessible(true); - method.invoke(null, (Object) args.toArray(new String[args.size()])); - return null; - } - } - @Persistable(id = 432002) public static final class PersistList extends Persistance { public PersistList() { From 22de1c49b8f8b62176afaeef04665ff903022e0c Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 09:17:27 +0200 Subject: [PATCH 13/26] Removing usage of NetBeans Lookup from benchmarks --- build.sbt | 3 +-- lib/scala/bench-processor/src/main/java/module-info.java | 1 - .../java/org/enso/benchmarks/processor/BenchProcessor.java | 3 --- .../logging-service-logback/src/test/java/module-info.java | 1 - 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/build.sbt b/build.sbt index e993e90f07a6..bb42d6530424 100644 --- a/build.sbt +++ b/build.sbt @@ -4381,8 +4381,7 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) "org.netbeans.modules.openide.util.ServiceProviderProcessor" )), Compile / moduleDependencies := Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion ), Compile / internalModuleDependencies := Seq( (`engine-common` / Compile / exportedModule).value, diff --git a/lib/scala/bench-processor/src/main/java/module-info.java b/lib/scala/bench-processor/src/main/java/module-info.java index 853fec063934..b307254cf889 100644 --- a/lib/scala/bench-processor/src/main/java/module-info.java +++ b/lib/scala/bench-processor/src/main/java/module-info.java @@ -5,7 +5,6 @@ requires org.enso.polyglot.api; requires org.enso.runtime; requires org.graalvm.polyglot; - requires org.openide.util.lookup.RELEASE180; exports org.enso.benchmarks; exports org.enso.benchmarks.processor; diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index fbadc65f4a4b..48b671438b74 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -16,17 +16,14 @@ import java.util.function.BiConsumer; import javax.annotation.processing.AbstractProcessor; import javax.annotation.processing.ProcessingEnvironment; -import javax.annotation.processing.Processor; import javax.annotation.processing.RoundEnvironment; import javax.annotation.processing.SupportedAnnotationTypes; import javax.lang.model.SourceVersion; import javax.lang.model.element.TypeElement; import javax.tools.Diagnostic.Kind; import org.enso.benchmarks.Utils; -import org.openide.util.lookup.ServiceProvider; @SupportedAnnotationTypes("org.enso.benchmarks.processor.GenerateBenchSources") -@ServiceProvider(service = Processor.class) public class BenchProcessor extends AbstractProcessor { private static final String MODULE_PATH_PROP_NAME = diff --git a/lib/scala/logging-service-logback/src/test/java/module-info.java b/lib/scala/logging-service-logback/src/test/java/module-info.java index c44701510d2f..3bc35804aeb7 100644 --- a/lib/scala/logging-service-logback/src/test/java/module-info.java +++ b/lib/scala/logging-service-logback/src/test/java/module-info.java @@ -8,7 +8,6 @@ requires org.enso.logging.service.logback; requires org.enso.logging.config; requires org.slf4j; - requires org.openide.util.lookup.RELEASE180; provides SLF4JServiceProvider with org.enso.logging.service.logback.test.provider.TestLogProvider; From 5e1a8e7e5793c68c8be09ee2aced255ef43120e6 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 09:46:37 +0200 Subject: [PATCH 14/26] Removing org-openide-util-lookup from componentModulesPaths to fix JPMSUtils warnings --- build.sbt | 3 +-- project/JPMSUtils.scala | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/build.sbt b/build.sbt index bb42d6530424..1e386699f462 100644 --- a/build.sbt +++ b/build.sbt @@ -709,7 +709,6 @@ lazy val componentModulesPaths = jline ++ slf4jApi ++ Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, "org.netbeans.api" % "org-netbeans-modules-sampler" % netbeansApiVersion, "com.google.flatbuffers" % "flatbuffers-java" % flatbuffersVersion, "com.google.protobuf" % "protobuf-java" % googleProtobufVersion, @@ -4381,7 +4380,7 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) "org.netbeans.modules.openide.util.ServiceProviderProcessor" )), Compile / moduleDependencies := Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion ), Compile / internalModuleDependencies := Seq( (`engine-common` / Compile / exportedModule).value, diff --git a/project/JPMSUtils.scala b/project/JPMSUtils.scala index 56c4cb246b57..c525e6865a6f 100644 --- a/project/JPMSUtils.scala +++ b/project/JPMSUtils.scala @@ -42,31 +42,32 @@ object JPMSUtils { ): Def.Classpath = { val distinctModules = modules.distinct - val ret = cp.filter(dep => { + val foundFiles = cp.filter(dep => { val moduleID = dep.metadata.get(AttributeKey[ModuleID]("moduleID")).get shouldFilterModule(distinctModules, scalaBinaryVersion)(moduleID) }) if (shouldContainAll) { - if (ret.size < distinctModules.size) { + if (foundFiles.size < distinctModules.size) { log.error( s"[JPMSUtils/$projName] Not all modules from classpath were found" ) - log.error( - s"[JPMSUtils/$projName] Ensure libraryDependencies and moduleDependencies are correct" + log.debug( + s"[JPMSUtils/$projName] Returned (${foundFiles.size}): $foundFiles" ) - log.error(s"[JPMSUtils/$projName] Returned (${ret.size}): $ret") - log.error( + log.debug( s"[JPMSUtils/$projName] Expected: (${distinctModules.size}): $distinctModules" ) - val names = ret.map(f => { + val names = foundFiles.map(f => { val i = f.data.getName.lastIndexOf("-") f.data.getName.substring(0, i) }) - log.error("diff: " + distinctModules.map(_.name).diff(names)) + log.error( + s"[JPMSUtils/$projName] Ensure libraryDependencies and moduleDependencies are correct (${foundFiles.size} != ${distinctModules.size}): ${distinctModules.map(_.name).diff(names)}" + ) } } - ret + foundFiles } /** Filters all the requested modules from the given [[UpdateReport]]. @@ -97,15 +98,19 @@ object JPMSUtils { log.error( s"[JPMSUtils/$projName] Not all modules from update were found" ) - log.error( - s"[JPMSUtils/$projName] Ensure libraryDependencies and moduleDependencies are correct" - ) - log.error( + log.debug( s"[JPMSUtils/$projName] Returned (${foundFiles.size}): $foundFiles" ) - log.error( + log.debug( s"[JPMSUtils/$projName] Expected: (${distinctModules.size}): $distinctModules" ) + val names = foundFiles.map(f => { + val i = f.getName.lastIndexOf("-") + f.getName.substring(0, i) + }) + log.error( + s"[JPMSUtils/$projName] Ensure libraryDependencies and moduleDependencies are correct (${foundFiles.size} != ${distinctModules.size}): ${distinctModules.map(_.name).diff(names)}" + ) } } foundFiles From e28e6eabdc158878a2b4a62a2274b797b5f30178 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 12:33:34 +0200 Subject: [PATCH 15/26] Allocate both peers of the Channel at once --- .../org/enso/os/environment/jni/Channel.java | 117 +++++++++++++----- .../org/enso/os/environment/jni/JVMPeer.java | 12 -- .../os/environment/jni/LoadClassTest.java | 14 +-- 3 files changed, 95 insertions(+), 48 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index 31365a8487b3..7ad168c6eacc 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -4,6 +4,8 @@ import java.lang.foreign.Linker; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.util.HashMap; +import java.util.Map; import org.enso.persist.Persistance; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageInfo; @@ -11,50 +13,101 @@ import org.graalvm.nativeimage.c.type.CTypeConversion; /** Channel connects two {@link JVM} instances. */ -public final class Channel { +public final class Channel implements AutoCloseable { + /** + * @GuardedBy("Channel.class") + */ + private static final Map ID_TO_CHANNEL = new HashMap<>(); + + /** + * @GuardedBy("Channel.class") + */ + private static long idCounter = 1; /** persistance pool associated with this channel object */ private final Persistance.Pool pool; + private final long id; private final JNI.JNIEnv env; private final long isolate; private final long callbackFn; - /* private */ - Channel(Persistance.Pool pool, JNI.JNIEnv env) { + /** The SubstrateVM side of a channel. */ + private Channel(long id, Persistance.Pool pool, JNI.JNIEnv env) { + this.id = id; this.pool = pool; this.env = env; this.isolate = -1; this.callbackFn = -1; } - /* private */ - Channel(Persistance.Pool pool, long isolate, long callbackFn) { + /** The HotSpot JVM side of a channel. */ + private Channel(long id, Persistance.Pool pool, long isolate, long callbackFn) { if (ImageInfo.inImageCode()) { throw new IllegalStateException("Only usable in HotSpot"); } + this.id = id; this.pool = pool; this.env = null; this.isolate = isolate; this.callbackFn = callbackFn; } + /** + * Factory method to initialize the Channel in the SubstrateVM. + * + * @param e + * @return + */ + static synchronized Channel create(JNI.JNIEnv e) { + var id = idCounter++; + + var classNameWithSlashes = "org/enso/os/environment/jni/Channel"; + var methodName = "createJvmPeerChannel"; + try (var classInC = CTypeConversion.toCString(classNameWithSlashes); + var methodInC = CTypeConversion.toCString(methodName); + var signatureInC = CTypeConversion.toCString("(JJJ)Z")) { + var fn = e.getFunctions(); + var clazz = fn.getFindClass().call(e, classInC.get()); + assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; + var method = fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); + assert method.isNonNull() : "method not found in " + classNameWithSlashes; + var arg = StackValue.get(2, JNI.JValue.class); + arg.addressOf(0).setLong(id); + arg.addressOf(1).setLong(CurrentIsolate.getCurrentThread().rawValue()); + arg.addressOf(2).setLong(JVM.CALLBACK_FN.getFunctionPointer().rawValue()); + var replyOk = fn.getCallStaticBooleanMethodA().call(e, clazz, method, arg); + assert replyOk : "Failed to create peer in HotSpot JVM"; + + var channel = new Channel(id, JVMPeer.POOL, e); + ID_TO_CHANNEL.put(id, channel); + return channel; + } + } + + /** Allocates new channel with given ID in the HotSpot VM. Called via JNI/foreign interface. */ + private static boolean createJvmPeerChannel(long id, long threadId, long callbackFn) { + var channel = new Channel(id, JVMPeer.POOL, threadId, callbackFn); + var prev = ID_TO_CHANNEL.put(id, channel); + return prev == null; + } + /** * Executes a message in the other JVM. The message is any subclass of {@link Message} * registered for persistance via {@link Persistable @Persistable} annotation into the {@link * Persistance.Pool pool associated with this JVM}. The result (which is of type {@code R}) also * has to be registered for serde. * - * @param msg the message that gets serialized, transfered into the other JVM, deserialized on the - * other side and {@link Message#evaluate() evaluated} there + * @param msg the message that gets serialized, transferred into the other JVM, deserialized on + * the other side and {@link Message#evaluate() evaluated} there * @param the type of result we expect the message to return * @return the value gets computed via {@link Message#evaluate()} in the other JVM and then it - * gets serialized and transfered back to us. Deserialized and the value is then returned from - * this method + * gets serialized and transferred back to us. Deserialized and the value is then returned + * from this method */ public final R execute(Message msg) { if (this.isolate == -1) { - return JVM.executeImpl(pool, msg, memory -> toHotSpotMessage(env, memory)); + return JVM.executeImpl(pool, msg, memory -> toHotSpotMessage(env, id, memory)); } else { java.lang.foreign.MemorySegment fnCallbackAddress = MemorySegment.ofAddress(callbackFn); java.lang.foreign.FunctionDescriptor fnDescriptor = @@ -81,33 +134,39 @@ public final R execute(Message msg) { } } - private static long toHotSpotMessage(JNI.JNIEnv e, MemorySegment segment) { - java.lang.String classNameWithSlashes = "org/enso/os/environment/jni/JVMPeer"; - java.lang.String methodName = "handle"; - try (org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder classInC = - CTypeConversion.toCString(classNameWithSlashes); - org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder methodInC = - CTypeConversion.toCString(methodName); - org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder signatureInC = - CTypeConversion.toCString("(JJJJ)J")) { - org.enso.os.environment.jni.JNINativeInterface fn = e.getFunctions(); - org.enso.os.environment.jni.JNI.JClass clazz = fn.getFindClass().call(e, classInC.get()); + private static long toHotSpotMessage(JNI.JNIEnv e, long id, MemorySegment segment) { + var classNameWithSlashes = "org/enso/os/environment/jni/Channel"; + var methodName = "handleJvmMessage"; + try (var classInC = CTypeConversion.toCString(classNameWithSlashes); + var methodInC = CTypeConversion.toCString(methodName); + var signatureInC = CTypeConversion.toCString("(JJJ)J")) { + var fn = e.getFunctions(); + var clazz = fn.getFindClass().call(e, classInC.get()); assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; - org.enso.os.environment.jni.JNI.JMethodID method = - fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); + var method = fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); assert method.isNonNull() : "method not found in " + classNameWithSlashes; long address = segment.address(); assert address > 0 : "We need an address"; - org.enso.os.environment.jni.JNI.JValue arg = StackValue.get(4, JNI.JValue.class); - arg.addressOf(0).setLong(CurrentIsolate.getCurrentThread().rawValue()); - arg.addressOf(1).setLong(JVM.CALLBACK_FN.getFunctionPointer().rawValue()); - arg.addressOf(2).setLong(address); - arg.addressOf(3).setLong(segment.byteSize()); - long replySize = fn.getCallStaticLongMethodA().call(e, clazz, method, arg); + var arg = StackValue.get(3, JNI.JValue.class); + arg.addressOf(0).setLong(id); + arg.addressOf(1).setLong(address); + arg.addressOf(2).setLong(segment.byteSize()); + var replySize = fn.getCallStaticLongMethodA().call(e, clazz, method, arg); return replySize; } } + private static long handleJvmMessage(long id, long address, long size) throws Throwable { + var channel = ID_TO_CHANNEL.get(id); + return JVMPeer.handleWithChannel(channel, address, size); + } + + @Override + public void close() throws Exception { + ID_TO_CHANNEL.remove(id, this); + // TBD remove on the peer as well + } + /** * Subclasses of message denote a computational task to be performed in the "other {@link JVM}". * diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index f42ab95aceba..8fe44955dba2 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -13,18 +13,6 @@ final class JVMPeer { private JVMPeer() {} - /** called via JNI from SVM to HotSpot */ - static long handle(long threadId, long callbackFn, long address, long size) { - try { - var channel = new Channel(JVMPeer.POOL, threadId, callbackFn); - return handleWithChannel(channel, address, size); - } catch (Throwable t) { - // TBD: proper handling of exceptions is needed - t.printStackTrace(); - return -1; - } - } - static long handleWithChannel(Channel channel, long address, long size) throws Throwable { var seg = MemorySegment.ofAddress(address).reinterpret(size); var buf = seg.asByteBuffer(); diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index c83b3adedae6..1ffa6d7a04d2 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -138,7 +138,7 @@ public void executeMainClass() throws Exception { public void computeFactorialViaMessages() throws Exception { TestMain.CORRECT_RESULTS.clear(); assertEquals("Results are empty", 0, TestMain.CORRECT_RESULTS.size()); - var channel = new Channel(JVMPeer.POOL, env()); + var channel = Channel.create(env()); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { @@ -155,7 +155,7 @@ public void computeFactorialViaMessages() throws Exception { @Test public void computeFactorialViaSingleMessage() throws Exception { - var channel = new Channel(JVMPeer.POOL, env()); + var channel = Channel.create(env()); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { @@ -168,14 +168,14 @@ public void computeFactorialViaSingleMessage() throws Exception { @Test public void backAndForthFactorialOne() throws Exception { - var channel = new Channel(JVMPeer.POOL, env()); + var channel = Channel.create(env()); var fac = channel.execute(new TestMain.CountDownAndReturn(1, 1)); assertEquals(1, fac.longValue()); } @Test public void backAndForthFactorialTwo() throws Exception { - var channel = new Channel(JVMPeer.POOL, env()); + var channel = Channel.create(env()); var fac = channel.execute(new TestMain.CountDownAndReturn(2, 1)); assertEquals(2, fac.longValue()); } @@ -183,7 +183,7 @@ public void backAndForthFactorialTwo() throws Exception { @Ignore @Test public void backAndForthFactorialThree() throws Exception { - var channel = new Channel(JVMPeer.POOL, env()); + var channel = Channel.create(env()); var fac = channel.execute(new TestMain.CountDownAndReturn(3, 1)); assertEquals(6, fac.longValue()); } @@ -191,7 +191,7 @@ public void backAndForthFactorialThree() throws Exception { @Ignore @Test public void backAndForthFactorialFour() throws Exception { - var channel = new Channel(JVMPeer.POOL, env()); + var channel = Channel.create(env()); var fac = channel.execute(new TestMain.CountDownAndReturn(4, 1)); assertEquals(24, fac.longValue()); } @@ -199,7 +199,7 @@ public void backAndForthFactorialFour() throws Exception { @Ignore @Test public void backAndForthFactorialFive() throws Exception { - var channel = new Channel(JVMPeer.POOL, env()); + var channel = Channel.create(env()); var fac = channel.execute(new TestMain.CountDownAndReturn(5, 1)); assertEquals(120, fac.longValue()); } From 713383007f5c5e3280e26dd36b420a138eef048b Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 14:32:48 +0200 Subject: [PATCH 16/26] Look a Channel peer up by its ID --- .../org/enso/os/environment/jni/Channel.java | 75 ++++++++++++++++--- .../java/org/enso/os/environment/jni/JVM.java | 44 ----------- .../org/enso/os/environment/jni/JVMPeer.java | 12 --- .../org/enso/os/environment/jni/TestMain.java | 42 ++--------- .../os/environment/jni/LoadClassTest.java | 22 ++---- 5 files changed, 80 insertions(+), 115 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index 7ad168c6eacc..e65af8021df3 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -1,15 +1,23 @@ package org.enso.os.environment.jni; +import java.io.IOException; +import java.lang.foreign.Arena; import java.lang.foreign.FunctionDescriptor; import java.lang.foreign.Linker; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; import org.enso.persist.Persistance; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageInfo; +import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.function.CEntryPoint; +import org.graalvm.nativeimage.c.function.CEntryPointLiteral; +import org.graalvm.nativeimage.c.function.CFunctionPointer; +import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; /** Channel connects two {@link JVM} instances. */ @@ -75,7 +83,7 @@ static synchronized Channel create(JNI.JNIEnv e) { var arg = StackValue.get(2, JNI.JValue.class); arg.addressOf(0).setLong(id); arg.addressOf(1).setLong(CurrentIsolate.getCurrentThread().rawValue()); - arg.addressOf(2).setLong(JVM.CALLBACK_FN.getFunctionPointer().rawValue()); + arg.addressOf(2).setLong(CALLBACK_FN.getFunctionPointer().rawValue()); var replyOk = fn.getCallStaticBooleanMethodA().call(e, clazz, method, arg); assert replyOk : "Failed to create peer in HotSpot JVM"; @@ -107,25 +115,25 @@ private static boolean createJvmPeerChannel(long id, long threadId, long callbac */ public final R execute(Message msg) { if (this.isolate == -1) { - return JVM.executeImpl(pool, msg, memory -> toHotSpotMessage(env, id, memory)); + return executeImpl(pool, msg, memory -> toHotSpotMessage(env, id, memory)); } else { - java.lang.foreign.MemorySegment fnCallbackAddress = MemorySegment.ofAddress(callbackFn); - java.lang.foreign.FunctionDescriptor fnDescriptor = + var fnCallbackAddress = MemorySegment.ofAddress(callbackFn); + var fnDescriptor = FunctionDescriptor.of( ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG); - java.lang.invoke.MethodHandle fnHandle = - Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); - return JVM.executeImpl( + var fnHandle = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); + return executeImpl( pool, msg, seg -> { Object res = -1L; try { - java.lang.foreign.MemorySegment isoRef = MemorySegment.ofAddress(isolate); - res = fnHandle.invoke(isoRef, seg, seg.byteSize()); + var isoRef = MemorySegment.ofAddress(isolate); + res = fnHandle.invoke(isoRef, id, seg, seg.byteSize()); } catch (Throwable ex) { ex.printStackTrace(); } @@ -134,6 +142,35 @@ public final R execute(Message msg) { } } + private static final CEntryPointLiteral CALLBACK_FN = + CEntryPointLiteral.create( + Channel.class, + "acceptRequestFromHotSpotJvm", + IsolateThread.class, + long.class, + CCharPointer.class, + long.class); + + @CEntryPoint + private static long acceptRequestFromHotSpotJvm( + IsolateThread threadId, long id, CCharPointer data, long size) throws Throwable { + var channel = ID_TO_CHANNEL.get(id); + assert channel != null : "There must be a channel " + id + " but " + ID_TO_CHANNEL; + var len = handleWithChannel(channel, data.rawValue(), size); + return len; + } + + private static long handleWithChannel(Channel channel, long address, long size) throws Throwable { + var seg = MemorySegment.ofAddress(address).reinterpret(size); + var buf = seg.asByteBuffer(); + var ref = JVMPeer.POOL.read(buf, null); + var msg = ref.get(Channel.Message.class); + var res = msg.evaluate(channel); + var bytes = Persistables.POOL.write(res, null); + seg.copyFrom(MemorySegment.ofArray(bytes)); + return bytes.length; + } + private static long toHotSpotMessage(JNI.JNIEnv e, long id, MemorySegment segment) { var classNameWithSlashes = "org/enso/os/environment/jni/Channel"; var methodName = "handleJvmMessage"; @@ -156,9 +193,27 @@ private static long toHotSpotMessage(JNI.JNIEnv e, long id, MemorySegment segmen } } + static R executeImpl( + Persistance.Pool pool, Channel.Message msg, Function send) { + try (var arena = Arena.ofConfined()) { + var bytes = pool.write(msg, null); + var memory = arena.allocate(Math.max(bytes.length, 4096)); + memory.copyFrom(MemorySegment.ofArray(bytes)); + long len = send.apply(memory); + assert len >= 0; + var reply = memory.asByteBuffer(); + reply.position(0); + reply.limit((int) len); + var result = pool.read(reply, null); + return result.get(msg.replyType); + } catch (IOException ex) { + throw new IllegalStateException(ex); + } + } + private static long handleJvmMessage(long id, long address, long size) throws Throwable { var channel = ID_TO_CHANNEL.get(id); - return JVMPeer.handleWithChannel(channel, address, size); + return handleWithChannel(channel, address, size); } @Override diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java index 825214364f6f..af55973123de 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVM.java @@ -1,22 +1,12 @@ package org.enso.os.environment.jni; import java.io.File; -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.Arrays; -import java.util.function.Function; import org.enso.common.Platform; -import org.enso.persist.Persistance; -import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.UnmanagedMemory; -import org.graalvm.nativeimage.c.function.CEntryPoint; -import org.graalvm.nativeimage.c.function.CEntryPointLiteral; -import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.word.WordFactory; @@ -64,40 +54,6 @@ public static JVM create(File javaHome, String... options) { return new JVM(createJvmFn, jvmArgs.toArray(new String[0])); } - static R executeImpl( - Persistance.Pool pool, Channel.Message msg, Function send) { - try (var arena = Arena.ofConfined()) { - var bytes = pool.write(msg, null); - var memory = arena.allocate(Math.max(bytes.length, 4096)); - memory.copyFrom(MemorySegment.ofArray(bytes)); - long len = send.apply(memory); - assert len >= 0; - var reply = memory.asByteBuffer(); - reply.position(0); - reply.limit((int) len); - var result = pool.read(reply, null); - return result.get(msg.replyType); - } catch (IOException ex) { - throw new IllegalStateException(ex); - } - } - - @CEntryPoint - private static long acceptRequestFromHotSpotJvm( - IsolateThread threadId, CCharPointer data, long size) throws Throwable { - // TBD: recursive calls will need to find a proper channel - var len = JVMPeer.handleWithChannel(null, data.rawValue(), size); - return len; - } - - static final CEntryPointLiteral CALLBACK_FN = - CEntryPointLiteral.create( - JVM.class, - "acceptRequestFromHotSpotJvm", - IsolateThread.class, - CCharPointer.class, - long.class); - /** * Executes main method of provided class * diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index 8fe44955dba2..07a3712775b6 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -1,7 +1,6 @@ package org.enso.os.environment.jni; import java.io.IOException; -import java.lang.foreign.MemorySegment; import java.math.BigInteger; import java.util.ArrayList; import java.util.List; @@ -13,17 +12,6 @@ final class JVMPeer { private JVMPeer() {} - static long handleWithChannel(Channel channel, long address, long size) throws Throwable { - var seg = MemorySegment.ofAddress(address).reinterpret(size); - var buf = seg.asByteBuffer(); - var ref = POOL.read(buf, null); - var msg = ref.get(Channel.Message.class); - var res = msg.evaluate(channel); - var bytes = Persistables.POOL.write(res, null); - seg.copyFrom(MemorySegment.ofArray(bytes)); - return bytes.length; - } - @Persistable(id = 432002) public static final class PersistList extends Persistance { public PersistList() { diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index ac48486c705e..8fc5a93f6d76 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -1,10 +1,7 @@ package org.enso.os.environment.jni; -import java.lang.foreign.FunctionDescriptor; -import java.lang.foreign.Linker; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.lang.invoke.MethodHandle; +import java.io.File; +import java.io.FileWriter; import java.math.BigInteger; import java.util.HashMap; import java.util.Map; @@ -15,17 +12,12 @@ final class TestMain { private TestMain() {} - static void main(String... args) throws Throwable { - var jvmIsolate = Long.parseLong(args[0]); - var fnCallbackAddress = MemorySegment.ofAddress(Long.parseLong(args[1])); - var fnDescriptor = - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG); - var fnHandle = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); - var n = Integer.parseInt(args[2]); - - var res = factorial(n).toString(); - reportResultToSvmIsolate(jvmIsolate, fnHandle, n, res); + public static void main(String... args) throws Exception { + var out = new File(args[0]); + var n = Integer.parseInt(args[1]); + try (java.io.FileWriter os = new FileWriter(out)) { + os.write(factorial(n).toString()); + } } static BigInteger factorial(long n) { @@ -39,24 +31,6 @@ static BigInteger factorial(long n) { return acc; } - private static void reportResultToSvmIsolate( - long jvmIsolate, MethodHandle fn, long key, String value) throws Throwable { - ReportResult rr = new ReportResult(key, value); - JVM.executeImpl( - JVMPeer.POOL, - rr, - (seg) -> { - Object res = -1L; - try { - var isoRef = MemorySegment.ofAddress(jvmIsolate); - res = fn.invoke(isoRef, seg, seg.byteSize()); - } catch (Throwable ex) { - ex.printStackTrace(); - } - return (long) res; - }); - } - @Persistable(id = 430607) static final class RequestFactorial extends Channel.Message { private long n; diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index 1ffa6d7a04d2..72cb10ff0472 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -4,9 +4,9 @@ import static org.junit.Assert.assertTrue; import java.io.File; +import java.nio.file.Files; import java.util.Random; import org.enso.os.environment.jni.JNI.JValue; -import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.junit.Ignore; @@ -115,22 +115,14 @@ public void setSystemProperty() { @Test public void executeMainClass() throws Exception { - var jvmIsolate = CurrentIsolate.getCurrentThread().rawValue(); - var callbackFn = JVM.CALLBACK_FN.getFunctionPointer().rawValue(); - TestMain.CORRECT_RESULTS.clear(); - assertEquals("Results are empty", 0, TestMain.CORRECT_RESULTS.size()); + var out = File.createTempFile("check-main", ".log"); var gen = new Random(); - var n = 0L; for (var i = 0; i < 5; i++) { - n += gen.nextLong(MIN, MAX); - var mainClass = "org/enso/os/environment/jni/TestMain"; - jvm().executeMain(mainClass, "" + jvmIsolate, "" + callbackFn, "" + n); - } - assertEquals( - "Five results found: " + TestMain.CORRECT_RESULTS, 5, TestMain.CORRECT_RESULTS.size()); - for (var e : TestMain.CORRECT_RESULTS.entrySet()) { - var expecting = TestMain.factorial(e.getKey()); - assertEquals("fac(" + e.getKey() + ") should be", expecting.toString(), e.getValue()); + var n = gen.nextInt(MIN, MAX); + jvm().executeMain("org/enso/os/environment/jni/TestMain", out.getPath(), "" + n); + var content = Files.readString(out.toPath()); + assertEquals("Factorial of " + n + " is the same", TestMain.factorial(n).toString(), content); + out.delete(); } } From 3f16786ee9050417bdce8cb3303ade74f7421a2b Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 15:16:59 +0200 Subject: [PATCH 17/26] Add debugging symbols when ENSO_LAUNCHER=debug --- build.sbt | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 1e386699f462..05a60562f9f7 100644 --- a/build.sbt +++ b/build.sbt @@ -4331,7 +4331,22 @@ lazy val `os-environment` = "--features=org.enso.os.environment.TestCollectorFeature", "-H:+ForeignAPISupport", "-R:-InstallSegfaultHandler" - ) + ) ++ (if (GraalVM.EnsoLauncher.debug) { + // useful perf & debug switches: + Seq( + "-g", + "-O0", + "-H:+SourceLevelDebug", + "-H:-DeleteLocalSymbols", + // you may need to set smallJdk := None to use following flags: + // "--trace-class-initialization=org.enso.syntax2.Parser", + // "--diagnostics-mode", + // "--verbose", + "-Dnic=nic" + ) + } else { + Seq() + }) ) }.value, Test / test := Def From b04963d7254cac6ae962410d2cc70bb38fe57c60 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 15:17:26 +0200 Subject: [PATCH 18/26] Multiple nested callbacks are possible --- .../org/enso/os/environment/jni/Channel.java | 75 ++++++++++--------- .../os/environment/jni/LoadClassTest.java | 4 - 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index e65af8021df3..fc2ab1128389 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -39,14 +39,23 @@ public final class Channel implements AutoCloseable { private final JNI.JNIEnv env; private final long isolate; private final long callbackFn; + private final JNI.JClass channelClass; + private final JNI.JMethodID channelHandle; /** The SubstrateVM side of a channel. */ - private Channel(long id, Persistance.Pool pool, JNI.JNIEnv env) { + private Channel( + long id, + Persistance.Pool pool, + JNI.JNIEnv env, + JNI.JClass handleClass, + JNI.JMethodID handleFn) { this.id = id; this.pool = pool; this.env = env; this.isolate = -1; this.callbackFn = -1; + this.channelClass = handleClass; + this.channelHandle = handleFn; } /** The HotSpot JVM side of a channel. */ @@ -56,9 +65,11 @@ private Channel(long id, Persistance.Pool pool, long isolate, long callbackFn) { } this.id = id; this.pool = pool; - this.env = null; this.isolate = isolate; this.callbackFn = callbackFn; + this.env = null; + this.channelClass = null; + this.channelHandle = null; } /** @@ -69,25 +80,30 @@ private Channel(long id, Persistance.Pool pool, long isolate, long callbackFn) { */ static synchronized Channel create(JNI.JNIEnv e) { var id = idCounter++; - - var classNameWithSlashes = "org/enso/os/environment/jni/Channel"; - var methodName = "createJvmPeerChannel"; + var classNameWithSlashes = Channel.class.getName().replace('.', '/'); try (var classInC = CTypeConversion.toCString(classNameWithSlashes); - var methodInC = CTypeConversion.toCString(methodName); - var signatureInC = CTypeConversion.toCString("(JJJ)Z")) { + var createInC = CTypeConversion.toCString("createJvmPeerChannel"); + var createSigInC = CTypeConversion.toCString("(JJJ)Z"); // + var handleInC = CTypeConversion.toCString("handleJvmMessage"); + var handleSigInC = CTypeConversion.toCString("(JJJ)J"); // + ) { var fn = e.getFunctions(); - var clazz = fn.getFindClass().call(e, classInC.get()); - assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; - var method = fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); - assert method.isNonNull() : "method not found in " + classNameWithSlashes; + var channelClass = fn.getFindClass().call(e, classInC.get()); + assert channelClass.isNonNull() : "Class not found " + classNameWithSlashes; + var createMethod = + fn.getGetStaticMethodID().call(e, channelClass, createInC.get(), createSigInC.get()); + assert createMethod.isNonNull() : "method not found in " + classNameWithSlashes; var arg = StackValue.get(2, JNI.JValue.class); arg.addressOf(0).setLong(id); arg.addressOf(1).setLong(CurrentIsolate.getCurrentThread().rawValue()); arg.addressOf(2).setLong(CALLBACK_FN.getFunctionPointer().rawValue()); - var replyOk = fn.getCallStaticBooleanMethodA().call(e, clazz, method, arg); + var replyOk = fn.getCallStaticBooleanMethodA().call(e, channelClass, createMethod, arg); assert replyOk : "Failed to create peer in HotSpot JVM"; - var channel = new Channel(id, JVMPeer.POOL, e); + var handleMethod = + fn.getGetStaticMethodID().call(e, channelClass, handleInC.get(), handleSigInC.get()); + + var channel = new Channel(id, JVMPeer.POOL, e, channelClass, handleMethod); ID_TO_CHANNEL.put(id, channel); return channel; } @@ -115,7 +131,7 @@ private static boolean createJvmPeerChannel(long id, long threadId, long callbac */ public final R execute(Message msg) { if (this.isolate == -1) { - return executeImpl(pool, msg, memory -> toHotSpotMessage(env, id, memory)); + return executeImpl(pool, msg, memory -> toHotSpotMessage(memory)); } else { var fnCallbackAddress = MemorySegment.ofAddress(callbackFn); var fnDescriptor = @@ -154,6 +170,7 @@ public final R execute(Message msg) { @CEntryPoint private static long acceptRequestFromHotSpotJvm( IsolateThread threadId, long id, CCharPointer data, long size) throws Throwable { + var channel = ID_TO_CHANNEL.get(id); assert channel != null : "There must be a channel " + id + " but " + ID_TO_CHANNEL; var len = handleWithChannel(channel, data.rawValue(), size); @@ -171,26 +188,16 @@ private static long handleWithChannel(Channel channel, long address, long size) return bytes.length; } - private static long toHotSpotMessage(JNI.JNIEnv e, long id, MemorySegment segment) { - var classNameWithSlashes = "org/enso/os/environment/jni/Channel"; - var methodName = "handleJvmMessage"; - try (var classInC = CTypeConversion.toCString(classNameWithSlashes); - var methodInC = CTypeConversion.toCString(methodName); - var signatureInC = CTypeConversion.toCString("(JJJ)J")) { - var fn = e.getFunctions(); - var clazz = fn.getFindClass().call(e, classInC.get()); - assert clazz.isNonNull() : "Class not found " + classNameWithSlashes; - var method = fn.getGetStaticMethodID().call(e, clazz, methodInC.get(), signatureInC.get()); - assert method.isNonNull() : "method not found in " + classNameWithSlashes; - long address = segment.address(); - assert address > 0 : "We need an address"; - var arg = StackValue.get(3, JNI.JValue.class); - arg.addressOf(0).setLong(id); - arg.addressOf(1).setLong(address); - arg.addressOf(2).setLong(segment.byteSize()); - var replySize = fn.getCallStaticLongMethodA().call(e, clazz, method, arg); - return replySize; - } + private long toHotSpotMessage(MemorySegment segment) { + var fn = env.getFunctions(); + long address = segment.address(); + assert address > 0 : "We need an address"; + var arg = StackValue.get(3, JNI.JValue.class); + arg.addressOf(0).setLong(id); + arg.addressOf(1).setLong(address); + arg.addressOf(2).setLong(segment.byteSize()); + var replySize = fn.getCallStaticLongMethodA().call(env, channelClass, channelHandle, arg); + return replySize; } static R executeImpl( diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index 72cb10ff0472..543688e72702 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -9,7 +9,6 @@ import org.enso.os.environment.jni.JNI.JValue; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.type.CTypeConversion; -import org.junit.Ignore; import org.junit.Test; public class LoadClassTest { @@ -172,7 +171,6 @@ public void backAndForthFactorialTwo() throws Exception { assertEquals(2, fac.longValue()); } - @Ignore @Test public void backAndForthFactorialThree() throws Exception { var channel = Channel.create(env()); @@ -180,7 +178,6 @@ public void backAndForthFactorialThree() throws Exception { assertEquals(6, fac.longValue()); } - @Ignore @Test public void backAndForthFactorialFour() throws Exception { var channel = Channel.create(env()); @@ -188,7 +185,6 @@ public void backAndForthFactorialFour() throws Exception { assertEquals(24, fac.longValue()); } - @Ignore @Test public void backAndForthFactorialFive() throws Exception { var channel = Channel.create(env()); From d11a0cff4b76c2cd99a1a6278b54e06206ab65fa Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 15:53:36 +0200 Subject: [PATCH 19/26] Allow implementation of Channel messages as records --- .../org/enso/os/environment/jni/Channel.java | 54 ++++--------- .../org/enso/os/environment/jni/TestMain.java | 77 +++---------------- .../os/environment/jni/LoadClassTest.java | 15 ++-- 3 files changed, 33 insertions(+), 113 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index fc2ab1128389..18b44afcc508 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -117,11 +117,13 @@ private static boolean createJvmPeerChannel(long id, long threadId, long callbac } /** - * Executes a message in the other JVM. The message is any subclass of {@link Message} + * Executes a message in the other JVM. The message is any subclass of {@link Function} * registered for persistance via {@link Persistable @Persistable} annotation into the {@link * Persistance.Pool pool associated with this JVM}. The result (which is of type {@code R}) also * has to be registered for serde. * + *

+ * * @param msg the message that gets serialized, transferred into the other JVM, deserialized on * the other side and {@link Message#evaluate() evaluated} there * @param the type of result we expect the message to return @@ -129,9 +131,9 @@ private static boolean createJvmPeerChannel(long id, long threadId, long callbac * gets serialized and transferred back to us. Deserialized and the value is then returned * from this method */ - public final R execute(Message msg) { + public final R execute(Class resultType, Function msg) { if (this.isolate == -1) { - return executeImpl(pool, msg, memory -> toHotSpotMessage(memory)); + return executeImpl(pool, resultType, msg, memory -> toHotSpotMessage(memory)); } else { var fnCallbackAddress = MemorySegment.ofAddress(callbackFn); var fnDescriptor = @@ -144,6 +146,7 @@ public final R execute(Message msg) { var fnHandle = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); return executeImpl( pool, + resultType, msg, seg -> { Object res = -1L; @@ -181,8 +184,9 @@ private static long handleWithChannel(Channel channel, long address, long size) var seg = MemorySegment.ofAddress(address).reinterpret(size); var buf = seg.asByteBuffer(); var ref = JVMPeer.POOL.read(buf, null); - var msg = ref.get(Channel.Message.class); - var res = msg.evaluate(channel); + var msg = ref.get(Function.class); + @SuppressWarnings("unchecked") + var res = msg.apply(channel); var bytes = Persistables.POOL.write(res, null); seg.copyFrom(MemorySegment.ofArray(bytes)); return bytes.length; @@ -201,7 +205,10 @@ private long toHotSpotMessage(MemorySegment segment) { } static R executeImpl( - Persistance.Pool pool, Channel.Message msg, Function send) { + Persistance.Pool pool, + Class replyType, + Function msg, + Function send) { try (var arena = Arena.ofConfined()) { var bytes = pool.write(msg, null); var memory = arena.allocate(Math.max(bytes.length, 4096)); @@ -212,7 +219,7 @@ static R executeImpl( reply.position(0); reply.limit((int) len); var result = pool.read(reply, null); - return result.get(msg.replyType); + return result.get(replyType); } catch (IOException ex) { throw new IllegalStateException(ex); } @@ -228,37 +235,4 @@ public void close() throws Exception { ID_TO_CHANNEL.remove(id, this); // TBD remove on the peer as well } - - /** - * Subclasses of message denote a computational task to be performed in the "other {@link JVM}". - * - * @param type of the return value - */ - public abstract static class Message { - - final Class replyType; - - /** - * Constructor for subclasses. Use it as {@code super(Integer.class)} to specify the reply type - * of the exception which is then returned from the {@link #evaluate} method. - * - * @param replyType the type of the reply - */ - protected Message(Class replyType) { - this.replyType = replyType; - } - - /** - * Handles evaluation of the exception. Use {@link Channel#execute} to pass this messages to the - * other {@link JVM}. After all the serde and transfer to the other {@link JVM} this method is - * executed to perform its operation. Then the result is passed back via serde again and - * returned from the {@link Channel#execute} method. - * - * @param channel allows sending messages to the other JVM - * @return the result of the evaluation or {@code null} - * @throws Throwable the computation may yield exceptions or errors which are then transferred - * back to the callee JVM - */ - protected abstract R evaluate(Channel channel) throws Throwable; - } } diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 8fc5a93f6d76..428e0298f67f 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -5,6 +5,7 @@ import java.math.BigInteger; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; import org.enso.persist.Persistable; final class TestMain { @@ -32,67 +33,28 @@ static BigInteger factorial(long n) { } @Persistable(id = 430607) - static final class RequestFactorial extends Channel.Message { - private long n; - - RequestFactorial(long n) { - super(Void.class); - this.n = n; - } - + record RequestFactorial(long n) implements Function { @Override - protected Void evaluate(Channel channel) throws Throwable { + public Void apply(Channel channel) { var res = factorial(n).toString(); - channel.execute(new ReportResult(n, res)); + channel.execute(Void.class, new ReportResult(n, res)); return null; } - - long n() { - return n; - } } @Persistable(id = 430608) - static final class ComputeFactorial extends Channel.Message { - private long n; - - ComputeFactorial(long n) { - super(BigInteger.class); - this.n = n; - } - + record ComputeFactorial(long n) implements Function { @Override - protected BigInteger evaluate(Channel channel) throws Throwable { + public BigInteger apply(Channel channel) { var res = factorial(n); return res; } - - long n() { - return n; - } } @Persistable(id = 430606) - static final class ReportResult extends Channel.Message { - private final long key; - private final String value; - - ReportResult(long key, String value) { - super(Void.class); - this.key = key; - this.value = value; - } - - public long key() { - return key; - } - - public String value() { - return value; - } - + record ReportResult(long key, String value) implements Function { @Override - protected Void evaluate(Channel otherVM) throws Throwable { + public Void apply(Channel otherVM) { var vm = System.getProperty("java.vm.name"); assert "Substrate VM".equals(vm) : "Running in SVM again: " + vm; CORRECT_RESULTS.put(key, value); @@ -101,30 +63,13 @@ protected Void evaluate(Channel otherVM) throws Throwable { } @Persistable(id = 430609) - static final class CountDownAndReturn extends Channel.Message { - private final long value; - private final long acc; - - CountDownAndReturn(long value, long acc) { - super(Long.class); - this.value = value; - this.acc = acc; - } - - long value() { - return value; - } - - long acc() { - return acc; - } - + record CountDownAndReturn(long value, long acc) implements Function { @Override - protected Long evaluate(Channel otherVM) throws Throwable { + public Long apply(Channel otherVM) { if (value <= 1) { return acc; } else { - return otherVM.execute(new CountDownAndReturn(value - 1, acc * value)); + return otherVM.execute(Long.class, new CountDownAndReturn(value - 1, acc * value)); } } } diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index 543688e72702..1181fcd72e20 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; +import java.math.BigInteger; import java.nio.file.Files; import java.util.Random; import org.enso.os.environment.jni.JNI.JValue; @@ -134,7 +135,7 @@ public void computeFactorialViaMessages() throws Exception { var n = 0L; for (var i = 0; i < 5; i++) { n += gen.nextLong(MIN, MAX); - channel.execute(new TestMain.RequestFactorial(n)); + channel.execute(Void.class, new TestMain.RequestFactorial(n)); } assertEquals( "Five results found: " + TestMain.CORRECT_RESULTS, 5, TestMain.CORRECT_RESULTS.size()); @@ -151,7 +152,7 @@ public void computeFactorialViaSingleMessage() throws Exception { var n = 0L; for (var i = 0; i < 5; i++) { n += gen.nextLong(MIN, MAX); - var res = channel.execute(new TestMain.ComputeFactorial(n)); + var res = channel.execute(BigInteger.class, new TestMain.ComputeFactorial(n)); var expecting = TestMain.factorial(n); assertEquals("fac(" + n + ") should be", expecting, res); } @@ -160,35 +161,35 @@ public void computeFactorialViaSingleMessage() throws Exception { @Test public void backAndForthFactorialOne() throws Exception { var channel = Channel.create(env()); - var fac = channel.execute(new TestMain.CountDownAndReturn(1, 1)); + var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(1, 1)); assertEquals(1, fac.longValue()); } @Test public void backAndForthFactorialTwo() throws Exception { var channel = Channel.create(env()); - var fac = channel.execute(new TestMain.CountDownAndReturn(2, 1)); + var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(2, 1)); assertEquals(2, fac.longValue()); } @Test public void backAndForthFactorialThree() throws Exception { var channel = Channel.create(env()); - var fac = channel.execute(new TestMain.CountDownAndReturn(3, 1)); + var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(3, 1)); assertEquals(6, fac.longValue()); } @Test public void backAndForthFactorialFour() throws Exception { var channel = Channel.create(env()); - var fac = channel.execute(new TestMain.CountDownAndReturn(4, 1)); + var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(4, 1)); assertEquals(24, fac.longValue()); } @Test public void backAndForthFactorialFive() throws Exception { var channel = Channel.create(env()); - var fac = channel.execute(new TestMain.CountDownAndReturn(5, 1)); + var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(5, 1)); assertEquals(120, fac.longValue()); } } From 42d24f4cc706d5ac26996d40241fe53723c7a720 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 16:13:23 +0200 Subject: [PATCH 20/26] Avoid global pool reference when local one is available --- .../src/main/java/org/enso/os/environment/jni/Channel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index 18b44afcc508..07c691ddb603 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -183,7 +183,7 @@ private static long acceptRequestFromHotSpotJvm( private static long handleWithChannel(Channel channel, long address, long size) throws Throwable { var seg = MemorySegment.ofAddress(address).reinterpret(size); var buf = seg.asByteBuffer(); - var ref = JVMPeer.POOL.read(buf, null); + var ref = channel.pool.read(buf, null); var msg = ref.get(Function.class); @SuppressWarnings("unchecked") var res = msg.apply(channel); From 21fb77716d1afba79dd9391ae8a31ecc61f0afb6 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 19:35:07 +0200 Subject: [PATCH 21/26] Load pool from a specified Class supplying it --- .../org/enso/os/environment/jni/Channel.java | 41 ++++++++++++++----- .../org/enso/os/environment/jni/JVMPeer.java | 23 +++++++---- .../os/environment/TestCollectorFeature.java | 4 ++ .../os/environment/jni/LoadClassTest.java | 15 +++---- 4 files changed, 57 insertions(+), 26 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index 07c691ddb603..8d1568d19171 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; import org.enso.persist.Persistance; import org.graalvm.nativeimage.CurrentIsolate; import org.graalvm.nativeimage.ImageInfo; @@ -75,15 +76,19 @@ private Channel(long id, Persistance.Pool pool, long isolate, long callbackFn) { /** * Factory method to initialize the Channel in the SubstrateVM. * - * @param e - * @return + * @param e JNI environment to talk to the HotSpot JVM + * @param poolClass the class which has public default constructor and can suply instance of + * persistance pool to use for communication + * @return channel for sending messages to the HotSpot JVM */ - static synchronized Channel create(JNI.JNIEnv e) { + static synchronized Channel create( + JNI.JNIEnv e, Class> poolClass) { var id = idCounter++; var classNameWithSlashes = Channel.class.getName().replace('.', '/'); try (var classInC = CTypeConversion.toCString(classNameWithSlashes); + var poolClassInC = CTypeConversion.toCString(poolClass.getName()); var createInC = CTypeConversion.toCString("createJvmPeerChannel"); - var createSigInC = CTypeConversion.toCString("(JJJ)Z"); // + var createSigInC = CTypeConversion.toCString("(JJJLjava/lang/String;)Z"); // var handleInC = CTypeConversion.toCString("handleJvmMessage"); var handleSigInC = CTypeConversion.toCString("(JJJ)J"); // ) { @@ -93,25 +98,40 @@ static synchronized Channel create(JNI.JNIEnv e) { var createMethod = fn.getGetStaticMethodID().call(e, channelClass, createInC.get(), createSigInC.get()); assert createMethod.isNonNull() : "method not found in " + classNameWithSlashes; - var arg = StackValue.get(2, JNI.JValue.class); + var poolClassInHotSpot = fn.getNewStringUTF().call(e, poolClassInC.get()); + var arg = StackValue.get(4, JNI.JValue.class); arg.addressOf(0).setLong(id); arg.addressOf(1).setLong(CurrentIsolate.getCurrentThread().rawValue()); arg.addressOf(2).setLong(CALLBACK_FN.getFunctionPointer().rawValue()); + arg.addressOf(3).setJObject(poolClassInHotSpot); var replyOk = fn.getCallStaticBooleanMethodA().call(e, channelClass, createMethod, arg); + if (!replyOk) { + fn.getExceptionDescribe().call(e); + } assert replyOk : "Failed to create peer in HotSpot JVM"; var handleMethod = fn.getGetStaticMethodID().call(e, channelClass, handleInC.get(), handleSigInC.get()); - var channel = new Channel(id, JVMPeer.POOL, e, channelClass, handleMethod); - ID_TO_CHANNEL.put(id, channel); - return channel; + try { + var pool = poolClass.getConstructor().newInstance().get(); + var channel = new Channel(id, pool, e, channelClass, handleMethod); + ID_TO_CHANNEL.put(id, channel); + return channel; + } catch (ReflectiveOperationException ex) { + throw new IllegalStateException(ex); + } } } /** Allocates new channel with given ID in the HotSpot VM. Called via JNI/foreign interface. */ - private static boolean createJvmPeerChannel(long id, long threadId, long callbackFn) { - var channel = new Channel(id, JVMPeer.POOL, threadId, callbackFn); + private static boolean createJvmPeerChannel( + long id, long threadId, long callbackFn, String poolClassName) throws Throwable { + @SuppressWarnings("unchecked") + var factory = + (Supplier) Class.forName(poolClassName).getConstructor().newInstance(); + var pool = factory.get(); + var channel = new Channel(id, pool, threadId, callbackFn); var prev = ID_TO_CHANNEL.put(id, channel); return prev == null; } @@ -124,6 +144,7 @@ private static boolean createJvmPeerChannel(long id, long threadId, long callbac * *

* + * @param resultType class with the type of {@code R} to use for deserialization * @param msg the message that gets serialized, transferred into the other JVM, deserialized on * the other side and {@link Message#evaluate() evaluated} there * @param the type of result we expect the message to return diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java index 07a3712775b6..288faa107bf7 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/JVMPeer.java @@ -4,17 +4,22 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import org.enso.persist.Persistable; import org.enso.persist.Persistance; -final class JVMPeer { - static final Persistance.Pool POOL = Persistables.POOL; +public final class JVMPeer implements Supplier { - private JVMPeer() {} + public JVMPeer() {} + + @Override + public Persistance.Pool get() { + return Persistables.POOL; + } @Persistable(id = 432002) - public static final class PersistList extends Persistance { - public PersistList() { + static final class PersistList extends Persistance { + PersistList() { super(List.class, true, 432002); } @@ -58,8 +63,8 @@ protected String readObject(Persistance.Input in) throws IOException, ClassNotFo } @Persistable(id = 4438) - public static final class PersistBigInt extends Persistance { - public PersistBigInt() { + static final class PersistBigInt extends Persistance { + PersistBigInt() { super(BigInteger.class, true, 4438); } @@ -81,8 +86,8 @@ protected BigInteger readObject(Persistance.Input in) } @Persistable(id = 4439) - public static final class PersistLong extends Persistance { - public PersistLong() { + static final class PersistLong extends Persistance { + PersistLong() { super(Long.class, true, 4439); } diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/TestCollectorFeature.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/TestCollectorFeature.java index 1893822b878d..56587497ef0c 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/TestCollectorFeature.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/TestCollectorFeature.java @@ -25,6 +25,10 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { RuntimeReflection.registerAllMethods(testClazz); } System.err.println("Registered test classes for reflection: " + ListOfTests.TEST_CLASSES); + + var jvmPeerClass = access.findClassByName("org.enso.os.environment.jni.JVMPeer"); + RuntimeReflection.register(jvmPeerClass); + RuntimeReflection.register(jvmPeerClass.getConstructors()); } private static void recordModulePath(BeforeAnalysisAccess access) { diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index 1181fcd72e20..a6fdd5ceef29 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -10,6 +10,7 @@ import org.enso.os.environment.jni.JNI.JValue; import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.junit.Before; import org.junit.Test; public class LoadClassTest { @@ -44,6 +45,13 @@ private static JNI.JNIEnv env() { return jvm().env(); } + private Channel channel; + + @Before + public void initializeChannel() throws Exception { + channel = Channel.create(env(), JVMPeer.class); + } + @Test public void invokeParseShortMethod() { var env = env(); @@ -130,7 +138,6 @@ public void executeMainClass() throws Exception { public void computeFactorialViaMessages() throws Exception { TestMain.CORRECT_RESULTS.clear(); assertEquals("Results are empty", 0, TestMain.CORRECT_RESULTS.size()); - var channel = Channel.create(env()); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { @@ -147,7 +154,6 @@ public void computeFactorialViaMessages() throws Exception { @Test public void computeFactorialViaSingleMessage() throws Exception { - var channel = Channel.create(env()); var gen = new Random(); var n = 0L; for (var i = 0; i < 5; i++) { @@ -160,35 +166,30 @@ public void computeFactorialViaSingleMessage() throws Exception { @Test public void backAndForthFactorialOne() throws Exception { - var channel = Channel.create(env()); var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(1, 1)); assertEquals(1, fac.longValue()); } @Test public void backAndForthFactorialTwo() throws Exception { - var channel = Channel.create(env()); var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(2, 1)); assertEquals(2, fac.longValue()); } @Test public void backAndForthFactorialThree() throws Exception { - var channel = Channel.create(env()); var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(3, 1)); assertEquals(6, fac.longValue()); } @Test public void backAndForthFactorialFour() throws Exception { - var channel = Channel.create(env()); var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(4, 1)); assertEquals(24, fac.longValue()); } @Test public void backAndForthFactorialFive() throws Exception { - var channel = Channel.create(env()); var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(5, 1)); assertEquals(120, fac.longValue()); } From bc9aab7db9cea18f457f47f962d84551ecd2d81a Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 3 Jun 2025 19:42:10 +0200 Subject: [PATCH 22/26] Public factory method to create a Channel --- .../java/org/enso/os/environment/jni/Channel.java | 11 +++++++---- .../org/enso/os/environment/jni/LoadClassTest.java | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index 8d1568d19171..d3a305966870 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -76,14 +76,17 @@ private Channel(long id, Persistance.Pool pool, long isolate, long callbackFn) { /** * Factory method to initialize the Channel in the SubstrateVM. * - * @param e JNI environment to talk to the HotSpot JVM - * @param poolClass the class which has public default constructor and can suply instance of + * @param jvm instance of HotSpot JVM to connect to + * @param poolClass the class which has public default constructor and can supply an instance of * persistance pool to use for communication * @return channel for sending messages to the HotSpot JVM */ - static synchronized Channel create( - JNI.JNIEnv e, Class> poolClass) { + public static synchronized Channel create( // + JVM jvm, // + Class> poolClass // + ) { var id = idCounter++; + var e = jvm.env(); var classNameWithSlashes = Channel.class.getName().replace('.', '/'); try (var classInC = CTypeConversion.toCString(classNameWithSlashes); var poolClassInC = CTypeConversion.toCString(poolClass.getName()); diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index a6fdd5ceef29..ea6bc22c9f0f 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -49,7 +49,7 @@ private static JNI.JNIEnv env() { @Before public void initializeChannel() throws Exception { - channel = Channel.create(env(), JVMPeer.class); + channel = Channel.create(jvm(), JVMPeer.class); } @Test From 45310257fb81ba3bb3330a33e3c5f2e3196adea5 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 4 Jun 2025 05:13:10 +0200 Subject: [PATCH 23/26] RuntimeVisualizationsTest uses HandlerFactory --- .../src/main/java/module-info.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/runtime-instrument-runtime-server/src/main/java/module-info.java b/engine/runtime-instrument-runtime-server/src/main/java/module-info.java index ed46a77bf65e..5a3d6b5ffdb8 100644 --- a/engine/runtime-instrument-runtime-server/src/main/java/module-info.java +++ b/engine/runtime-instrument-runtime-server/src/main/java/module-info.java @@ -10,4 +10,6 @@ provides com.oracle.truffle.api.instrumentation.provider.TruffleInstrumentProvider with org.enso.interpreter.instrument.runtime.server.RuntimeServerInstrumentProvider; + + uses org.enso.interpreter.instrument.HandlerFactory; } From fa46a7cf51556029ae804ec8b7ae26370a7ab6ad Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 4 Jun 2025 06:49:49 +0200 Subject: [PATCH 24/26] Basic handling of exceptions in the other JVM --- .../org/enso/os/environment/jni/Channel.java | 54 ++++++++++++++----- .../org/enso/os/environment/jni/TestMain.java | 12 +++++ .../os/environment/jni/LoadClassTest.java | 40 ++++++++++++++ 3 files changed, 92 insertions(+), 14 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index d3a305966870..f8949b1d6ab3 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -102,28 +102,25 @@ public static synchronized Channel create( // fn.getGetStaticMethodID().call(e, channelClass, createInC.get(), createSigInC.get()); assert createMethod.isNonNull() : "method not found in " + classNameWithSlashes; var poolClassInHotSpot = fn.getNewStringUTF().call(e, poolClassInC.get()); + var handleMethod = + fn.getGetStaticMethodID().call(e, channelClass, handleInC.get(), handleSigInC.get()); + + var pool = poolClass.getConstructor().newInstance().get(); + var channel = new Channel(id, pool, e, channelClass, handleMethod); + var arg = StackValue.get(4, JNI.JValue.class); arg.addressOf(0).setLong(id); arg.addressOf(1).setLong(CurrentIsolate.getCurrentThread().rawValue()); arg.addressOf(2).setLong(CALLBACK_FN.getFunctionPointer().rawValue()); arg.addressOf(3).setJObject(poolClassInHotSpot); var replyOk = fn.getCallStaticBooleanMethodA().call(e, channelClass, createMethod, arg); - if (!replyOk) { - fn.getExceptionDescribe().call(e); - } + checkForException(e); assert replyOk : "Failed to create peer in HotSpot JVM"; - var handleMethod = - fn.getGetStaticMethodID().call(e, channelClass, handleInC.get(), handleSigInC.get()); - - try { - var pool = poolClass.getConstructor().newInstance().get(); - var channel = new Channel(id, pool, e, channelClass, handleMethod); - ID_TO_CHANNEL.put(id, channel); - return channel; - } catch (ReflectiveOperationException ex) { - throw new IllegalStateException(ex); - } + ID_TO_CHANNEL.put(id, channel); + return channel; + } catch (ReflectiveOperationException ex) { + throw new IllegalStateException(ex); } } @@ -225,9 +222,38 @@ private long toHotSpotMessage(MemorySegment segment) { arg.addressOf(1).setLong(address); arg.addressOf(2).setLong(segment.byteSize()); var replySize = fn.getCallStaticLongMethodA().call(env, channelClass, channelHandle, arg); + checkForException(env); return replySize; } + private static void checkForException(JNI.JNIEnv e) { + var fn = e.getFunctions(); + if (fn.getExceptionCheck().call(e)) { + var throwable = fn.getExceptionOccurred().call(e); + assert throwable.isNonNull() : "There must be a throwable"; + var assertsOn = false; + assert assertsOn = true; + if (assertsOn) { + fn.getExceptionDescribe().call(e); + } + fn.getExceptionClear().call(e); + try (var throwableInC = CTypeConversion.toCString("java/lang/Throwable"); + var messageInC = CTypeConversion.toCString("getMessage"); + var messageSigInC = CTypeConversion.toCString("()Ljava/lang/String;")) { + var throwableClass = fn.getFindClass().call(e, throwableInC.get()); + var messageMethod = + fn.getGetMethodID().call(e, throwableClass, messageInC.get(), messageSigInC.get()); + var args = StackValue.get(1, JNI.JValue.class); + var msg = (JNI.JString) fn.getCallObjectMethodA().call(e, throwable, messageMethod, args); + args.addressOf(0).setBoolean(false); + var cStr = fn.getGetStringUTFChars().call(e, msg, args); + var javaMsg = CTypeConversion.toJavaString(cStr); + fn.getReleaseStringUTFChars().call(e, msg, cStr); + throw new IllegalStateException(javaMsg); + } + } + } + static R executeImpl( Persistance.Pool pool, Class replyType, diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java index 428e0298f67f..787790bc5fa1 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/TestMain.java @@ -73,4 +73,16 @@ public Long apply(Channel otherVM) { } } } + + @Persistable(id = 430610) + record CountDownAndThrow(long value, long acc) implements Function { + @Override + public Void apply(Channel otherVM) { + if (value <= 1) { + throw new IllegalStateException("" + acc); + } else { + return otherVM.execute(Void.class, new CountDownAndThrow(value - 1, acc * value)); + } + } + } } diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index ea6bc22c9f0f..9a10aa01174e 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.math.BigInteger; @@ -11,6 +12,7 @@ import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class LoadClassTest { @@ -193,4 +195,42 @@ public void backAndForthFactorialFive() throws Exception { var fac = channel.execute(Long.class, new TestMain.CountDownAndReturn(5, 1)); assertEquals(120, fac.longValue()); } + + @Test + public void throwFactorialOne() throws Exception { + assertException("1", new TestMain.CountDownAndThrow(1, 1)); + } + + @Ignore + @Test + public void throwFactorialTwo() throws Exception { + assertException("2", new TestMain.CountDownAndThrow(2, 1)); + } + + @Ignore + @Test + public void throwFactorialThree() throws Exception { + assertException("6", new TestMain.CountDownAndThrow(3, 1)); + } + + @Ignore + @Test + public void throwFactorialFour() throws Exception { + assertException("24", new TestMain.CountDownAndThrow(4, 1)); + } + + @Ignore + @Test + public void throwFactorialFive() throws Exception { + assertException("120", new TestMain.CountDownAndThrow(5, 1)); + } + + private void assertException(String msg, TestMain.CountDownAndThrow action) { + try { + channel.execute(Void.class, action); + fail("Expecting an exception to be thrown for " + msg); + } catch (IllegalStateException ex) { + assertEquals(msg, ex.getMessage()); + } + } } From 04459624882482ed0255f4490bf492ecad394e92 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 4 Jun 2025 07:35:20 +0200 Subject: [PATCH 25/26] Propagate exceptions back and forth among JVMs --- .../org/enso/os/environment/jni/Channel.java | 50 +++++++++++++++---- .../os/environment/jni/LoadClassTest.java | 5 -- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index f8949b1d6ab3..d4aef5bb9620 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -20,6 +20,7 @@ import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; +import org.graalvm.word.WordFactory; /** Channel connects two {@link JVM} instances. */ public final class Channel implements AutoCloseable { @@ -114,7 +115,7 @@ public static synchronized Channel create( // arg.addressOf(2).setLong(CALLBACK_FN.getFunctionPointer().rawValue()); arg.addressOf(3).setJObject(poolClassInHotSpot); var replyOk = fn.getCallStaticBooleanMethodA().call(e, channelClass, createMethod, arg); - checkForException(e); + channel.checkForException(e); assert replyOk : "Failed to create peer in HotSpot JVM"; ID_TO_CHANNEL.put(id, channel); @@ -170,14 +171,14 @@ public final R execute(Class resultType, Function msg) { resultType, msg, seg -> { - Object res = -1L; try { var isoRef = MemorySegment.ofAddress(isolate); - res = fnHandle.invoke(isoRef, id, seg, seg.byteSize()); + var res = fnHandle.invoke(isoRef, id, seg, seg.byteSize()); + return (long) res; } catch (Throwable ex) { - ex.printStackTrace(); + printStackTrace(ex, false); + return -1L; } - return (long) res; }); } } @@ -197,8 +198,17 @@ private static long acceptRequestFromHotSpotJvm( var channel = ID_TO_CHANNEL.get(id); assert channel != null : "There must be a channel " + id + " but " + ID_TO_CHANNEL; - var len = handleWithChannel(channel, data.rawValue(), size); - return len; + try { + var len = handleWithChannel(channel, data.rawValue(), size); + return len; + } catch (Throwable ex) { + channel.printStackTrace(ex, true); + var exceptionMessage = + ex.getMessage() + .subSequence(0, Math.min(ex.getMessage().length(), (int) Math.min(2048, size))); + CTypeConversion.toCString(exceptionMessage, data, WordFactory.unsigned(size)); + return -2L; + } } private static long handleWithChannel(Channel channel, long address, long size) throws Throwable { @@ -226,14 +236,12 @@ private long toHotSpotMessage(MemorySegment segment) { return replySize; } - private static void checkForException(JNI.JNIEnv e) { + private void checkForException(JNI.JNIEnv e) { var fn = e.getFunctions(); if (fn.getExceptionCheck().call(e)) { var throwable = fn.getExceptionOccurred().call(e); assert throwable.isNonNull() : "There must be a throwable"; - var assertsOn = false; - assert assertsOn = true; - if (assertsOn) { + if (printStackTrace(null, true)) { fn.getExceptionDescribe().call(e); } fn.getExceptionClear().call(e); @@ -264,6 +272,11 @@ static R executeImpl( var memory = arena.allocate(Math.max(bytes.length, 4096)); memory.copyFrom(MemorySegment.ofArray(bytes)); long len = send.apply(memory); + if (len == -2) { + // signals exception + var exceptionMessage = memory.getString(0); + throw new IllegalStateException(exceptionMessage); + } assert len >= 0; var reply = memory.asByteBuffer(); reply.position(0); @@ -285,4 +298,19 @@ public void close() throws Exception { ID_TO_CHANNEL.remove(id, this); // TBD remove on the peer as well } + + /** + * @param ex exception to print stack trace for or {@code null} + * @param userCode is the exception from user code or is it unexpected + * @return {@code true} if the exception was printed and further details should be printed + */ + private boolean printStackTrace(Throwable ex, boolean userCode) { + if (!userCode) { + if (ex != null) { + ex.printStackTrace(); + } + return true; + } + return false; + } } diff --git a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java index 9a10aa01174e..27a6923aa6c0 100644 --- a/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java +++ b/lib/java/os-environment/src/test/java/org/enso/os/environment/jni/LoadClassTest.java @@ -12,7 +12,6 @@ import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; public class LoadClassTest { @@ -201,25 +200,21 @@ public void throwFactorialOne() throws Exception { assertException("1", new TestMain.CountDownAndThrow(1, 1)); } - @Ignore @Test public void throwFactorialTwo() throws Exception { assertException("2", new TestMain.CountDownAndThrow(2, 1)); } - @Ignore @Test public void throwFactorialThree() throws Exception { assertException("6", new TestMain.CountDownAndThrow(3, 1)); } - @Ignore @Test public void throwFactorialFour() throws Exception { assertException("24", new TestMain.CountDownAndThrow(4, 1)); } - @Ignore @Test public void throwFactorialFive() throws Exception { assertException("120", new TestMain.CountDownAndThrow(5, 1)); From 93c6fd5eb9b02e066b50f816730a1d805211f21e Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 4 Jun 2025 07:46:03 +0200 Subject: [PATCH 26/26] Allocate the method handle at Channel creation time --- .../org/enso/os/environment/jni/Channel.java | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java index d4aef5bb9620..306938b6cebf 100644 --- a/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java +++ b/lib/java/os-environment/src/main/java/org/enso/os/environment/jni/Channel.java @@ -6,6 +6,7 @@ import java.lang.foreign.Linker; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; import java.util.HashMap; import java.util.Map; import java.util.function.Function; @@ -40,7 +41,7 @@ public final class Channel implements AutoCloseable { private final long id; private final JNI.JNIEnv env; private final long isolate; - private final long callbackFn; + private final MethodHandle callbackFn; private final JNI.JClass channelClass; private final JNI.JMethodID channelHandle; @@ -55,7 +56,7 @@ private Channel( this.pool = pool; this.env = env; this.isolate = -1; - this.callbackFn = -1; + this.callbackFn = null; this.channelClass = handleClass; this.channelHandle = handleFn; } @@ -68,10 +69,19 @@ private Channel(long id, Persistance.Pool pool, long isolate, long callbackFn) { this.id = id; this.pool = pool; this.isolate = isolate; - this.callbackFn = callbackFn; this.env = null; this.channelClass = null; this.channelHandle = null; + + var fnCallbackAddress = MemorySegment.ofAddress(callbackFn); + var fnDescriptor = + FunctionDescriptor.of( + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG); + this.callbackFn = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); } /** @@ -155,31 +165,9 @@ private static boolean createJvmPeerChannel( */ public final R execute(Class resultType, Function msg) { if (this.isolate == -1) { - return executeImpl(pool, resultType, msg, memory -> toHotSpotMessage(memory)); + return executeImpl(pool, resultType, msg, this::toHotSpotMessage); } else { - var fnCallbackAddress = MemorySegment.ofAddress(callbackFn); - var fnDescriptor = - FunctionDescriptor.of( - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG, - ValueLayout.ADDRESS, - ValueLayout.JAVA_LONG); - var fnHandle = Linker.nativeLinker().downcallHandle(fnCallbackAddress, fnDescriptor); - return executeImpl( - pool, - resultType, - msg, - seg -> { - try { - var isoRef = MemorySegment.ofAddress(isolate); - var res = fnHandle.invoke(isoRef, id, seg, seg.byteSize()); - return (long) res; - } catch (Throwable ex) { - printStackTrace(ex, false); - return -1L; - } - }); + return executeImpl(pool, resultType, msg, this::toSubstrateMessage); } } @@ -236,6 +224,17 @@ private long toHotSpotMessage(MemorySegment segment) { return replySize; } + private long toSubstrateMessage(MemorySegment seg) { + try { + var isoRef = MemorySegment.ofAddress(isolate); + var res = callbackFn.invoke(isoRef, id, seg, seg.byteSize()); + return (long) res; + } catch (Throwable ex) { + printStackTrace(ex, false); + return -1L; + } + } + private void checkForException(JNI.JNIEnv e) { var fn = e.getFunctions(); if (fn.getExceptionCheck().call(e)) {