rapidyaml: Working with native-image

Author: biojppm

common/native.mk

@@ -9,6 +9,7 @@ NATIVE_OPTS := \
   --emit=build-report \
   -march=compatibility \
   -H:IncludeResources=SCI_VERSION \
+  -H:IncludeResources='librapidyaml\.0\.8\.0\.so' \
   -H:ReflectionConfigurationFiles=reflection.json \
   -H:+ReportExceptionStackTraces \
   -H:Log=registerResource: \

common/vars.mk

@@ -223,21 +223,23 @@ RAPIDYAML := $(ROOT)/rapidyaml
 
 RAPIDYAML_VERSION := 0.8.0
 #RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION)
-RAPIDYAML_TAG ?= 8c37616378aefd376690a19459c31a56ce596b5e
+RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302
 RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml
 RAPIDYAML_BUILD_TYPE ?= Release
 RAPIDYAML_DBG ?= 0
 RAPIDYAML_TIMED ?= 1
 RAPIDYAML_JAVA := \
   $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \
+  $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java \
   $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \
   $(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java
 RAPIDYAML_JNI_H := $(ROOT)/rapidyaml/native/org_rapidyaml_Rapidyaml.h
-RAPIDYAML_SO := $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO)
+RAPIDYAML_SO_NAME := librapidyaml.$(RAPIDYAML_VERSION).$(SO)
+RAPIDYAML_SO := $(ROOT)/rapidyaml/native/$(RAPIDYAML_SO_NAME)
 RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB)
 RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar
 RAPIDYAML_INSTALLED_DIR := \
-  $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)/
+  $(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)
 RAPIDYAML_INSTALLED := \
   $(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar
 

core/src/yamlscript/parser.clj

@@ -10,6 +10,7 @@
    [yamlscript.common])
   (:import
    (java.util Optional)
+   (java.nio ByteBuffer)
    (java.nio.charset StandardCharsets)
    (org.rapidyaml Evt Rapidyaml)
    (org.snakeyaml.engine.v2.api LoadSettings)
@@ -31,17 +32,20 @@
 
 (declare parse-fn)
 
+(defn TIMER [] (System/getenv "YS_TIMER"))
+
 (def shebang-ys #"^#!.*/env ys-0(?:\.d+\.\d+)?\n")
 (def shebang-bash #"^#!.*[/ ]bash\n+source +<\(")
+
 (defn parse
   "Parse a YAML string into a sequence of event objects."
   [yaml-string]
   (let [has-code-mode-shebang (or
                                 (re-find shebang-ys yaml-string)
                                 (re-find shebang-bash yaml-string))
-        events (if (System/getenv "YS_PARSER_TIME")
-                 (time (parse-fn yaml-string))
-                 (parse-fn yaml-string))
+        events (if (TIMER)
+                 (time ((parse-fn) yaml-string))
+                 ((parse-fn) yaml-string))
         [first-event & rest-events] events
         first-event-tag (:! first-event)
         first-event (if (and has-code-mode-shebang
@@ -184,12 +188,16 @@
 (defn parse-rapidyaml [^String yaml-string]
   (rest
     (let [parser ^Rapidyaml (new Rapidyaml)
+          _ (when (TIMER)
+              (.timingEnabled parser true))
           buffer (.getBytes yaml-string StandardCharsets/UTF_8)
           masks (int-array 5)
           needed (.parseYsToEvt parser buffer masks)
           buffer (.getBytes yaml-string StandardCharsets/UTF_8)
           masks (int-array needed)
           _ (.parseYsToEvt parser buffer masks)
+          ;; TODO: aget slow?
+          ;; https://stackoverflow.com/questions/10133094/clojure-why-is-aget-so-slow
           get-str (fn [i]
                     (let [off (aget masks (inc i))
                           len (aget masks (+ i 2))]
@@ -228,17 +236,70 @@
               (recur i tag anchor events)))
           events)))))
 
-(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")]
-                (condp = parser-name
-                  "" parse-snakeyaml
-                  "snake" parse-snakeyaml
-                  "rapid" parse-rapidyaml
-                  "ryml" parse-rapidyaml
-                  ; TODO:
-                  ;"rapid-buf" parse-rapidyaml-buf
-                  ;"ryml-buf" parse-rapidyaml-buf
-                  (die "Unknown YS_PARSER value: " parser-name))
-                parse-snakeyaml))
+(defn parse-rapidyaml-buf [^String yaml-string]
+  (rest
+    (let [parser (new Rapidyaml)
+          _ (when (TIMER)
+              (.timingEnabled parser true))
+          srcbytes (.getBytes yaml-string StandardCharsets/UTF_8)
+          srcbuffer (ByteBuffer/allocateDirect (alength srcbytes))
+          _ (.put srcbuffer srcbytes)
+          masks (Rapidyaml/mkIntBuffer 5)
+          needed (.parseYsToEvtBuf parser srcbuffer masks)
+          _ (.position srcbuffer 0)
+          _ (.put srcbuffer srcbytes)
+          masks (Rapidyaml/mkIntBuffer needed)
+          _ (.parseYsToEvtBuf parser srcbuffer masks)
+          get-str (fn [i]
+                    (let [off (.get masks ^Long (inc i))
+                          len (.get masks ^Long (+ i 2))]
+                      (reduce
+                        (fn [slice i] (str slice
+                                        (char (.get srcbuffer ^Long i))))
+                        "" (range off (+ off len)))))]
+
+      (loop [i 0, tag nil, anchor nil, events []]
+        (if (< i needed)
+          (let [mask (.get masks i)
+                type (event-type mask)
+                ; _ (WWW (Integer/toString mask 2) type)
+                sval (when (flag? HAS_STR mask) (get-str i))
+                tag (if (flag? TAG_ mask) sval tag)
+                anchor (if (flag? ANCH mask) sval anchor)
+                event (when type
+                        (let [event {:+ type}
+                              event (if (flag? FLOW mask)
+                                      (assoc event :flow true) event)
+                              event (if anchor (assoc event :& anchor) event)
+                              event (if tag
+                                      (let [tag (str/replace tag
+                                                  #"^!!"
+                                                  "tag:yaml.org,2002:")]
+                                        (assoc event :! tag)) event)
+                              event (if sval (assoc event
+                                               (get-skey mask) sval) event)
+                              event (if (= type "=ALI")
+                                      {:+ "=ALI" :* sval}
+                                      event)]
+                          event))
+                events (if event (conj events event) events)
+                i (+ i (if sval 3 1))]
+            (if event
+              (recur i nil nil events)
+              (recur i tag anchor events)))
+          events)))))
+
+(defn parse-fn []
+  (if-let [parser-name (System/getenv "YS_PARSER")]
+    (condp = parser-name
+      "" parse-snakeyaml
+      "snake" parse-snakeyaml
+      "rapid" parse-rapidyaml
+      "rapid-buf" parse-rapidyaml-buf
+      "ryml" parse-rapidyaml
+      "ryml-buf" parse-rapidyaml-buf
+      (die "Unknown YS_PARSER value: " parser-name))
+    parse-snakeyaml))
 
 (comment
   )

mvp/Makefile

@@ -0,0 +1,20 @@
+include ../common/base.mk
+include $(COMMON)/java.mk
+
+JAVA_FILE := src/main/java/com/example/App.java
+JAR_FILE := target/my-app-1.0-SNAPSHOT.jar
+
+
+default::
+	which mvn
+
+build:: $(JAR_FILE)
+
+$(JAR_FILE): $(JAVA_FILE)
+	mvn clean package
+
+run:: build
+	java -jar target/my-app-1.0-SNAPSHOT.jar
+
+test clean package::
+	mvn $@

mvp/pom.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project
+  xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.example</groupId>
+  <artifactId>my-app</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <properties>
+    <maven.compiler.source>11</maven.compiler.source>
+    <maven.compiler.target>11</maven.compiler.target>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <!-- Apache Commons Lang library
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.12.0</version>
+    </dependency -->
+
+    <!-- JUnit for testing -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.13.2</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.rapidyaml</groupId>
+      <artifactId>rapidyaml</artifactId>
+      <version>0.8.0</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <!-- Maven JAR Plugin -->
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>3.2.0</version>
+        <configuration>
+          <archive>
+            <manifest>
+              <addClasspath>true</addClasspath>
+              <mainClass>com.example.App</mainClass>
+            </manifest>
+          </archive>
+        </configuration>
+      </plugin>
+
+      <!-- Maven Shade Plugin for creating an uber jar with dependencies
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>3.2.4</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin -->
+    </plugins>
+  </build>
+</project>

mvp/src/main/java/com/example/App.java

@@ -0,0 +1,12 @@
+package com.example;
+
+// import org.apache.commons.lang3.StringUtils;
+import org.rapidyaml.Rapidyaml;
+
+public class App {
+    public static void main(String[] args) {
+        String text = "Hello, World!";
+        System.out.println("Original text: " + text);
+        // System.out.println("Reversed text: " + StringUtils.reverse(text));
+    }
+}

mvp/src/test/java/com/example/AppTest.java

@@ -0,0 +1,13 @@
+package com.example;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class AppTest {
+    @Test
+    public void testApp() {
+        // Simple test to verify the App class can be instantiated
+        App app = new App();
+        assertNotNull(app);
+    }
+}

rapidyaml/Makefile

@@ -2,39 +2,31 @@ include ../common/base.mk
 include $(COMMON)/clojure.mk
 include $(COMMON)/java.mk
 
-RAPIDYAML_JAR_DEPS := \
-  $(wildcard \
-      build \
-      $(JAVA_INSTALLED) \
-      Makefile \
-      pom.xml \
-  )
-
 RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class)
+RAPIDYAML_CLASSES := \
+  $(RAPIDYAML_CLASS:$(RAPIDYAML)/src/main/java/%=-C src/main/java %)
 
 
 #------------------------------------------------------------------------------
 default::
-	echo $(RAPIDYAML_SO)
-	echo $(RAPIDYAML_LIB)
-
-x:
-	YS -ce 'foo: var'
 
-build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB)
+build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) $(RAPIDYAML_JAR)
 	@:
 
 jar: $(RAPIDYAML_JAR)
 
+jar-test: $(RAPIDYAML_JAR)
+	java -jar $<
+
 install:: $(RAPIDYAML_INSTALLED)
 
 test:: build $(JAVA_INSTALLED)
 	$(MAKE) -C native $@
-	$(MVN) $@
+	YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) $@
 
 test-x: build $(JAVA_INSTALLED)
-	$(MAKE) -C native $@
-	$(MVN) -X -e test
+	$(MAKE) -C native test
+	YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) -X -e test
 
 clean::
 	$(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB)
@@ -43,6 +35,7 @@ clean::
 
 realclean:: clean
 	$(MAKE) -C native $@
+	$(RM) -r $(HOME)/.m2/repository/org/rapidyaml/
 
 sysclean::
 	$(RM) -r $(MAVEN_REPOSITORY)/org/rapidyaml
@@ -53,14 +46,23 @@ jni: $(RAPIDYAML_JNI_H)
 #------------------------------------------------------------------------------
 $(RAPIDYAML_SO): $(RAPIDYAML_JNI_H)
 	$(MAKE) -C native $@
+
+# XXX Probably don't need this:
 $(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H)
 	$(MAKE) -C native $@
 
-$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED)
-	$(MVN) install
-
-$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED)
-	$(JAR) -cvf $@ $<
+$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR)
+ifdef YS_MAVEN_TEST
+	YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) install
+else
+	$(MVN) install -Dmaven.test.skip
+endif
+	touch $@
+
+$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(RAPIDYAML_SO)
+	$(JAR) -cf $@ $(RAPIDYAML_CLASSES)
+	$(JAR) -uf $@ -C $(RAPIDYAML)/native/ $(RAPIDYAML_SO_NAME)
+	$(JAR) -umf manifest.txt $@
 
 $(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED)
 	@# this doesn't work:

rapidyaml/manifest.txt

@@ -0,0 +1 @@
+Main-Class: org.rapidyaml.Rapidyaml