Merge branch '129-switchToJava11AndUpdateMFDependencies' of https://github.com/hbz/rpb into main

Resolves https://jira.hbz-nrw.de/browse/RPB-266

Author: fsteeg

.github/workflows/build.yml

@@ -7,20 +7,15 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 11
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
-    - name: Install metafacture-core 5.7.1
+        java-version: 11
+    - name: Install metafacture-core 7.0.0-rc1
       run: |
-        git clone https://github.com/metafacture/metafacture-core.git -b metafacture-core-5.7.1
+        git clone https://github.com/metafacture/metafacture-core.git -b metafacture-core-7.0.0-rc1
         cd metafacture-core
-        ./gradlew publishToMavenLocal
-        cd ..
-    - name: Install metafacture-fix 0.7.1
-      run: |
-        git clone https://github.com/metafacture/metafacture-fix.git -b 0.7.1
-        cd metafacture-fix
+        touch .temp # don't build release, which requires signing
         ./gradlew publishToMavenLocal
         cd ..
     - name: Run tests

README.md

@@ -188,10 +188,19 @@ sh transformAndIndex.sh
 
 Start the web application:
 
+### Test mode
+
 ```
 sbt run
 ```
 
+### Prod mode
+
+```
+sbt stage
+./target/universal/stage/bin/rpb -no-version-check
+```
+
 http://localhost:9000
 
 ## Java development

app/controllers/nwbib/Application.java

@@ -9,6 +9,7 @@
 import java.io.UnsupportedEncodingException;
 import java.net.URL;
 import java.net.URLEncoder;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.text.Collator;
@@ -32,10 +33,9 @@
 import java.util.stream.StreamSupport;
 
 import org.antlr.runtime.RecognitionException;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.tuple.Pair;
-import org.elasticsearch.common.base.Charsets;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.io.Streams;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonMappingException;
@@ -80,6 +80,8 @@
  */
 public class Application extends Controller {
 
+	private static final String UTF_8 = "UTF-8";
+
 	static final int MAX_FACETS = 150;
 
 	private static final String STARRED = "starred";
@@ -154,7 +156,7 @@ public static Result advanced() {
 	 */
 	public static String currentUri() {
 		try {
-			return URLEncoder.encode(request().host() + request().uri(), "UTF-8");
+			return URLEncoder.encode(request().host() + request().uri(), UTF_8);
 		} catch (UnsupportedEncodingException e) {
 			Logger.error("Could not get current URI", e);
 		}
@@ -353,7 +355,7 @@ public static Result register(final String t) {
 	public static Result journals() throws IOException {
 		try (InputStream stream = Play.application().classloader()
 				.getResourceAsStream("nwbib-journals.csv")) {
-			String csv = new String(Streams.copyToByteArray(stream), Charsets.UTF_8);
+			String csv = IOUtils.toString(stream, UTF_8);
 			List<String> lines = Arrays.asList(csv.split("\n"));
 			List<HashMap<String, String>> maps = lines.stream()
 					.filter(line -> line.split("\",\"").length == 2).map(line -> {
@@ -1007,10 +1009,9 @@ private static JsonNode transform(JsonNode jsonBody)
 			throws IOException, FileNotFoundException, RecognitionException {
 		File input = new File("conf/output/test-output-strapi.json");
 		File output = new File("conf/output/test-output-0.json");
-		Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charsets.UTF_8));
-		ETL.main(new String[] { "conf/rpb-test-titel-to-lobid.flux" });
-		String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream()
-				.collect(Collectors.joining("\n"));
+		Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charset.forName(UTF_8)));
+		ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"});
+		String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n"));
 		return Json.parse(result);
 	}
 
@@ -1050,6 +1051,6 @@ private static Object mergeValues(Object transformedObject, Object lobidObject)
 
 	private static String elasticsearchUrl(String id) throws UnsupportedEncodingException {
 		return "http://weywot3:9200/resources-rpb-test/resource/"
-				+ URLEncoder.encode("https://lobid.org/resources/" + id, "UTF-8");
+				+ URLEncoder.encode("https://lobid.org/resources/" + id, UTF_8);
 	}
 }

app/controllers/nwbib/Classification.java

@@ -13,6 +13,7 @@
 import java.text.Collator;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -42,14 +43,17 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.transport.Netty4Plugin;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jsonldjava.core.JsonLdError;
@@ -194,6 +198,12 @@ public static Comparator<JsonNode> comparator(Function<JsonNode, String> fun) {
 				.compare(fun.apply(o1), fun.apply(o2));
 	}
 
+	private static class EmbeddedNode extends Node {
+		public EmbeddedNode(Settings preparedSettings, Collection<Class<? extends Plugin>> classpathPlugins) {
+			super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), classpathPlugins);
+		}
+	}
+
 	private Classification() {
 		/* Use via static functions, no instantiation. */
 	}
@@ -237,7 +247,7 @@ public static List<String> toJsonLd(final URL turtleUrl) {
 	public static JsonNode ids(String q, String t) {
 		QueryBuilder queryBuilder = QueryBuilders.boolQuery()
 				.must(QueryBuilders.idsQuery(Type.NWBIB.elasticsearchType,
-						Type.SPATIAL.elasticsearchType).ids(q));
+						Type.SPATIAL.elasticsearchType).addIds(q));
 		SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX)
 				.setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(queryBuilder);
 		if (t.isEmpty()) {
@@ -473,16 +483,21 @@ public static String shortId(String uri) {
 
 	/** Start up the embedded Elasticsearch classification index. */
 	public static void indexStartup() {
-		Settings clientSettings = ImmutableSettings.settingsBuilder()
-				.put("path.home", new File(".").getAbsolutePath())
-				.put("http.port",
-						play.Play.application().isTest() ? "8855"
-								: CONFIG.getString("index.es.port.http"))
-				.put("transport.tcp.port", play.Play.application().isTest() ? "8856"
-						: CONFIG.getString("index.es.port.tcp"))
-				.build();
-		node =
-				NodeBuilder.nodeBuilder().settings(clientSettings).local(true).node();
+		boolean isTest = play.Play.application().isTest();
+		String httpPort = isTest ? "8855" : CONFIG.getString("index.es.port.http");
+		String tcpPort = isTest ? "8856" : CONFIG.getString("index.es.port.tcp");
+		node = new EmbeddedNode(Settings.builder().put("transport.type", "netty4")//
+				.put("http.type", "netty4")//
+				.put("http.enabled", "true")//
+				.put("path.home", new File(".").getAbsolutePath())//
+				.put("http.port", httpPort)//
+				.put("transport.tcp.port", tcpPort)//
+				.build(), Arrays.asList(Netty4Plugin.class));
+		try {
+			node.start();
+		} catch (NodeValidationException e) {
+			e.printStackTrace();
+		}
 		client = node.client();
 		client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute()
 				.actionGet();
@@ -502,8 +517,8 @@ private static void indexData(String dataUrl, Type type) {
 			List<String> jsonLd = toJsonLd(new URL(dataUrl));
 			for (String concept : jsonLd) {
 				String id = Json.parse(concept).findValue("@id").textValue();
-				IndexRequestBuilder indexRequest = client
-						.prepareIndex(INDEX, type.elasticsearchType, id).setSource(concept);
+				IndexRequestBuilder indexRequest = client.prepareIndex(INDEX, type.elasticsearchType, id)
+						.setSource(concept.getBytes(), XContentType.JSON);
 				bulkRequest.add(indexRequest);
 			}
 		} catch (MalformedURLException e) {
@@ -517,7 +532,11 @@ private static void indexData(String dataUrl, Type type) {
 
 	/** Shut down the embedded Elasticsearch classification index. */
 	public static void indexShutdown() {
-		node.close();
+		try {
+			node.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
 	}
 
 	/**

app/controllers/nwbib/Lobid.java

@@ -20,10 +20,10 @@
 import java.util.stream.StreamSupport;
 
 import org.apache.commons.lang3.tuple.Pair;
-import org.elasticsearch.common.collect.Lists;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import com.google.common.html.HtmlEscapers;
 
 import controllers.nwbib.Classification.Type;

build.sbt

@@ -2,26 +2,28 @@ name := "rpb"
 
 version := "0.1.1-SNAPSHOT"
 
-scalaVersion := "2.11.11"
+scalaVersion := "2.11.12"
 
 resolvers += Resolver.mavenLocal
 
 libraryDependencies ++= Seq(
   cache,
   javaWs,
   "com.typesafe.play" % "play-test_2.11" % "2.4.11",
-  "org.metafacture" % "metafacture-elasticsearch" % "5.7.1",
-  "org.metafacture" % "metafacture-io" % "5.7.1",
-  "org.metafacture" % "metafacture-strings" % "5.7.1",
-  "org.metafacture" % "metafacture-json" % "5.7.1",
-  "org.metafacture" % "metafacture-flux" % "5.7.1",
-  "org.metafacture" % "metafacture-triples" % "5.7.1",
-  "org.metafacture" % "metafacture-formatting" % "5.7.1",
-  "org.metafacture" % "metafacture-monitoring" % "5.7.1",
-  "org.metafacture" % "metafacture-csv" % "5.7.1",
-  "org.metafacture" % "metafix" % "0.7.1",
-  "org.metafacture" % "metafacture-linkeddata" % "5.7.1",
-  "org.elasticsearch" % "elasticsearch" % "1.7.5" withSources(),
+  "org.metafacture" % "metafacture-elasticsearch" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-io" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-strings" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-json" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-flux" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-triples" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-formatting" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-monitoring" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-csv" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafacture-linkeddata" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.metafacture" % "metafix" % "commit-b9eebb41b26bac5b1083b90112c0eefdc17ad25e-SNAPSHOT",
+  "org.elasticsearch" % "elasticsearch" % "5.6.16" withSources(),
+  "org.elasticsearch.plugin" % "transport-netty4-client" % "5.6.16" withSources(),
+  "com.sun.xml.bind" % "jaxb-impl" % "2.3.3" withSources(),
   "com.github.jsonld-java" % "jsonld-java" % "0.5.0",
   "org.apache.commons" % "commons-rdf-jena" % "0.5.0",
   "org.apache.commons" % "commons-csv" % "1.6",
@@ -43,7 +45,7 @@ dependencyOverrides ++= Set(
 
 lazy val root = (project in file(".")).enablePlugins(PlayJava)
 
-javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
+javacOptions ++= Seq("-source", "11", "-target", "11")
 
 import com.typesafe.sbteclipse.core.EclipsePlugin.EclipseKeys
 

monit_restart.sh

@@ -25,6 +25,7 @@ HOME="/home/sol"
 
 # it is important to set the proper locale
 . $HOME/.locale
+export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
 JAVA_OPTS=$(echo "$JAVA_OPTS" |sed 's#,#\ #g')
 
 cd $HOME/git/$REPO
@@ -34,10 +35,15 @@ case $ACTION in
                         kill $(cat target/universal/stage/RUNNING_PID)
 			rm target/universal/stage/RUNNING_PID
                 fi
-                JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" sbt "start $PORT"
+                export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError -DpreferIPv4Stack"
+                sbt --java-home $JAVA_HOME stage >> ./target/universal/stage/logs/application.log 2>&1
+                ./target/universal/stage/bin/rpb -Dhttp.port=$PORT -no-version-check >> ./target/universal/stage/logs/application.log 2>&1
                 ;;
         stop)
                 kill $(cat target/universal/stage/RUNNING_PID)
+                sleep 14
+                kill -9 $(cat target/universal/stage/RUNNING_PID)
+                rm target/universal/stage/RUNNING_PID
                 ;;
         *)
                 echo "usage: $USAGE"

project/build.properties

@@ -1 +1 @@
-sbt.version=0.13.16
+sbt.version=0.13.18

transformAndImportFull.sh

@@ -9,29 +9,31 @@ HOST=$1
 # Create token in Strapi admin UI (Settings > API Tokens), set here or in environment
 API_TOKEN=$API_TOKEN
 
+export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
+
 # Notationen
 # curl --request DELETE "http://$HOST:1337/api/rpb-notations/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations HOST=$HOST API_TOKEN=$API_TOKEN"
 # curl --request DELETE "http://$HOST:1337/api/rpb-spatials/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials HOST=$HOST API_TOKEN=$API_TOKEN"
 # curl --request DELETE "http://$HOST:1337/api/fachgebiete/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-fachgebiete.ndjson PATH=fachgebiete HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-fachgebiete.ndjson PATH=fachgebiete HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Personen
 # curl --request DELETE "http://$HOST:1337/api/persons/[1-5]"
-sbt "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio.txt OUT_FILE=output-rppd-strapi.ndjson"
-sbt "runMain rpb.ETL conf/rppd-import.flux IN_FILE=output-rppd-strapi.ndjson HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio.txt OUT_FILE=output-rppd-strapi.ndjson"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rppd-import.flux IN_FILE=output-rppd-strapi.ndjson HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Normdaten
 # curl --request DELETE "http://$HOST:1337/api/rpb-authorities/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-sw-to-strapi.flux IN_FILE=RPB-Export_HBZ_SW.txt OUT_FILE=output-sw-strapi.ndjson"
-sbt "runMain rpb.ETL conf/rpb-sw-import.flux IN_FILE=output-sw-strapi.ndjson HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-sw-to-strapi.flux IN_FILE=RPB-Export_HBZ_SW.txt OUT_FILE=output-sw-strapi.ndjson"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-sw-import.flux IN_FILE=output-sw-strapi.ndjson HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Titeldaten
-sbt "runMain rpb.ETL conf/rpb-titel-to-strapi.flux"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-titel-to-strapi.flux"
 # curl --request DELETE "http://$HOST:1337/api/articles/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('type','u') PATH=articles HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('type','u') PATH=articles HOST=$HOST API_TOKEN=$API_TOKEN"
 # curl --request DELETE "http://$HOST:1337/api/independent-works/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('type','Monografie') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('type','Band') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_contain('type','Mehrt') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('type','Monografie') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('type','Band') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt --java-home $JAVA_HOME "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_contain('type','Mehrt') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"