Update Java and Elasticsearch versions (RPB-266)

- We need Java 11 for using the latest Metafacture release
- Elasticsearch 1.x.x does not support Java 11, Elasticsearch
2.x.x does not support dots in field names, 5.6.16 does both

Author: fsteeg

README.md

@@ -188,10 +188,19 @@ sh transformAndIndex.sh
 
 Start the web application:
 
+### Test mode
+
 ```
 sbt run
 ```
 
+### Prod mode
+
+```
+sbt stage
+./target/universal/stage/bin/rpb -no-version-check
+```
+
 http://localhost:9000
 
 ## Java development

app/controllers/nwbib/Application.java

@@ -9,6 +9,7 @@
 import java.io.UnsupportedEncodingException;
 import java.net.URL;
 import java.net.URLEncoder;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.text.Collator;
@@ -32,10 +33,9 @@
 import java.util.stream.StreamSupport;
 
 import org.antlr.runtime.RecognitionException;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.tuple.Pair;
-import org.elasticsearch.common.base.Charsets;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.io.Streams;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -76,6 +76,8 @@
  */
 public class Application extends Controller {
 
+	private static final String UTF_8 = "UTF-8";
+
 	static final int MAX_FACETS = 150;
 
 	private static final String STARRED = "starred";
@@ -150,7 +152,7 @@ public static Result advanced() {
 	 */
 	public static String currentUri() {
 		try {
-			return URLEncoder.encode(request().host() + request().uri(), "UTF-8");
+			return URLEncoder.encode(request().host() + request().uri(), UTF_8);
 		} catch (UnsupportedEncodingException e) {
 			Logger.error("Could not get current URI", e);
 		}
@@ -349,7 +351,7 @@ public static Result register(final String t) {
 	public static Result journals() throws IOException {
 		try (InputStream stream = Play.application().classloader()
 				.getResourceAsStream("nwbib-journals.csv")) {
-			String csv = new String(Streams.copyToByteArray(stream), Charsets.UTF_8);
+			String csv = IOUtils.toString(stream, UTF_8);
 			List<String> lines = Arrays.asList(csv.split("\n"));
 			List<HashMap<String, String>> maps = lines.stream()
 					.filter(line -> line.split("\",\"").length == 2).map(line -> {
@@ -978,7 +980,7 @@ private static Promise<Result> transformAndIndex(String id, JsonNode jsonBody)
 			throws IOException, FileNotFoundException, RecognitionException, UnsupportedEncodingException {
 		File input = new File("conf/output/test-output-strapi.json");
 		File output = new File("conf/output/test-output-0.json");
-		Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charsets.UTF_8));
+		Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charset.forName(UTF_8)));
 		ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"});
 		String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n"));
 		Cache.remove(String.format("/%s", id));
@@ -988,6 +990,6 @@ private static Promise<Result> transformAndIndex(String id, JsonNode jsonBody)
 
 	private static String elasticsearchUrl(String id) throws UnsupportedEncodingException {
 		return "http://weywot3:9200/resources-rpb-test/resource/"
-				+ URLEncoder.encode("https://lobid.org/resources/" + id, "UTF-8");
+				+ URLEncoder.encode("https://lobid.org/resources/" + id, UTF_8);
 	}
 }

app/controllers/nwbib/Classification.java

@@ -13,6 +13,7 @@
 import java.text.Collator;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -42,14 +43,17 @@
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.transport.Netty4Plugin;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jsonldjava.core.JsonLdError;
@@ -194,6 +198,12 @@ public static Comparator<JsonNode> comparator(Function<JsonNode, String> fun) {
 				.compare(fun.apply(o1), fun.apply(o2));
 	}
 
+	private static class EmbeddedNode extends Node {
+		public EmbeddedNode(Settings preparedSettings, Collection<Class<? extends Plugin>> classpathPlugins) {
+			super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), classpathPlugins);
+		}
+	}
+
 	private Classification() {
 		/* Use via static functions, no instantiation. */
 	}
@@ -237,7 +247,7 @@ public static List<String> toJsonLd(final URL turtleUrl) {
 	public static JsonNode ids(String q, String t) {
 		QueryBuilder queryBuilder = QueryBuilders.boolQuery()
 				.must(QueryBuilders.idsQuery(Type.NWBIB.elasticsearchType,
-						Type.SPATIAL.elasticsearchType).ids(q));
+						Type.SPATIAL.elasticsearchType).addIds(q));
 		SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX)
 				.setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(queryBuilder);
 		if (t.isEmpty()) {
@@ -473,16 +483,21 @@ public static String shortId(String uri) {
 
 	/** Start up the embedded Elasticsearch classification index. */
 	public static void indexStartup() {
-		Settings clientSettings = ImmutableSettings.settingsBuilder()
-				.put("path.home", new File(".").getAbsolutePath())
-				.put("http.port",
-						play.Play.application().isTest() ? "8855"
-								: CONFIG.getString("index.es.port.http"))
-				.put("transport.tcp.port", play.Play.application().isTest() ? "8856"
-						: CONFIG.getString("index.es.port.tcp"))
-				.build();
-		node =
-				NodeBuilder.nodeBuilder().settings(clientSettings).local(true).node();
+		boolean isTest = play.Play.application().isTest();
+		String httpPort = isTest ? "8855" : CONFIG.getString("index.es.port.http");
+		String tcpPort = isTest ? "8856" : CONFIG.getString("index.es.port.tcp");
+		node = new EmbeddedNode(Settings.builder().put("transport.type", "netty4")//
+				.put("http.type", "netty4")//
+				.put("http.enabled", "true")//
+				.put("path.home", new File(".").getAbsolutePath())//
+				.put("http.port", httpPort)//
+				.put("transport.tcp.port", tcpPort)//
+				.build(), Arrays.asList(Netty4Plugin.class));
+		try {
+			node.start();
+		} catch (NodeValidationException e) {
+			e.printStackTrace();
+		}
 		client = node.client();
 		client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute()
 				.actionGet();
@@ -502,8 +517,8 @@ private static void indexData(String dataUrl, Type type) {
 			List<String> jsonLd = toJsonLd(new URL(dataUrl));
 			for (String concept : jsonLd) {
 				String id = Json.parse(concept).findValue("@id").textValue();
-				IndexRequestBuilder indexRequest = client
-						.prepareIndex(INDEX, type.elasticsearchType, id).setSource(concept);
+				IndexRequestBuilder indexRequest = client.prepareIndex(INDEX, type.elasticsearchType, id)
+						.setSource(concept.getBytes(), XContentType.JSON);
 				bulkRequest.add(indexRequest);
 			}
 		} catch (MalformedURLException e) {
@@ -517,7 +532,11 @@ private static void indexData(String dataUrl, Type type) {
 
 	/** Shut down the embedded Elasticsearch classification index. */
 	public static void indexShutdown() {
-		node.close();
+		try {
+			node.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
 	}
 
 	/**

app/controllers/nwbib/Lobid.java

@@ -20,10 +20,10 @@
 import java.util.stream.StreamSupport;
 
 import org.apache.commons.lang3.tuple.Pair;
-import org.elasticsearch.common.collect.Lists;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import com.google.common.html.HtmlEscapers;
 
 import controllers.nwbib.Classification.Type;

build.sbt

@@ -2,7 +2,7 @@ name := "rpb"
 
 version := "0.1.0-SNAPSHOT"
 
-scalaVersion := "2.11.11"
+scalaVersion := "2.11.12"
 
 resolvers += Resolver.mavenLocal
 
@@ -21,7 +21,9 @@ libraryDependencies ++= Seq(
   "org.metafacture" % "metafacture-csv" % "6.2.0",
   "org.metafacture" % "metafacture-linkeddata" % "6.2.0",
   "org.metafacture" % "metafix" % "1.2.0",
-  "org.elasticsearch" % "elasticsearch" % "1.7.5" withSources(),
+  "org.elasticsearch" % "elasticsearch" % "5.6.16" withSources(),
+  "org.elasticsearch.plugin" % "transport-netty4-client" % "5.6.16" withSources(),
+  "com.sun.xml.bind" % "jaxb-impl" % "2.3.3" withSources(),
   "com.github.jsonld-java" % "jsonld-java" % "0.5.0",
   "org.apache.commons" % "commons-rdf-jena" % "0.5.0",
   "org.apache.commons" % "commons-csv" % "1.6",
@@ -43,7 +45,7 @@ dependencyOverrides ++= Set(
 
 lazy val root = (project in file(".")).enablePlugins(PlayJava)
 
-javacOptions ++= Seq("-source", "1.8", "-target", "1.8")
+javacOptions ++= Seq("-source", "11", "-target", "11")
 
 import com.typesafe.sbteclipse.core.EclipsePlugin.EclipseKeys
 

project/build.properties

@@ -1 +1 @@
-sbt.version=0.13.16
+sbt.version=0.13.18