Skip to content

Commit 2667961

Browse files
fsteegfsteeg
committed
Merge remote-tracking branch 'origin/main' into 129-switchToJava11AndUpdateMFDependencies
2 parents 0b59e22 + f555c6a commit 2667961

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+863
-224
lines changed

‎app/controllers/nwbib/Application.java

Lines changed: 65 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,12 @@
3737
import org.apache.commons.lang3.tuple.Pair;
3838
import org.elasticsearch.common.geo.GeoPoint;
3939

40+
import com.fasterxml.jackson.core.JsonProcessingException;
41+
import com.fasterxml.jackson.databind.JsonMappingException;
4042
import com.fasterxml.jackson.databind.JsonNode;
4143
import com.fasterxml.jackson.databind.ObjectMapper;
44+
import com.fasterxml.jackson.databind.type.MapType;
45+
import com.fasterxml.jackson.databind.type.TypeFactory;
4246
import com.typesafe.config.Config;
4347
import com.typesafe.config.ConfigFactory;
4448

@@ -508,6 +512,11 @@ private static Promise<Result> okPromise(final String q, final String person,
508512
issued, medium, rpbspatial, rpbsubject, from, size, owner, t, sort,
509513
details, location, word, corporation, raw, format);
510514
return result.recover((Throwable throwable) -> {
515+
Logger.error("Error on Lobid call with q={}, person={}, name={}, subject={}, id={}, publisher={},\n"
516+
+ "issued={}, medium={}, rpbspatial={}, rpbsubject={}, from={}, size={}, owner={}, t={}, sort={},\n"
517+
+ "details={}, location={}, word={}, corporation={}, raw={}, format={}", //
518+
q, person, name, subject, id, publisher, issued, medium, rpbspatial, rpbsubject, from, size, owner,
519+
t, sort, details, location, word, corporation, raw, format);
511520
Logger.error("Could not call Lobid", throwable);
512521
flashError();
513522
return internalServerError(search.render("[]", q, person, name, subject,
@@ -970,6 +979,14 @@ public static Promise<Result> delete(String id, String secret) throws FileNotFou
970979
}
971980
}
972981

982+
public static Promise<Result> putIdFromData(String secret) throws FileNotFoundException, RecognitionException, IOException {
983+
return put(request().body().asJson().get("rpbId").textValue(), secret);
984+
}
985+
986+
public static Promise<Result> deleteIdFromData(String secret) throws FileNotFoundException, RecognitionException, IOException {
987+
return delete(request().body().asJson().get("rpbId").textValue(), secret);
988+
}
989+
973990
private static Promise<Result> deleteFromIndex(String id) throws UnsupportedEncodingException {
974991
Cache.remove(String.format("/%s", id));
975992
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
@@ -978,16 +995,60 @@ private static Promise<Result> deleteFromIndex(String id) throws UnsupportedEnco
978995

979996
private static Promise<Result> transformAndIndex(String id, JsonNode jsonBody)
980997
throws IOException, FileNotFoundException, RecognitionException, UnsupportedEncodingException {
998+
JsonNode transformedJson = transform(jsonBody);
999+
Promise<JsonNode> dataPromise = id.startsWith("f") && transformedJson.has("hbzId") ? // hbz-Fremddaten
1000+
addToLobidData(transformedJson) : Promise.pure(transformedJson);
1001+
return dataPromise.flatMap(result -> {
1002+
Cache.remove(String.format("/%s", id));
1003+
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
1004+
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
1005+
});
1006+
}
1007+
1008+
private static JsonNode transform(JsonNode jsonBody)
1009+
throws IOException, FileNotFoundException, RecognitionException {
9811010
File input = new File("conf/output/test-output-strapi.json");
9821011
File output = new File("conf/output/test-output-0.json");
9831012
Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charset.forName(UTF_8)));
9841013
ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"});
9851014
String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n"));
986-
Cache.remove(String.format("/%s", id));
987-
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
988-
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
1015+
return Json.parse(result);
9891016
}
990-
1017+
1018+
private static Promise<JsonNode> addToLobidData(JsonNode transformedJson) {
1019+
String lobidUrl = transformedJson.get("hbzId").textValue();
1020+
WSRequest lobidRequest = WS.url(lobidUrl).setQueryParameter("format", "json");
1021+
Promise<JsonNode> lobidPromise = lobidRequest.get().map(WSResponse::asJson);
1022+
Promise<JsonNode> merged = lobidPromise.map(lobidJson -> mergeRecords(transformedJson, lobidJson));
1023+
return merged;
1024+
}
1025+
1026+
private static JsonNode mergeRecords(JsonNode transformedJson, JsonNode lobidJson)
1027+
throws JsonMappingException, JsonProcessingException {
1028+
ObjectMapper objectMapper = new ObjectMapper();
1029+
MapType mapType = TypeFactory.defaultInstance().constructMapType(Map.class, String.class, Object.class);
1030+
Map<String, Object> transformedMap = objectMapper.readValue(transformedJson.toString(), mapType);
1031+
Map<String, Object> lobidMap = objectMapper.readValue(lobidJson.toString(), mapType);
1032+
lobidMap.remove("describedBy");
1033+
transformedMap.put("hbzId", lobidMap.get("hbzId"));
1034+
transformedMap.remove("type");
1035+
transformedMap.keySet().forEach(key -> {
1036+
Object transformedObject = transformedMap.get(key);
1037+
Object lobidObject = lobidMap.getOrDefault(key, new ArrayList<Object>());
1038+
Object values = transformedObject instanceof List ? mergeValues(transformedObject, lobidObject)
1039+
: transformedObject;
1040+
lobidMap.put(key, values);
1041+
});
1042+
return Json.toJson(lobidMap);
1043+
}
1044+
1045+
private static Object mergeValues(Object transformedObject, Object lobidObject) {
1046+
List<Object> mergedValues = lobidObject instanceof List ? new ArrayList<>((List<?>) lobidObject)
1047+
: Arrays.asList(lobidObject);
1048+
mergedValues.addAll((List<?>) transformedObject);
1049+
return mergedValues;
1050+
}
1051+
9911052
private static String elasticsearchUrl(String id) throws UnsupportedEncodingException {
9921053
return "http://weywot3:9200/resources-rpb-test/resource/"
9931054
+ URLEncoder.encode("https://lobid.org/resources/" + id, UTF_8);

‎app/views/TableRow.java

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
import controllers.nwbib.Classification;
1919
import controllers.nwbib.Lobid;
2020
import play.Logger;
21+
import play.libs.F.Promise;
22+
import play.libs.ws.WS;
23+
import play.libs.ws.WSRequest;
24+
import play.libs.ws.WSResponse;
2125

2226
/**
2327
* Different ways of serializing a table row
@@ -107,27 +111,29 @@ public String process(JsonNode doc, String property, String param,
107111
private String label(JsonNode doc, String value, List<String> properties) {
108112
List<String> results = new ArrayList<>();
109113
List<String> resultValues = labelsFor(doc, value, properties);
110-
JsonNode labelNode = doc.get(properties.get(0)).iterator().next().get("label");
111-
for (int i = 0; i < resultValues.size(); i++) {
112-
String currentValue = resultValues.get(i);
113-
String[] refAndLabel =
114-
refAndLabel(properties.get(i), currentValue, Optional.empty());
115-
String label = labelNode != null ? labelNode.textValue() : refAndLabel[1];
116-
String result =
117-
properties.get(i).equals("numbering") || value.equals("--")
118-
? currentValue
119-
: String.format(
120-
"<a title=\"Titeldetails anzeigen\" href=\"%s\">%s</a>",
121-
refAndLabel[0], label);
122-
results.add(result.replace("Band", "").trim());
114+
if (doc.get(properties.get(0)) != null) {
115+
JsonNode labelNode = doc.get(properties.get(0)).iterator().next().get("label");
116+
for (int i = 0; i < resultValues.size(); i++) {
117+
String currentValue = resultValues.get(i);
118+
String[] refAndLabel =
119+
refAndLabel(properties.get(i), currentValue, Optional.empty());
120+
String label = labelNode != null ? labelNode.textValue() : refAndLabel[1];
121+
String result =
122+
properties.get(i).equals("numbering") || value.equals("--")
123+
? currentValue
124+
: !value.startsWith("http") ? label : String.format(
125+
"<a title=\"Titeldetails anzeigen\" href=\"%s\">%s</a>",
126+
refAndLabel[0], label);
127+
results.add(result.replace("Band", "").trim());
128+
}
123129
}
124130
return results.stream().collect(Collectors.joining(", Band "));
125131
}
126132

127133
private List<String> labelsFor(JsonNode doc, String value,
128134
List<String> keys) {
129135
List<String> result = new ArrayList<>();
130-
if (doc != null) {
136+
if (doc != null && doc.get(keys.get(0)) != null) {
131137
JsonNode node = doc.get(keys.get(0)).iterator().next();
132138
JsonNode id = node.get("id");
133139
JsonNode label = node.get("label");
@@ -214,11 +220,14 @@ private static String lifeDates(JsonNode node) {
214220

215221
String[] refAndLabel(String property, String value,
216222
Optional<List<String>> labels) {
223+
if (value.contains("lobid.org/resources/")) {
224+
value = rpbUrlIfInRpb(value);
225+
}
217226
if ((property.equals("containedIn") || property.equals("hasPart")
218227
|| property.equals("isPartOf") || property.equals("hasSuperordinate")
219228
|| property.equals("bibliographicCitation")) && value.contains("lobid.org")) {
220-
return new String[] { value.matches(".*?[as]\\d+.*|.*?\\d{3}[a-z]\\d+.*") // rpbId
221-
? value.replace("https://lobid.org/resources/", "/")
229+
return new String[] { value.matches(".*?[asf]\\d+.*|.*?\\d{3}[a-z]\\d+.*") // rpbId
230+
? value.replaceAll("http.+/", "/") // full URL -> relative link
222231
: value, Lobid.resourceLabel(value) };
223232
}
224233
String label =
@@ -227,6 +236,12 @@ String[] refAndLabel(String property, String value,
227236
return new String[] { value, label };
228237
}
229238

239+
String rpbUrlIfInRpb(String value) {
240+
WSRequest lobidRequest = WS.url(value).setHeader("Content-Type", "application/json");
241+
JsonNode lobidJson = lobidRequest.get().map(WSResponse::asJson).get(Lobid.API_TIMEOUT);
242+
return lobidJson.has("rpbId") ? "https://rpb.lobid.org/" + lobidJson.get("rpbId").textValue() : value;
243+
}
244+
230245
public abstract String process(JsonNode doc, String property, String param,
231246
String label, List<String> values, Optional<List<String>> labels);
232247
}

‎app/views/tags/result_doc.scala.html

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838

3939
@publication(pub: JsValue) = {
4040
@defining(((pub\"startDate"),(pub\"endDate"))) { case (start,end) =>
41-
@if(start.asOpt[String].isDefined || end.asOpt[String].isDefined){
4241
<tr>
4342
<td>
4443
@if((pub \ "type").toString.contains("SecondaryPublicationEvent")){
@@ -53,11 +52,12 @@
5352
(doc\"type").toString.contains("Series") || (doc\"type").toString.contains("Periodical"))) {
5453
@* Show range if either both start and end are given, or if we want open ranges like "- 2015" *@
5554
@start.asOpt[String].getOrElse("") &ndash; @end.asOpt[String].getOrElse("")
56-
} else {
55+
<br/>
56+
} else { @if(start.asOpt[String].isDefined || end.asOpt[String].isDefined) {
5757
@* Otherwise show only one date, no range (no periodical or series, and just one date given) *@
5858
@start.asOpt[String].getOrElse(end.asOpt[String].getOrElse(""))
59-
}
60-
<br/>
59+
<br/>
60+
}}
6161
@optional("Erscheinungsort", "location", pub)
6262
@optional("Verlag", "publishedBy", pub)
6363
@((pub\"frequency").asOpt[Seq[JsValue]].map { freq =>
@@ -66,7 +66,6 @@
6666
@optional("Erscheinungsverlauf", "publicationHistory", pub)
6767
</td>
6868
</tr>
69-
}
7069
}
7170
}
7271

@@ -96,10 +95,10 @@
9695
}
9796
}
9897

99-
@subordinate(field:String, id:String, label:String, values:(String,String)) = {
100-
@defining(Lobid.getTotalHits(field, id, CONFIG.getString("nwbib.filter")).get(Lobid.API_TIMEOUT)){ hits => @if(hits > 0) {
98+
@subordinate(field:String, ids:Seq[String], label:String, values:(String,String)) = {
99+
@defining(ids.map((id) => Lobid.getTotalHits(field, id, CONFIG.getString("nwbib.filter")).get(Lobid.API_TIMEOUT)).reduce((a, b) => a + b)){ hits => @if(hits > 0) {
101100
@defining(if(hits==1){values._1} else {values._2}){ value =>
102-
<tr><td>@label</td><td><a title="@value" href="@nwbib.routes.Application.search(raw=field+":\""+Lobid.escapeUri(id)+"\"")">@hits @value</a></td></tr>
101+
<tr><td>@label</td><td><a title="@value" href="@nwbib.routes.Application.search(raw=ids.map((id)=>field+":\""+Lobid.escapeUri(id)+"\"").mkString(" OR "))">@hits @value</a></td></tr>
103102
}}}
104103
}
105104

@@ -188,11 +187,11 @@
188187

189188
@raumsystematik(source: String) = {
190189
@defining(((doc\"spatial").asOpt[Seq[JsValue]],(doc\"coverage").asOpt[Seq[JsValue]])) { case (spatial, coverage) =>
191-
@for(vs <- spatial.orElse(coverage)) {
190+
@for(vs <- spatial.orElse(coverage); if vs.toString.contains(source)) {
192191
<tr>
193192
<td>Raumsystematik</td>
194193
<td>
195-
@for(v <- vs; id = (v\"id").asOpt[String].getOrElse("--"); if(!Seq(35, 37, 96, 97).find((i: Int) => id.endsWith("#N"+i)))) {
194+
@for(v <- vs; if v.toString.contains(source); id = (v\"id").asOpt[String].getOrElse("--"); if(!Seq(35, 37, 96, 97).find((i: Int) => id.endsWith("#N"+i)))) {
196195
@breadcrumb("",id)
197196
@if(spatial.isDefined){
198197
| <a href='@nwbib.routes.Application.search()?rpbspatial=@((v\"id").asOpt[String].getOrElse("--").replace("#", "%23"))' title='Nach weiteren Titeln zu @((v\"label").asOpt[String].getOrElse("--")) (@(id.split("#").last)) suchen'>
@@ -237,9 +236,9 @@
237236
}
238237
}
239238

240-
@subordinateSearchFor(id: String) = {
241-
@subordinate("isPartOf.hasSuperordinate.id", id, "Bände", ("zugehöriger Band", "zugehörige Bände"))
242-
@subordinate("containedIn.id", id, "Enthält", ("Beitrag", "Beiträge"))
239+
@subordinateSearchFor(ids: Seq[String]) = {
240+
@subordinate("isPartOf.hasSuperordinate.id", ids, "Bände", ("zugehöriger Band", "zugehörige Bände"))
241+
@subordinate("containedIn.id", ids, "Enthält", ("Beitrag", "Beiträge"))
243242
}
244243

245244
@table(){
@@ -267,14 +266,17 @@
267266
@labelled("In", "containedIn")
268267

269268
@part_of("isPartOf", "hasSuperordinate")
270-
@subordinateSearchFor(String.format("http://lobid.org/resources/%s#!", (doc \ "hbzId").asOpt[String].getOrElse("")))
271-
@subordinateSearchFor(String.format("https://lobid.org/resources/%s", (doc \ "rpbId").asOpt[String].getOrElse("")))
272-
@subordinateSearchFor(String.format("http://lobid.org/resources/"+ZDB_PREFIX+"%s#!", (doc \ "zdbId").asOpt[String].getOrElse("")))
269+
@subordinateSearchFor(Seq(
270+
String.format("http://lobid.org/resources/"+ZDB_PREFIX+"%s#!", (doc \ "zdbId").asOpt[String].getOrElse("")),
271+
String.format("http://lobid.org/resources/%s#!", (doc \ "hbzId").asOpt[String].getOrElse("")),
272+
String.format("https://lobid.org/resources/%s", (doc \ "rpbId").asOpt[String].getOrElse("")),
273+
String.format("http://rpb.lobid.org/%s", (doc \ "rpbId").asOpt[String].getOrElse(""))
274+
))
273275
@parallelausgabe()
274276
@raumsystematik("https://rpb.lobid.org/spatial")
275277
@sachsystematik("http://purl.org/lobid/rpb")
276278

277-
@subjects((doc \ "subject").asOpt[Seq[JsValue]].getOrElse(Seq()).filter(v => !(v \ "source").toString.matches(".*(Systematik|Notationen|Dewey).*")))
279+
@subjects((doc \ "subject").asOpt[Seq[JsValue]].getOrElse(Seq()).filter(v => !(v \ "source").toString.matches(".*(Systematik|Notationen|Sachgruppen|Dewey).*")))
278280

279281
@result_field("Schlagwortfolge", "subjectChain", doc, TableRow.VALUES, valueLabel = Option(Seq()))
280282

‎build.sbt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name := "rpb"
22

3-
version := "0.1.0-SNAPSHOT"
3+
version := "0.1.1-SNAPSHOT"
44

55
scalaVersion := "2.11.12"
66

0 commit comments

Comments
 (0)