Create lookup file maps from Strapi, not Allegro data (RPB-154)

Author: fsteeg

.gitignore

@@ -22,4 +22,5 @@ RPB-Export_HBZ_SWN.txt
 RPB-Export_HBZ_Syst.txt
 RPB-Export_HBZ_ZSS.txt
 conf/RPBEXP/*.ZIP
+conf/strapi-export.tar.gz
 nohup.out*

conf/maps/gndId-to-rppdId.tsv

@@ -2,9 +2,9 @@ unless any_contain("gndIdentifier", "Keine")
 	reject()
 end
 
-copy_field("rppdId", "rppdUri")
-prepend("rppdUri", "https://rppd.lobid.org/")
+copy_field("rppdId", "uri")
+prepend("uri", "https://rppd.lobid.org/")
 replace_all("preferredName", "\\/\\s(ca\\.|um)?-?\\s?\\d.+$", "")
 trim("preferredName")
 
-retain("rppdId", "rppdUri", "preferredName")
+retain("rppdId", "preferredName", "uri")

conf/maps/rppdId-with-label.tsv

@@ -1,7 +1,7 @@
-FLUX_DIR + "output/output-rppd-strapi.ndjson"
+FLUX_DIR + "output/rppd-export.jsonl"
 | open-file
 | as-lines
-| decode-json
+| decode-json(recordPath="data")
 | fix(FLUX_DIR + "rppd-rppdId-with-label-map.fix")
 | encode-csv(includeheader="true", noquotes="true",separator="\t")
 | write(FLUX_DIR + "maps/rppdId-with-label.tsv")

conf/rppd-rppdId-with-label-map.fix

@@ -2,7 +2,9 @@ if any_contain("gndIdentifier", "Keine")
 	reject()
 end
 
-prepend("gndIdentifier", "https://d-nb.info/gnd/")
+copy_field("gndIdentifier", "uri")
+prepend("uri", "https://d-nb.info/gnd/")
 replace_all("preferredName", "\\/\\s(ca\\.|um)?-?\\s?\\d.+$", "")
 trim("preferredName")
-retain("rppdId", "gndIdentifier", "preferredName")
+
+retain("rppdId", "preferredName", "uri")

conf/rppd-rppdId-with-label-map.flux

@@ -1,7 +1,7 @@
-FLUX_DIR + "output/output-rppd-strapi.ndjson"
+FLUX_DIR + "output/rppd-export.jsonl"
 | open-file
 | as-lines
-| decode-json
+| decode-json(recordPath="data")
 | fix(FLUX_DIR + "rppd-to-gnd-map.fix")
 | encode-csv(includeheader="true", noquotes="true",separator="\t")
 | write(FLUX_DIR + "maps/gndId-to-rppdId.tsv")

conf/rppd-to-gnd-map.fix

@@ -26,7 +26,7 @@ do once("map")
   put_filemap("conf/maps/gndGeographicName.tsv", "gnd_spatial_map", key_column:"0", value_column:"1", sep_char: "\t", expected_columns:"2")
 
   # maps für lookup relatedPerson
-  put_filemap("conf/maps/gndId-to-rppdId.tsv", "map_rel_preferredName",key_column:"1",value_column:"2", sep_char: "\t", expected_columns:"-1")
+  put_filemap("conf/maps/gndId-to-rppdId.tsv", "map_rel_preferredName",key_column:"2",value_column:"1", sep_char: "\t", expected_columns:"-1")
   put_filemap("conf/maps/rppdId-with-label.tsv", "map_rel_rppdLabel",key_column:"0",value_column:"1", sep_char: "\t", expected_columns:"-1")
 
   # maps für depiction

conf/rppd-to-gnd-mapping.flux

@@ -3,13 +3,13 @@ set -u
 
 bash transformBeacons.sh
 rm conf/output/bulk/rppd/*
-sbt "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio.txt OUT_FILE=output-rppd-strapi.ndjson"
-sbt "runMain rpb.ETL conf/rppd-to-gnd-mapping.flux"
-sbt "runMain rpb.ETL conf/rppd-rppdId-with-label-map.flux"
 # Here, we used to import Allegro data:
+# sbt "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio.txt OUT_FILE=output-rppd-strapi.ndjson"
 # sbt "runMain rpb.ETL conf/rppd-to-lobid.flux"
 # But now we use the Strapi export:
 zgrep -a '"type":"api::person.person"' conf/strapi-export.tar.gz > conf/output/rppd-export.jsonl
+sbt "runMain rpb.ETL conf/rppd-to-gnd-mapping.flux"
+sbt "runMain rpb.ETL conf/rppd-rppdId-with-label-map.flux"
 sbt "runMain rpb.ETL conf/rppd-to-lobid.flux IN_FILE=rppd-export.jsonl RECORD_PATH=data"
 
 # Indexing happens in rppd/transformAndIndexRppd.sh (lobid-gnd repo, branch 'rppd'), which calls this script