Skip to content

Commit 44c825e

Browse files
Merge pull request #223 from ChrisJohnNOAA/2.25.1-Patches
2.25.1 patches
2 parents c1718a7 + 16c1bb2 commit 44c825e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+291
-95
lines changed

DEPLOY_INSTALL.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ ERDDAP™ can run on any server that supports Java and Tomcat (and other applica
212212

213213
4. [Install the erddap.war file.](#erddap.war)
214214
On Linux, Mac, and Windows, download [erddap.war](https://github.com/ERDDAP/erddap/releases/download/v2.25/erddap.war) into _tomcat_/webapps .
215-
(version 2.25, 592,429,675 bytes, MD5=0D93F045A3F38018117C0BB5BA419C99, dated 2024-10-17)
215+
(version 2.25, 592,291,920 bytes, MD5=BEEBE386A3514C0FB8898C6EA597F40D, dated 2024-10-31)
216216

217217
The .war file is big because it contains high resolution coastline, boundary, and elevation data needed to create maps.
218218

DEPLOY_UPDATE.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
2. If you are upgrading from ERDDAP™ version 2.18 or below, you need to switch to Java 21 (or newer) and the related Tomcat 10. See the regular ERDDAP™ installation instructions for [Java](#java) and [Tomcat](#tomcat). You'll also have to copy your _tomcat_/content/erddap directory from your old Tomcat installation to your new Tomcat installation.
77

88
3. Download [erddap.war](https://github.com/ERDDAP/erddap/releases/download/v2.25/erddap.war) into _tomcat_/webapps .
9-
(version 2.25, 592,429,675 bytes, MD5=0D93F045A3F38018117C0BB5BA419C99, dated 2024-10-17)
9+
(version 2.25, 592,291,920 bytes, MD5=BEEBE386A3514C0FB8898C6EA597F40D, dated 2024-10-31)
1010

1111
4. [messages.xml](#messages.xml)
1212
* Common: If you are upgrading from ERDDAP™ version 1.46 (or above) and you just use the standard messages, the new standard messages.xml will be installed automatically (amongst the .class files via erddap.war).

WEB-INF/classes/com/cohort/util/Calendar2.java

+1
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ public class Calendar2 {
9090
public static final long MILLIS_PER_DAY = SECONDS_PER_DAY * 1000L;
9191

9292
public static final String SECONDS_SINCE_1970 = "seconds since 1970-01-01T00:00:00Z";
93+
public static final String MILLISECONDS_SINCE_1970 = "milliseconds since 1970-01-01T00:00:00Z";
9394

9495
public static final String zulu = "Zulu";
9596
public static final TimeZone zuluTimeZone = TimeZone.getTimeZone(zulu);

WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java

+105-32
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,12 @@
7878
import org.apache.parquet.io.LocalOutputFile;
7979
import org.apache.parquet.io.MessageColumnIO;
8080
import org.apache.parquet.io.RecordReader;
81+
import org.apache.parquet.schema.LogicalTypeAnnotation;
82+
import org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit;
8183
import org.apache.parquet.schema.MessageType;
82-
import org.apache.parquet.schema.MessageTypeParser;
84+
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
8385
import org.apache.parquet.schema.Type;
86+
import org.apache.parquet.schema.Types.MessageTypeBuilder;
8487
import org.xml.sax.InputSource;
8588
import org.xml.sax.XMLReader;
8689
import ucar.ma2.*;
@@ -588,7 +591,7 @@ private static interface WithColumnNames {
588591
public static BitSet ncCFcc = null; // null=inactive, new BitSet() = active
589592

590593
/** An arrayList to hold 0 or more PrimitiveArray's with data. */
591-
protected ArrayList<PrimitiveArray> columns = new ArrayList();
594+
protected ArrayList<PrimitiveArray> columns = new ArrayList<>();
592595

593596
/** An arrayList to hold the column names. */
594597
protected StringArray columnNames = new StringArray();
@@ -605,7 +608,7 @@ private static interface WithColumnNames {
605608
* Although a HashTable is more appropriate for name=value pairs, this uses ArrayList to preserve
606609
* the order of the attributes. This may be null if not in use.
607610
*/
608-
protected ArrayList<Attributes> columnAttributes = new ArrayList();
611+
protected ArrayList<Attributes> columnAttributes = new ArrayList<>();
609612

610613
/** The one known valid url for readIobis. */
611614
public static final String IOBIS_URL = "http://www.iobis.org/OBISWEB/ObisControllerServlet";
@@ -16089,52 +16092,91 @@ public void readParquet(
1608916092
}
1609016093
}
1609116094

16092-
private MessageType getParquetSchemaForTable(String name) {
16093-
String schemaProto = "message m {";
16095+
private boolean isTimeColumn(int col) {
16096+
return "time".equalsIgnoreCase(getColumnName(col))
16097+
&& Calendar2.SECONDS_SINCE_1970.equals(columnAttributes.get(col).getString("units"));
16098+
}
16099+
16100+
private MessageType getParquetSchemaForTable() {
16101+
MessageTypeBuilder schemaBuilder = org.apache.parquet.schema.Types.buildMessage();
1609416102
for (int j = 0; j < nColumns(); j++) {
16095-
String schemaType = "String";
16103+
String columnName = getColumnName(j);
16104+
if (isTimeColumn(j)) {
16105+
schemaBuilder
16106+
.optional(PrimitiveTypeName.INT64)
16107+
.as(LogicalTypeAnnotation.timestampType(true, TimeUnit.MILLIS))
16108+
.named(columnName);
16109+
continue;
16110+
}
1609616111
switch (getColumn(j).elementType()) {
1609716112
case BYTE:
16098-
schemaType = "INT32";
16113+
schemaBuilder.optional(PrimitiveTypeName.INT32).named(columnName);
1609916114
break;
1610016115
case SHORT:
16101-
schemaType = "INT32";
16116+
schemaBuilder.optional(PrimitiveTypeName.INT32).named(columnName);
1610216117
break;
1610316118
case CHAR:
16104-
schemaType = "BINARY";
16119+
schemaBuilder
16120+
.optional(PrimitiveTypeName.BINARY)
16121+
.as(LogicalTypeAnnotation.stringType())
16122+
.named(columnName);
1610516123
break;
1610616124
case INT:
16107-
schemaType = "INT32";
16125+
schemaBuilder.optional(PrimitiveTypeName.INT32).named(columnName);
1610816126
break;
1610916127
case LONG:
16110-
schemaType = "INT64";
16128+
schemaBuilder.optional(PrimitiveTypeName.INT64).named(columnName);
1611116129
break;
1611216130
case FLOAT:
16113-
schemaType = "FLOAT";
16131+
schemaBuilder.optional(PrimitiveTypeName.FLOAT).named(columnName);
1611416132
break;
1611516133
case DOUBLE:
16116-
schemaType = "DOUBLE";
16134+
schemaBuilder.optional(PrimitiveTypeName.DOUBLE).named(columnName);
1611716135
break;
1611816136
case STRING:
16119-
schemaType = "BINARY";
16137+
schemaBuilder
16138+
.optional(PrimitiveTypeName.BINARY)
16139+
.as(LogicalTypeAnnotation.stringType())
16140+
.named(columnName);
1612016141
break;
1612116142
case UBYTE:
16122-
schemaType = "INT32";
16143+
schemaBuilder.optional(PrimitiveTypeName.INT32).named(columnName);
1612316144
break;
1612416145
case USHORT:
16125-
schemaType = "INT32";
16146+
schemaBuilder.optional(PrimitiveTypeName.INT32).named(columnName);
1612616147
break;
1612716148
case UINT:
16128-
schemaType = "INT64";
16149+
schemaBuilder.optional(PrimitiveTypeName.INT64).named(columnName);
1612916150
break;
1613016151
case ULONG:
16131-
schemaType = "DOUBLE";
16152+
schemaBuilder.optional(PrimitiveTypeName.DOUBLE).named(columnName);
1613216153
break;
16154+
case BOOLEAN:
16155+
schemaBuilder.optional(PrimitiveTypeName.BOOLEAN).named(columnName);
16156+
break;
16157+
}
16158+
}
16159+
return schemaBuilder.named("m");
16160+
}
16161+
16162+
private void addMetadata(Map<String, String> metadata, Attributes attributes, String prefix) {
16163+
String names[] = attributes.getNames();
16164+
for (int ni = 0; ni < names.length; ni++) {
16165+
String tName = names[ni];
16166+
if (!String2.isSomething(tName)) {
16167+
continue;
16168+
}
16169+
PrimitiveArray tValue = attributes.get(tName);
16170+
if (tValue == null || tValue.size() == 0 || tValue.toString().length() == 0) {
16171+
continue; // do nothing
16172+
}
16173+
if ("time_".equalsIgnoreCase(prefix)
16174+
&& Calendar2.SECONDS_SINCE_1970.equals(attributes.getString(tName))) {
16175+
metadata.put(prefix + tName, Calendar2.MILLISECONDS_SINCE_1970);
16176+
} else {
16177+
metadata.put(prefix + tName, tValue.toCSVString());
1613316178
}
16134-
schemaProto += " optional " + schemaType + " " + getColumnName(j) + ";\n";
1613516179
}
16136-
schemaProto += "}";
16137-
return MessageTypeParser.parseMessageType(schemaProto);
1613816180
}
1613916181

1614016182
/**
@@ -16143,23 +16185,49 @@ private MessageType getParquetSchemaForTable(String name) {
1614316185
* @param fullFileName This is just used for error messages.
1614416186
* @throws Exception if trouble, including observed nItems != expected nItems.
1614516187
*/
16146-
public void writeParquet(String fullFileName) throws Exception {
16188+
public void writeParquet(String fullFileName, boolean fullMetadata) throws Exception {
1614716189
String msg = " Table.writeParquet " + fullFileName;
1614816190
long time = System.currentTimeMillis();
1614916191

1615016192
int randomInt = Math2.random(Integer.MAX_VALUE);
16151-
16152-
int nameStart = fullFileName.lastIndexOf('/');
16153-
if (nameStart == -1) {
16154-
nameStart = fullFileName.lastIndexOf('\\');
16193+
MessageType schema = getParquetSchemaForTable();
16194+
16195+
Map<String, String> metadata = new HashMap<>();
16196+
if (fullMetadata) {
16197+
addMetadata(metadata, globalAttributes, "");
16198+
for (int col = 0; col < nColumns(); col++) {
16199+
Attributes colAttributes = columnAttributes.get(col);
16200+
if (colAttributes == null) {
16201+
continue;
16202+
}
16203+
addMetadata(metadata, colAttributes, getColumnName(col) + "_");
16204+
}
1615516205
}
16156-
int nameEnd = fullFileName.lastIndexOf('.');
16157-
String name = fullFileName.substring(nameStart + 1, nameEnd);
16158-
MessageType schema = getParquetSchemaForTable(name);
16159-
16206+
String columnNames = "";
16207+
String columnUnits = "";
16208+
for (int col = 0; col < nColumns(); col++) {
16209+
Attributes colAttributes = columnAttributes.get(col);
16210+
if (colAttributes == null) {
16211+
continue;
16212+
}
16213+
if (columnNames.length() > 0) {
16214+
columnNames += ",";
16215+
columnUnits += ",";
16216+
}
16217+
columnNames += getColumnName(col);
16218+
if (isTimeColumn(col)) {
16219+
columnUnits += Calendar2.MILLISECONDS_SINCE_1970;
16220+
} else {
16221+
columnUnits += colAttributes.getString("units");
16222+
}
16223+
}
16224+
metadata.put("column_names", columnNames);
16225+
metadata.put("column_units", columnUnits);
1616016226
try (ParquetWriter<List<PAOne>> writer =
1616116227
new ParquetWriterBuilder(
16162-
schema, new LocalOutputFile(java.nio.file.Path.of(fullFileName + randomInt)))
16228+
schema,
16229+
new LocalOutputFile(java.nio.file.Path.of(fullFileName + randomInt)),
16230+
metadata)
1616316231
.withCompressionCodec(CompressionCodecName.SNAPPY)
1616416232
.withRowGroupSize(ParquetWriter.DEFAULT_BLOCK_SIZE)
1616516233
.withPageSize(ParquetWriter.DEFAULT_PAGE_SIZE)
@@ -16171,7 +16239,12 @@ schema, new LocalOutputFile(java.nio.file.Path.of(fullFileName + randomInt)))
1617116239
for (int row = 0; row < nRows(); row++) {
1617216240
ArrayList<PAOne> record = new ArrayList<>();
1617316241
for (int j = 0; j < nColumns(); j++) {
16174-
record.add(getPAOneData(j, row));
16242+
if (isTimeColumn(j)) {
16243+
// Convert from seconds since epoch to millis since epoch.
16244+
record.add(getPAOneData(j, row).multiply(PAOne.fromInt(1000)));
16245+
} else {
16246+
record.add(getPAOneData(j, row));
16247+
}
1617516248
}
1617616249
writer.write(record);
1617716250
}

WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/parquet/CustomWriteSupport.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
package gov.noaa.pfel.coastwatch.pointdata.parquet;
22

33
import com.cohort.array.PAOne;
4-
import java.util.HashMap;
54
import java.util.List;
5+
import java.util.Map;
66
import org.apache.hadoop.conf.Configuration;
77
import org.apache.parquet.column.ColumnDescriptor;
88
import org.apache.parquet.hadoop.api.WriteSupport;
@@ -15,15 +15,17 @@ public class CustomWriteSupport extends WriteSupport<List<PAOne>> {
1515
MessageType schema;
1616
RecordConsumer recordConsumer;
1717
List<ColumnDescriptor> cols;
18+
private Map<String, String> metadata;
1819

19-
CustomWriteSupport(MessageType schema) {
20+
CustomWriteSupport(MessageType schema, Map<String, String> metadata) {
2021
this.schema = schema;
2122
this.cols = schema.getColumns();
23+
this.metadata = metadata;
2224
}
2325

2426
@Override
2527
public WriteContext init(Configuration config) {
26-
return new WriteContext(schema, new HashMap<String, String>());
28+
return new WriteContext(schema, metadata);
2729
}
2830

2931
@Override
@@ -51,7 +53,7 @@ public void write(List<PAOne> values) {
5153
// val.length() == 0 indicates a NULL value.
5254
if (val != null && !val.isMissingValue()) {
5355
recordConsumer.startField(cols.get(i).getPath()[0], i);
54-
switch (cols.get(i).getType()) {
56+
switch (cols.get(i).getPrimitiveType().getPrimitiveTypeName()) {
5557
case BOOLEAN:
5658
recordConsumer.addBoolean(Boolean.parseBoolean(val.getString()));
5759
break;

WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/parquet/ParquetWriterBuilder.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.cohort.array.PAOne;
44
import java.util.List;
5+
import java.util.Map;
56
import org.apache.hadoop.conf.Configuration;
67
import org.apache.parquet.hadoop.ParquetWriter;
78
import org.apache.parquet.hadoop.api.WriteSupport;
@@ -12,9 +13,9 @@ public class ParquetWriterBuilder extends ParquetWriter.Builder<List<PAOne>, Par
1213

1314
private CustomWriteSupport writeSupport;
1415

15-
public ParquetWriterBuilder(MessageType schema, OutputFile file) {
16+
public ParquetWriterBuilder(MessageType schema, OutputFile file, Map<String, String> metadata) {
1617
super(file);
17-
writeSupport = new CustomWriteSupport(schema);
18+
writeSupport = new CustomWriteSupport(schema, metadata);
1819
}
1920

2021
@Override

WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ public class Erddap extends HttpServlet {
157157
".jsonlCSV", ".jsonlKVP", ".nccsv", ".nccsvMetadata"
158158
};
159159
public static final String FILE_TYPES_184[] = {".dataTable", ".jsonlCSV1"};
160-
public static final String FILE_TYPES_225[] = {".parquet"};
160+
public static final String FILE_TYPES_225[] = {".parquet", ".parquetWMeta"};
161161
// General/relative width is determined by what looks good in Chrome.
162162
// But Firefox shows TextArea's as very wide, so leads to these values.
163163
public static final int dpfTFWidth = 56; // data provider form TextField width
@@ -4752,7 +4752,7 @@ public void doStatus(
47524752
+ EDStatic.youAreHere(language, loggedInAs, EDStatic.statusAr[language])
47534753
+ "<pre>");
47544754
StringBuilder sb = new StringBuilder();
4755-
EDStatic.addIntroStatistics(sb);
4755+
EDStatic.addIntroStatistics(sb, EDStatic.showLoadErrorsOnStatusPage);
47564756

47574757
// append number of active threads
47584758
String traces = MustBe.allStackTraces(true, true);

WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1215,7 +1215,7 @@ private void emailOrphanDatasetsRemoved(
12151215

12161216
private void emailUnusualActivity(String threadSummary, String threadList) {
12171217
StringBuilder sb = new StringBuilder();
1218-
EDStatic.addIntroStatistics(sb);
1218+
EDStatic.addIntroStatistics(sb, true /* includeErrors */);
12191219

12201220
if (threadSummary != null) sb.append(threadSummary + "\n");
12211221

@@ -1280,7 +1280,7 @@ private void emailDailyReport(String threadSummary, String threadList, String re
12801280
String stars = String2.makeString('*', 70);
12811281
String subject = "Daily Report";
12821282
StringBuilder contentSB = new StringBuilder(subject + "\n\n");
1283-
EDStatic.addIntroStatistics(contentSB);
1283+
EDStatic.addIntroStatistics(contentSB, true /* includeErrors */);
12841284

12851285
// append number of active threads
12861286
if (threadSummary != null) contentSB.append(threadSummary + "\n");

0 commit comments

Comments
 (0)