Skip to content

Commit 3f5be46

Browse files
Control versions of indirect dependencies from hadoop (which is neede… (#218)
* Control versions of indirect dependencies from hadoop (which is needed for parquet at least for now). * Include specific version of protobuf-java for indirect dependencies.
1 parent db58e31 commit 3f5be46

File tree

3 files changed

+56
-8
lines changed

3 files changed

+56
-8
lines changed

WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java

+2-6
Original file line numberDiff line numberDiff line change
@@ -66,16 +66,15 @@
6666
import javax.sound.sampled.AudioInputStream;
6767
import javax.sound.sampled.AudioSystem;
6868
import org.apache.hadoop.conf.Configuration;
69-
import org.apache.hadoop.fs.Path;
7069
import org.apache.parquet.ParquetReadOptions;
7170
import org.apache.parquet.column.page.PageReadStore;
7271
import org.apache.parquet.example.data.simple.convert.GroupRecordConverter;
7372
import org.apache.parquet.hadoop.ParquetFileReader;
7473
import org.apache.parquet.hadoop.ParquetWriter;
7574
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
76-
import org.apache.parquet.hadoop.util.HadoopInputFile;
7775
import org.apache.parquet.io.ColumnIOFactory;
7876
import org.apache.parquet.io.InputFile;
77+
import org.apache.parquet.io.LocalInputFile;
7978
import org.apache.parquet.io.LocalOutputFile;
8079
import org.apache.parquet.io.MessageColumnIO;
8180
import org.apache.parquet.io.RecordReader;
@@ -15956,10 +15955,7 @@ public void readParquet(
1595615955
String fullFileName, StringArray colNames, String[] colTypes, boolean simplify)
1595715956
throws Exception {
1595815957
clear();
15959-
Path filePath = new Path(fullFileName);
15960-
Configuration conf = new Configuration();
15961-
15962-
InputFile parquetFile = HadoopInputFile.fromPath(filePath, conf);
15958+
InputFile parquetFile = new LocalInputFile(java.nio.file.Path.of(fullFileName));
1596315959
ParquetFileReader fileReader =
1596415960
new ParquetFileReader(parquetFile, ParquetReadOptions.builder().build());
1596515961
try {

pom.xml

+50
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,50 @@
521521
<type>pom</type>
522522
<scope>import</scope>
523523
</dependency>
524+
525+
<!-- This is needed due to indirect dependency in cdm-core -->
526+
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
527+
<dependency>
528+
<groupId>com.google.protobuf</groupId>
529+
<artifactId>protobuf-java</artifactId>
530+
<version>3.25.5</version>
531+
</dependency>
532+
533+
<!--
534+
The below are here to force updated dependencies of the hadoop libraries.
535+
The default selections have security vulnerabilities. Ideally we wouldn't be including
536+
hadoop at all, but parquet currently has dependencies on hadoop to read and write files.
537+
-->
538+
<!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
539+
<dependency>
540+
<groupId>org.apache.avro</groupId>
541+
<artifactId>avro</artifactId>
542+
<version>1.12.0</version>
543+
</dependency>
544+
<!-- https://mvnrepository.com/artifact/dnsjava/dnsjava -->
545+
<dependency>
546+
<groupId>dnsjava</groupId>
547+
<artifactId>dnsjava</artifactId>
548+
<version>3.6.2</version>
549+
</dependency>
550+
<!-- https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on -->
551+
<dependency>
552+
<groupId>org.bouncycastle</groupId>
553+
<artifactId>bcprov-jdk18on</artifactId>
554+
<version>1.78.1</version>
555+
</dependency>
556+
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-configuration2 -->
557+
<dependency>
558+
<groupId>org.apache.commons</groupId>
559+
<artifactId>commons-configuration2</artifactId>
560+
<version>2.11.0</version>
561+
</dependency>
562+
<!-- https://mvnrepository.com/artifact/com.nimbusds/nimbus-jose-jwt -->
563+
<dependency>
564+
<groupId>com.nimbusds</groupId>
565+
<artifactId>nimbus-jose-jwt</artifactId>
566+
<version>9.41.2</version>
567+
</dependency>
524568
</dependencies>
525569
</dependencyManagement>
526570

@@ -932,6 +976,12 @@
932976
<groupId>org.apache.hadoop</groupId>
933977
<artifactId>hadoop-client</artifactId>
934978
<version>3.4.0</version>
979+
<exclusions>
980+
<exclusion>
981+
<groupId>org.bouncycastle</groupId>
982+
<artifactId>bcprov-jdk15on</artifactId>
983+
</exclusion>
984+
</exclusions>
935985
</dependency>
936986
<dependency>
937987
<groupId>org.apache.parquet</groupId>

src/test/java/gov/noaa/pfel/coastwatch/pointdata/TableTests.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -18897,8 +18897,10 @@ void testMdb() throws Exception {
1889718897
void testReadParquet() throws Exception {
1889818898

1889918899
String fileName =
18900-
TableTests.class
18901-
.getResource("/data/parquet/GHG_national_2012_m1_v2.0.0_a8c5929.parquet")
18900+
new java.io.File(
18901+
TableTests.class
18902+
.getResource("/data/parquet/GHG_national_2012_m1_v2.0.0_a8c5929.parquet")
18903+
.getFile())
1890218904
.getPath();
1890318905
// don't simplify
1890418906
Table table = new Table();

0 commit comments

Comments
 (0)