From 23ea4e1cebb029d2a47d2db66c6aee06d06b9acb Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 13 Jul 2023 11:52:38 -0600 Subject: [PATCH 1/3] Add tests for the location of the create grib index for local files --- .../ucar/nc2/grib/TestGribIndexLocation.java | 163 ++++++++++++++++++ grib/src/test/data/index/cosmo-eu.grib2.gbx9 | Bin 0 -> 446 bytes .../test/data/index/radar_national.grib1.gbx9 | Bin 0 -> 208 bytes 3 files changed, 163 insertions(+) create mode 100644 cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocation.java create mode 100644 grib/src/test/data/index/cosmo-eu.grib2.gbx9 create mode 100644 grib/src/test/data/index/radar_national.grib1.gbx9 diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocation.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocation.java new file mode 100644 index 0000000000..4a674932c2 --- /dev/null +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocation.java @@ -0,0 +1,163 @@ +package ucar.nc2.grib; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import thredds.filesystem.MFileOS; +import thredds.inventory.CollectionUpdateType; +import thredds.inventory.MFile; +import ucar.nc2.util.DiskCache2; + +@RunWith(Parameterized.class) +public class TestGribIndexLocation { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String DATA_DIR = "../grib/src/test/data/"; + private static final String INDEX_DIR = "../grib/src/test/data/index/"; + + @Parameterized.Parameters(name = "{0}") + public static List getTestParameters() { + return Arrays.asList(new Object[][] { + + {"radar_national.grib1"}, + + {"cosmo-eu.grib2"}, + + }); + } + + private final String filename; + private final String indexFilename; + private final boolean isGrib1; + + public TestGribIndexLocation(String filename) { + this.filename = filename; + this.indexFilename = filename + GribIndex.GBX9_IDX; + this.isGrib1 = filename.endsWith(".grib1"); + } + + @Rule + public final TemporaryFolder tempCacheFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setCacheLocation() { + System.getProperties().setProperty("nj22.cache", tempCacheFolder.getRoot().getPath()); + } + + @After + public void unsetCacheLocation() { + System.clearProperty("nj22.cache"); + } + + @Test + public void shouldCreateIndexInLocationOfDataFile() throws IOException { + useCache(false); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertTempFolderHasSize(2); + assertTempFolderHas(copiedFile.getName() + GribIndex.GBX9_IDX); + } + + @Test + public void shouldUseIndexInLocationOfDataFile() throws IOException { + useCache(false); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + final File copiedIndexFile = copyToTempFolder(INDEX_DIR + indexFilename); + assertThat(copiedIndexFile.setLastModified(0)).isTrue(); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.nocheck, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertTempFolderHasSize(2); + assertTempFolderHas(copiedFile.getName() + GribIndex.GBX9_IDX); + assertThat(copiedIndexFile.lastModified()).isEqualTo(0); + } + + @Test + public void shouldCreateIndexInDefaultCache() throws IOException { + final DiskCache2 diskCache = useCache(true); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertTempFolderHasSize(1); + assertThat(diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + } + + @Test + public void shouldReuseCachedIndex() throws IOException { + final DiskCache2 diskCache = useCache(true); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + final long cacheLastModified = diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX).lastModified(); + + final GribIndex rereadIndex = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.never, logger); + assertThat(rereadIndex).isNotNull(); + final File rereadCachedIndex = diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX); + assertThat(rereadCachedIndex.lastModified()).isEqualTo(cacheLastModified); + } + + // Helper functions + static DiskCache2 useCache(boolean useCache) { + final DiskCache2 diskCache = GribIndexCache.getDiskCache2(); + diskCache.setNeverUseCache(!useCache); + diskCache.setAlwaysUseCache(useCache); + return diskCache; + } + + private File copyToTempFolder(String filename) throws IOException { + final File file = new File(filename); + final File copiedFile = new File(tempFolder.getRoot(), file.getName()); + Files.copy(file.toPath(), copiedFile.toPath()); + assertTempFolderHas(copiedFile.getName()); + return copiedFile; + } + + private void assertTempFolderHas(String filename) { + final File[] filesInFolder = tempFolder.getRoot().listFiles(); + assertThat(filesInFolder).isNotNull(); + assertThat(Arrays.stream(filesInFolder).anyMatch(file -> file.getName().equals(filename))).isTrue(); + } + + private void assertTempFolderHasSize(int size) { + final File[] filesInFolder = tempFolder.getRoot().listFiles(); + assertThat(filesInFolder).isNotNull(); + assertThat(filesInFolder.length).isEqualTo(size); + } +} diff --git a/grib/src/test/data/index/cosmo-eu.grib2.gbx9 b/grib/src/test/data/index/cosmo-eu.grib2.gbx9 new file mode 100644 index 0000000000000000000000000000000000000000..f659a9d749f5d6e454722cab17638aa2f3366382 GIT binary patch literal 446 zcmZ=S%1koy%u7kFVB5&drJ|>&pAHn!FD^>fFG(#f(N9S%Nz_lyFV4-^O)b>}$r%ZS z3575)FoZAz2?j|ZW@P)13V>p)P|N_5X*ASj5J_rmuwVo-82vXkFwB?+#tuzMQ4C-X zP`T6wCN2hnZOjr3N)jA?0{(F(@iWT;6|WVwbC}W7m3qmEF{>C3efq&Fr?j->^GzDbDWJ zfhM~*zyH|%Pyb@i@%gm9kU^-ubmTpImBb)>UGoL@re`_^B`dOuUnI(n>=BCC5W=2NFCYGi~2Il6Lre^v@i7AOi@p*|QnfZB% zIeO_Ji-dH9v=|t)82&ReD=>axh(6PJ Date: Thu, 13 Jul 2023 11:54:44 -0600 Subject: [PATCH 2/3] Add tests for the location of the created grib index for s3 files --- .../nc2/grib/TestGribIndexLocationS3.java | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java new file mode 100644 index 0000000000..9eef8a1fcb --- /dev/null +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java @@ -0,0 +1,110 @@ +package ucar.nc2.grib; + +import static com.google.common.truth.Truth.assertThat; +import static ucar.nc2.grib.TestGribIndexLocation.useCache; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import thredds.inventory.CollectionUpdateType; +import thredds.inventory.MFile; +import thredds.inventory.s3.MFileS3; +import ucar.nc2.util.DiskCache2; + +@RunWith(Parameterized.class) +public class TestGribIndexLocationS3 { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String BUCKET = "cdms3:thredds-test-data"; + private static final String S3_DIR_WITHOUT_INDEX = BUCKET + "?" + "test-grib-without-index/"; + private static final String FRAGMENT = "#delimiter=/"; + + @Parameterized.Parameters(name = "{0}") + public static List getTestParameters() { + return Arrays.asList(new Object[][] { + + {"radar_national.grib1", ""}, + + {"radar_national.grib1", FRAGMENT}, + + {"cosmo-eu.grib2", ""}, + + {"cosmo-eu.grib2", FRAGMENT}, + + }); + } + + private final String filename; + private final String indexFilename; + private final boolean isGrib1; + + public TestGribIndexLocationS3(String filename, String fragment) { + this.filename = filename + fragment; + this.indexFilename = filename + GribIndex.GBX9_IDX; + this.isGrib1 = filename.endsWith(".grib1"); + } + + @Rule + public final TemporaryFolder tempCacheFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setCacheLocation() { + System.getProperties().setProperty("nj22.cache", tempCacheFolder.getRoot().getPath()); + } + + @After + public void unsetCacheLocation() { + System.clearProperty("nj22.cache"); + } + + @Test + public void shouldCreateIndexInDefaultCache() throws IOException { + final DiskCache2 diskCache = useCache(true); + + // Check that index file does not exist + final MFile s3FileIndex = new MFileS3(S3_DIR_WITHOUT_INDEX + indexFilename); + assertThat(s3FileIndex.exists()).isFalse(); + + final MFile s3File = new MFileS3(S3_DIR_WITHOUT_INDEX + filename); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + } + + @Test + public void shouldReuseCachedIndex() throws IOException { + final DiskCache2 diskCache = useCache(true); + + final MFile s3File = new MFileS3(S3_DIR_WITHOUT_INDEX + filename); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + final long cacheLastModified = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).lastModified(); + + final GribIndex rereadIndex = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.never, logger); + assertThat(rereadIndex).isNotNull(); + final File rereadCachedIndex = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX); + assertThat(rereadCachedIndex.lastModified()).isEqualTo(cacheLastModified); + } +} From 3db313412a8e2ae7cc2d9afd070e0b2fb4e6adb5 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 13 Jul 2023 12:00:18 -0600 Subject: [PATCH 3/3] Use getRaf which also works for s3 files --- grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java | 3 ++- grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java index a7d7747aa8..37adc7bcbf 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java @@ -8,6 +8,7 @@ import com.google.protobuf.ByteString; import java.nio.charset.StandardCharsets; import thredds.inventory.CollectionUpdateType; +import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.stream.NcStream; @@ -185,7 +186,7 @@ public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOExc rootBuilder.setFilename(filename); if (dataRaf == null) { // open if dataRaf not already open - raf = RandomAccessFile.acquire(filename); + raf = NetcdfFiles.getRaf(filename, -1); dataRaf = raf; } diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java index 7d39ee8ce2..a7e006be3c 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java @@ -8,6 +8,7 @@ import com.google.protobuf.ByteString; import java.nio.charset.StandardCharsets; import thredds.inventory.CollectionUpdateType; +import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.stream.NcStream; @@ -221,7 +222,7 @@ public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOExc rootBuilder.setFilename(filename); if (dataRaf == null) { - raf = RandomAccessFile.acquire(filename); + raf = NetcdfFiles.getRaf(filename, -1); dataRaf = raf; }