diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocation.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocation.java new file mode 100644 index 0000000000..4a674932c2 --- /dev/null +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocation.java @@ -0,0 +1,163 @@ +package ucar.nc2.grib; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import thredds.filesystem.MFileOS; +import thredds.inventory.CollectionUpdateType; +import thredds.inventory.MFile; +import ucar.nc2.util.DiskCache2; + +@RunWith(Parameterized.class) +public class TestGribIndexLocation { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String DATA_DIR = "../grib/src/test/data/"; + private static final String INDEX_DIR = "../grib/src/test/data/index/"; + + @Parameterized.Parameters(name = "{0}") + public static List getTestParameters() { + return Arrays.asList(new Object[][] { + + {"radar_national.grib1"}, + + {"cosmo-eu.grib2"}, + + }); + } + + private final String filename; + private final String indexFilename; + private final boolean isGrib1; + + public TestGribIndexLocation(String filename) { + this.filename = filename; + this.indexFilename = filename + GribIndex.GBX9_IDX; + this.isGrib1 = filename.endsWith(".grib1"); + } + + @Rule + public final TemporaryFolder tempCacheFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setCacheLocation() { + System.getProperties().setProperty("nj22.cache", tempCacheFolder.getRoot().getPath()); + } + + @After + public void unsetCacheLocation() { + System.clearProperty("nj22.cache"); + } + + @Test + public void shouldCreateIndexInLocationOfDataFile() throws IOException { + useCache(false); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertTempFolderHasSize(2); + assertTempFolderHas(copiedFile.getName() + GribIndex.GBX9_IDX); + } + + @Test + public void shouldUseIndexInLocationOfDataFile() throws IOException { + useCache(false); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + final File copiedIndexFile = copyToTempFolder(INDEX_DIR + indexFilename); + assertThat(copiedIndexFile.setLastModified(0)).isTrue(); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.nocheck, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertTempFolderHasSize(2); + assertTempFolderHas(copiedFile.getName() + GribIndex.GBX9_IDX); + assertThat(copiedIndexFile.lastModified()).isEqualTo(0); + } + + @Test + public void shouldCreateIndexInDefaultCache() throws IOException { + final DiskCache2 diskCache = useCache(true); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertTempFolderHasSize(1); + assertThat(diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + } + + @Test + public void shouldReuseCachedIndex() throws IOException { + final DiskCache2 diskCache = useCache(true); + + final MFile copiedFile = new MFileOS(copyToTempFolder(DATA_DIR + filename).getPath()); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + final long cacheLastModified = diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX).lastModified(); + + final GribIndex rereadIndex = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, copiedFile, CollectionUpdateType.never, logger); + assertThat(rereadIndex).isNotNull(); + final File rereadCachedIndex = diskCache.getCacheFile(copiedFile.getPath() + GribIndex.GBX9_IDX); + assertThat(rereadCachedIndex.lastModified()).isEqualTo(cacheLastModified); + } + + // Helper functions + static DiskCache2 useCache(boolean useCache) { + final DiskCache2 diskCache = GribIndexCache.getDiskCache2(); + diskCache.setNeverUseCache(!useCache); + diskCache.setAlwaysUseCache(useCache); + return diskCache; + } + + private File copyToTempFolder(String filename) throws IOException { + final File file = new File(filename); + final File copiedFile = new File(tempFolder.getRoot(), file.getName()); + Files.copy(file.toPath(), copiedFile.toPath()); + assertTempFolderHas(copiedFile.getName()); + return copiedFile; + } + + private void assertTempFolderHas(String filename) { + final File[] filesInFolder = tempFolder.getRoot().listFiles(); + assertThat(filesInFolder).isNotNull(); + assertThat(Arrays.stream(filesInFolder).anyMatch(file -> file.getName().equals(filename))).isTrue(); + } + + private void assertTempFolderHasSize(int size) { + final File[] filesInFolder = tempFolder.getRoot().listFiles(); + assertThat(filesInFolder).isNotNull(); + assertThat(filesInFolder.length).isEqualTo(size); + } +} diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java new file mode 100644 index 0000000000..9eef8a1fcb --- /dev/null +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java @@ -0,0 +1,110 @@ +package ucar.nc2.grib; + +import static com.google.common.truth.Truth.assertThat; +import static ucar.nc2.grib.TestGribIndexLocation.useCache; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import thredds.inventory.CollectionUpdateType; +import thredds.inventory.MFile; +import thredds.inventory.s3.MFileS3; +import ucar.nc2.util.DiskCache2; + +@RunWith(Parameterized.class) +public class TestGribIndexLocationS3 { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String BUCKET = "cdms3:thredds-test-data"; + private static final String S3_DIR_WITHOUT_INDEX = BUCKET + "?" + "test-grib-without-index/"; + private static final String FRAGMENT = "#delimiter=/"; + + @Parameterized.Parameters(name = "{0}") + public static List getTestParameters() { + return Arrays.asList(new Object[][] { + + {"radar_national.grib1", ""}, + + {"radar_national.grib1", FRAGMENT}, + + {"cosmo-eu.grib2", ""}, + + {"cosmo-eu.grib2", FRAGMENT}, + + }); + } + + private final String filename; + private final String indexFilename; + private final boolean isGrib1; + + public TestGribIndexLocationS3(String filename, String fragment) { + this.filename = filename + fragment; + this.indexFilename = filename + GribIndex.GBX9_IDX; + this.isGrib1 = filename.endsWith(".grib1"); + } + + @Rule + public final TemporaryFolder tempCacheFolder = new TemporaryFolder(); + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void setCacheLocation() { + System.getProperties().setProperty("nj22.cache", tempCacheFolder.getRoot().getPath()); + } + + @After + public void unsetCacheLocation() { + System.clearProperty("nj22.cache"); + } + + @Test + public void shouldCreateIndexInDefaultCache() throws IOException { + final DiskCache2 diskCache = useCache(true); + + // Check that index file does not exist + final MFile s3FileIndex = new MFileS3(S3_DIR_WITHOUT_INDEX + indexFilename); + assertThat(s3FileIndex.exists()).isFalse(); + + final MFile s3File = new MFileS3(S3_DIR_WITHOUT_INDEX + filename); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(index.getNRecords()).isNotEqualTo(0); + + assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + } + + @Test + public void shouldReuseCachedIndex() throws IOException { + final DiskCache2 diskCache = useCache(true); + + final MFile s3File = new MFileS3(S3_DIR_WITHOUT_INDEX + filename); + + final GribIndex index = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.always, logger); + assertThat(index).isNotNull(); + assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue(); + final long cacheLastModified = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).lastModified(); + + final GribIndex rereadIndex = + GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.never, logger); + assertThat(rereadIndex).isNotNull(); + final File rereadCachedIndex = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX); + assertThat(rereadCachedIndex.lastModified()).isEqualTo(cacheLastModified); + } +} diff --git a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java index a7d7747aa8..37adc7bcbf 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java @@ -8,6 +8,7 @@ import com.google.protobuf.ByteString; import java.nio.charset.StandardCharsets; import thredds.inventory.CollectionUpdateType; +import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.stream.NcStream; @@ -185,7 +186,7 @@ public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOExc rootBuilder.setFilename(filename); if (dataRaf == null) { // open if dataRaf not already open - raf = RandomAccessFile.acquire(filename); + raf = NetcdfFiles.getRaf(filename, -1); dataRaf = raf; } diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java index 7d39ee8ce2..a7e006be3c 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java @@ -8,6 +8,7 @@ import com.google.protobuf.ByteString; import java.nio.charset.StandardCharsets; import thredds.inventory.CollectionUpdateType; +import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.stream.NcStream; @@ -221,7 +222,7 @@ public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOExc rootBuilder.setFilename(filename); if (dataRaf == null) { - raf = RandomAccessFile.acquire(filename); + raf = NetcdfFiles.getRaf(filename, -1); dataRaf = raf; } diff --git a/grib/src/test/data/index/cosmo-eu.grib2.gbx9 b/grib/src/test/data/index/cosmo-eu.grib2.gbx9 new file mode 100644 index 0000000000..f659a9d749 Binary files /dev/null and b/grib/src/test/data/index/cosmo-eu.grib2.gbx9 differ diff --git a/grib/src/test/data/index/radar_national.grib1.gbx9 b/grib/src/test/data/index/radar_national.grib1.gbx9 new file mode 100644 index 0000000000..211994c520 Binary files /dev/null and b/grib/src/test/data/index/radar_national.grib1.gbx9 differ