-
Notifications
You must be signed in to change notification settings - Fork 0
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Geotiff improvements 12 06 2020 #12
Open
oanahogoiu
wants to merge
7
commits into
master
Choose a base branch
from
geotiff-improvements-12-06-2020
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
7 commits
Select commit
Hold shift + click to select a range
601cf19
Create less objects when reading the product and adding the MultiLeve…
jeancoravu e44ff02
Initialize the logger when running tests from classes: RunProductRead…
jeancoravu 3930a8a
Create less objects when reading the JP2, S2 products and adding the …
jeancoravu 000c6f4
Update the log messages in the RunProductReaderTest class.
jeancoravu 885ee33
Update the log messages in the RunProductReaderTest, RunGPTProductRea…
jeancoravu b3485db
fixed typo
marpet 8095f24
some renamings
marpet File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
305 changes: 305 additions & 0 deletions
305
src/test/java/org/esa/snap/dataio/RunProductReaderTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,305 @@ | ||
| package org.esa.snap.dataio; | ||
|
|
||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import org.esa.s2tbx.dataio.gdal.GDALLibraryInstaller; | ||
| import org.esa.snap.core.dataio.ProductReader; | ||
| import org.esa.snap.core.dataio.ProductReaderPlugIn; | ||
| import org.esa.snap.core.datamodel.Product; | ||
| import org.esa.snap.core.util.SystemUtils; | ||
| import org.esa.snap.dataio.netcdf.NetCdfActivator; | ||
| import org.esa.snap.jp2.reader.OpenJPEGLibraryInstaller; | ||
| import org.esa.snap.runtime.LogUtils4Tests; | ||
| import org.junit.BeforeClass; | ||
| import org.junit.Test; | ||
| import org.junit.rules.ErrorCollector; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.net.URL; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.logging.Level; | ||
| import java.util.logging.Logger; | ||
|
|
||
| import static org.junit.Assert.fail; | ||
|
|
||
| public class RunProductReaderTest { | ||
|
|
||
| private static final Logger logger = Logger.getLogger(RunProductReaderTest.class.getName()); | ||
|
|
||
| private static final String PROPERTY_NAME_FAIL_ON_MISSING_DATA = "snap.reader.tests.failOnMissingData"; | ||
| private static final String PROPERTY_NAME_DATA_DIR = "snap.reader.tests.data.dir"; | ||
| private static final String PROPERTY_NAME_CASS_NAME = "snap.reader.tests.class.name"; | ||
| private static final String PROPERTY_NAME_PRODUCT_IDS = "snap.reader.tests.product.ids"; | ||
|
|
||
| private static List<TestDefinition> testDefinitionList; | ||
| private static File dataRootDir; | ||
| private static boolean isFailOnMissingData; | ||
|
|
||
| private ErrorCollector errorCollector = new ErrorCollector(); | ||
|
|
||
| public RunProductReaderTest() { | ||
| } | ||
|
|
||
| @BeforeClass | ||
| public static void initialize() throws Exception { | ||
| LogUtils4Tests.initLogger(); | ||
|
|
||
| String dataDirPropertyValue = System.getProperty(PROPERTY_NAME_DATA_DIR); | ||
| if (dataDirPropertyValue == null) { | ||
| fail("The test data property '"+PROPERTY_NAME_DATA_DIR+"' representing the folder which contains the source products is not set."); | ||
| } | ||
|
|
||
| String readerPlugInClassNameToTest = System.getProperty(PROPERTY_NAME_CASS_NAME); | ||
| String productIdsToTest = System.getProperty(PROPERTY_NAME_PRODUCT_IDS); | ||
| String failOnMissingData = System.getProperty(PROPERTY_NAME_FAIL_ON_MISSING_DATA); | ||
|
|
||
| if (logger.isLoggable(Level.FINE)) { | ||
| StringBuilder message = new StringBuilder(); | ||
| message.append("Input test parameters: data folder path: ") | ||
| .append(dataDirPropertyValue) | ||
| .append(", reader plugin class: ") | ||
| .append(readerPlugInClassNameToTest) | ||
| .append(", product ids: ") | ||
| .append(productIdsToTest) | ||
| .append(", fail on missing data: ") | ||
| .append(failOnMissingData) | ||
| .append("."); | ||
| logger.log(Level.FINE, message.toString()); | ||
| } | ||
|
|
||
| dataRootDir = new File(dataDirPropertyValue); | ||
| if (!dataRootDir.exists()) { | ||
| fail("The path '" + dataDirPropertyValue+"' representing the folder which contains the source products does not exist."); | ||
| } | ||
| if (!dataRootDir.isDirectory()) { | ||
| fail("The path '" + dataDirPropertyValue+"' exists and it does not denote a folder."); | ||
| } | ||
|
|
||
| isFailOnMissingData = true; | ||
| if (failOnMissingData != null) { | ||
| isFailOnMissingData = Boolean.parseBoolean(failOnMissingData); | ||
| } | ||
|
|
||
| SystemUtils.init3rdPartyLibs(RunProductReaderTest.class); | ||
|
|
||
| if (!isFailOnMissingData) { | ||
| logger.warning("Tests will not fail if test data is missing!"); | ||
| } | ||
|
|
||
| testDefinitionList = loadProductReaderTestDefinitions(readerPlugInClassNameToTest, productIdsToTest); | ||
|
|
||
| OpenJPEGLibraryInstaller.install(); | ||
| GDALLibraryInstaller.install(); | ||
| NetCdfActivator.activate(); | ||
| } | ||
|
|
||
| /** | ||
| * The required parameters to run the method: | ||
| * -Dsnap.reader.tests.data.dir=\\CV-DEV-SRV01\Satellite_Imagery\TestingJUnitFiles | ||
| * | ||
| * Optional parameters: | ||
| * -Dsnap.reader.tests.class.name=org.esa.s2tbx.dataio.muscate.MuscateProductReaderPlugin | ||
| * -Dsnap.reader.tests.product.ids=MUSCATE-Zip-31TFK,MUSCATE-v18,MUSCATE-v20 | ||
| * -Dsnap.reader.tests.failOnMissingData=false | ||
| * | ||
| * @throws Exception | ||
| */ | ||
| @Test | ||
| public void testReadIntendedProductContent() throws IOException { | ||
| logger.info("Test product content for " + testDefinitionList.size() + " reader plugins."); | ||
|
|
||
| for (int k=0; k<testDefinitionList.size(); k++) { | ||
| TestDefinition testDefinition = testDefinitionList.get(k); | ||
| if (k > 0) { | ||
| logger.info(""); // write an empty line | ||
| } | ||
| final List<String> intendedProductIds = testDefinition.getDecodableProductIds(); | ||
| String readerPluginClassName = testDefinition.getProductReaderPlugin().getClass().getSimpleName(); | ||
|
|
||
| logger.info("Test products content: reader plugin class: " + readerPluginClassName+ ", product count: " + intendedProductIds.size()+"."); | ||
|
|
||
| for (String productId : intendedProductIds) { | ||
| TestProduct foundTestProduct = null; | ||
| for (TestProduct testProduct : testDefinition.getAllProducts()) { | ||
| if (testProduct.getId().equalsIgnoreCase(productId)) { | ||
| foundTestProduct = testProduct; | ||
| break; | ||
| } | ||
| } | ||
| if (foundTestProduct == null) { | ||
| String message = "The test product with id '"+ productId+"' does not exist."; | ||
| if (isFailOnMissingData) { | ||
| fail(message); | ||
| } else { | ||
| logger.info(message); | ||
| } | ||
| } else if (foundTestProduct.exists()) { | ||
| logger.info("Start testing the product content: reader plugin class: " + readerPluginClassName+ ", product id: " + foundTestProduct.getId()+", product relative path: " + foundTestProduct.getRelativePath()+"."); | ||
|
|
||
| long startTime = System.currentTimeMillis(); | ||
|
|
||
| final File testProductFile = new File(dataRootDir, foundTestProduct.getRelativePath()); | ||
|
|
||
| long startTimeReadProduct = System.currentTimeMillis(); | ||
|
|
||
| final ProductReader productReader = testDefinition.getProductReaderPlugin().createReaderInstance(); | ||
| final Product product = productReader.readProductNodes(testProductFile, null); | ||
| if (product == null) { | ||
| String message = "The product can not be read from file '" + testProductFile.getAbsolutePath()+"' does not exist."; | ||
| if (isFailOnMissingData) { | ||
| fail(message); | ||
| } else { | ||
| logger.log(Level.SEVERE, message); | ||
| } | ||
| } else { | ||
| try { | ||
| if (logger.isLoggable(Level.FINE)) { | ||
| double elapsedSecondsReadProduct = (System.currentTimeMillis() - startTimeReadProduct) / 1000.0d; | ||
| StringBuilder message = new StringBuilder(); | ||
| message.append("Finish reading the product id: ") | ||
| .append(foundTestProduct.getId()) | ||
| .append(", relative path: ") | ||
| .append(foundTestProduct.getRelativePath()) | ||
| .append(", reader class: ") | ||
| .append(productReader.getClass().getName()) | ||
| .append(", size: ") | ||
| .append(product.getSceneRasterWidth()) | ||
| .append("x") | ||
| .append(product.getSceneRasterHeight()) | ||
| .append(", elapsed time: ") | ||
| .append(elapsedSecondsReadProduct) | ||
| .append(" seconds."); | ||
| logger.log(Level.FINE, message.toString()); | ||
| } | ||
|
|
||
| assertExpectedContent(testDefinition, foundTestProduct, product); | ||
| } catch (Throwable t) { | ||
| errorCollector.addError(new Throwable("[" + productId + "] " + t.getMessage(), t)); | ||
| } finally { | ||
| product.dispose(); | ||
| } | ||
|
|
||
| if (logger.isLoggable(Level.FINE)) { | ||
| double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; | ||
| StringBuilder message = new StringBuilder(); | ||
| message.append("Finish testing the product content id: ") | ||
| .append(foundTestProduct.getId()) | ||
| .append(", relative path: ") | ||
| .append(foundTestProduct.getRelativePath()) | ||
| .append(", elapsed time: ") | ||
| .append(elapsedSeconds) | ||
| .append(" seconds."); | ||
| logger.log(Level.FINE, message.toString()); | ||
| } | ||
| } | ||
| } else { | ||
| logger.info("The test product with id '"+ foundTestProduct.getId()+"' is missing from the disc."); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static void assertExpectedContent(TestDefinition testDefinition, TestProduct testProduct, Product product) { | ||
| final ExpectedContent expectedContent = testDefinition.getExpectedContent(testProduct.getId()); | ||
| if (expectedContent == null) { | ||
| String message = "No expected content for product id '" + testProduct.getId()+"'."; | ||
| if (isFailOnMissingData) { | ||
| fail(message); | ||
| } else { | ||
| logger.log(Level.SEVERE, message); | ||
| } | ||
| } else { | ||
| long startTime = System.currentTimeMillis(); | ||
|
|
||
| final ContentAssert contentAssert = new ContentAssert(expectedContent, testProduct.getId(), product); | ||
| contentAssert.assertProductContent(); | ||
|
|
||
| if (logger.isLoggable(Level.FINE)) { | ||
| double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; | ||
| StringBuilder message = new StringBuilder(); | ||
| message.append("Finish testing expected content: product id: ") | ||
| .append(testProduct.getId()) | ||
| .append(", relative path: ") | ||
| .append(testProduct.getRelativePath()) | ||
| .append(", elapsed time: ") | ||
| .append(elapsedSeconds) | ||
| .append(" seconds."); | ||
| logger.log(Level.FINE, message.toString()); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static List<TestDefinition> loadProductReaderTestDefinitions(String readerPlugInClassNameToTest, String productIdsToTest) throws IOException { | ||
| final ObjectMapper mapper = new ObjectMapper(); | ||
| final Iterable<ProductReaderPlugIn> readerPlugIns = SystemUtils.loadServices(ProductReaderPlugIn.class); | ||
|
|
||
| String[] productIds = null; | ||
| if (productIdsToTest != null) { | ||
| productIds = productIdsToTest.split(","); | ||
| if (productIds.length == 0) { | ||
| throw new IllegalArgumentException("The product ids array is empty."); | ||
| } else { | ||
| for (int k=0; k<productIds.length; k++) { | ||
| productIds[k] = productIds[k].trim(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| List<TestDefinition> testDefinitionList = new ArrayList<>(); | ||
| for (ProductReaderPlugIn readerPlugIn : readerPlugIns) { | ||
| final Class<? extends ProductReaderPlugIn> readerPlugInClass = readerPlugIn.getClass(); | ||
| if (readerPlugInClassNameToTest != null && !readerPlugInClass.getName().startsWith(readerPlugInClassNameToTest)) { | ||
| continue; | ||
| } | ||
|
|
||
| final String path = readerPlugInClass.getName().replace(".", "/"); | ||
| final String dataResourceName = "/" + path + "-data.json"; | ||
| final URL dataResource = readerPlugInClass.getResource(dataResourceName); | ||
| if (dataResource == null) { | ||
| logger.warning("The reader plugin class '"+readerPlugInClass.getSimpleName() + "' does not define test data."); | ||
| continue; | ||
| } | ||
|
|
||
| final ProductList productList = mapper.readValue(dataResource, ProductList.class); | ||
|
|
||
| final TestDefinition testDefinition = new TestDefinition(); | ||
| testDefinition.setProductReaderPlugin(readerPlugIn); | ||
| for (TestProduct testProduct : productList.getAll()) { | ||
| boolean canTestProduct; | ||
| if (productIds == null) { | ||
| canTestProduct = true; | ||
| } else { | ||
| canTestProduct = false; | ||
| for (int k=0; k<productIds.length && !canTestProduct; k++) { | ||
| if (productIds[k].equalsIgnoreCase(testProduct.getId())) { | ||
| canTestProduct = true; | ||
| } | ||
| } | ||
| } | ||
| if (canTestProduct) { | ||
| final File productFile = new File(dataRootDir, testProduct.getRelativePath()); | ||
| if (!productFile.exists()) { | ||
| testProduct.exists(false); | ||
| String message = "The product '" + productFile.getAbsolutePath()+"' of the reader plugin '"+ readerPlugInClass.getSimpleName()+"' does not exist."; | ||
| if (isFailOnMissingData) { | ||
| fail(message); | ||
| } else { | ||
| logger.info(message); | ||
| } | ||
| } | ||
| testDefinition.getAllProducts().add(testProduct); | ||
| final String fileResourceName = testProduct.getId() + ".json"; | ||
| final URL fileResource = readerPlugInClass.getResource(fileResourceName); | ||
| if (fileResource == null) { | ||
| fail("The resource file '" + fileResourceName+"' of the reader plugin '"+ readerPlugInClass.getSimpleName()+"' does not exist."); | ||
| } | ||
| final ExpectedDataset expectedDataset = mapper.readValue(fileResource, ExpectedDataset.class); | ||
| testDefinition.addExpectedDataset(expectedDataset); | ||
| } | ||
| } | ||
| testDefinitionList.add(testDefinition); | ||
| } | ||
| return testDefinitionList; | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why is this class necessary? Is ProductReaderAcceptanceTest not sufficient?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The RunProductReaderTest is used to test certain product ids from the reader plugin JSON file. Also, the logger can be configured to enabled/disabled the messages for some classes to avoid unusefull messages in the console.
An example to run only the Muscate test for product id 'MUSCATE-v20':
-Dsnap.reader.tests.data.dir=\CV-DEV-SRV01\Satellite_Imagery\TestingJUnitFiles
-Dsnap.reader.tests.class.name=org.esa.s2tbx.dataio.spot.SpotViewProductReaderPlugin
-Dsnap.reader.tests.product.ids=MUSCATE-v20
-Dorg.esa.snap.level=FINE
-Dorg.esa.snap.core.metadata.level=SEVERE
-Dorg.esa.snap.jp2.reader.internal.level=SEVERE
-Dsnap.reader.tests.failOnMissingData=false
-Drapid.eye.force.read.product.with.nitf.api=true
-Dorg.esa.s2tbx.dataio.rapideye.level=FINE
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, to be able to specify the product IDs is a nice feature, but this would have been better adde to the ProductReaderAcceptanceTest class. and then documented on the wiki page.
Same for the logging options.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I decided to not modify the ProductReaderAcceptanceTest class and create a new class with implementation I need to run the tests for reading the products from the JSON file.
Of course, the implementation can be copied/moved in the ProductReaderAcceptanceTest class if it is needed there.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, it would be good if you can move into the ProductReaderAcceptanceTest.
I just want to avoid that we have several runners. That Luis creates next week another onot and the week after BC creates also one. And all implementations need to be maintained.
So better update and improve the existing one as long as it applicable. Of course very special needs can arise which make it necessary to create a new implementation. But then the name of the implementation should already expression the special case.