From 601cf1960eefa58b9d7ece5bcef445b1df6c58f4 Mon Sep 17 00:00:00 2001 From: jeancoravu Date: Mon, 15 Jun 2020 12:31:13 +0300 Subject: [PATCH 1/7] Create less objects when reading the product and adding the MultiLevelSource object to each band of the product. --- pom.xml | 26 ++ .../esa/snap/dataio/RunProductReaderTest.java | 330 ++++++++++++++++++ .../esa/snap/gpt/RunGPTProductReaderTest.java | 79 ++++- 3 files changed, 421 insertions(+), 14 deletions(-) create mode 100644 src/test/java/org/esa/snap/dataio/RunProductReaderTest.java diff --git a/pom.xml b/pom.xml index d07ee66..88044f1 100644 --- a/pom.xml +++ b/pom.xml @@ -220,6 +220,19 @@ test test-jar + + org.esa.snap + snap-ndvi + ${project.version} + test + + + org.esa.snap + snap-ndvi + ${project.version} + test + test-jar + @@ -523,6 +536,19 @@ test test-jar + + org.esa.s2tbx + s2tbx-radiometric-indices + ${s2tbx.test.version} + test + + + org.esa.s2tbx + s2tbx-radiometric-indices + ${s2tbx.test.version} + test + test-jar + diff --git a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java new file mode 100644 index 0000000..5005d94 --- /dev/null +++ b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java @@ -0,0 +1,330 @@ +package org.esa.snap.dataio; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.esa.snap.core.dataio.ProductReader; +import org.esa.snap.core.dataio.ProductReaderPlugIn; +import org.esa.snap.core.datamodel.Product; +import org.esa.snap.core.util.StopWatch; +import org.esa.snap.core.util.SystemUtils; +import org.esa.snap.gpt.RunGPTProductReaderTest; +import org.esa.snap.lib.openjpeg.activator.OpenJPEGInstaller; +import org.esa.snap.runtime.EngineConfig; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ErrorCollector; + +import java.io.*; +import java.net.URL; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.logging.*; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assert.fail; + +public class RunProductReaderTest { + + private final static String LINE_SEPARATOR = System.getProperty("line.separator", "\r\n"); + + private static final String PROPERTYNAME_FAIL_ON_MISSING_DATA = "snap.reader.tests.failOnMissingData"; + private static final String PROPERTY_NAME_DATA_DIR = "snap.reader.tests.data.dir"; + + private static final String PROPERTY_NAME_LOG_FILE_PATH = "snap.reader.tests.log.file"; + private static final String PROPERTY_NAME_CASS_NAME = "snap.reader.tests.class.name"; + private static final boolean FAIL_ON_MISSING_DATA = Boolean.parseBoolean(System.getProperty(PROPERTYNAME_FAIL_ON_MISSING_DATA, "true")); + private static final String INDENT = "\t"; + + private static TestDefinitionList testDefinitionList; + private static File dataRootDir; + + private static Logger logger; + @Rule + public ErrorCollector errorCollector = new ErrorCollector(); + private static final ProductList testProductList = new ProductList(); + + public RunProductReaderTest() { + } + + @BeforeClass + public static void initialize() throws Exception { + initLogger(); + + logFailOnMissingDataMessage(); + + assertTestDataDirectory(); + testDefinitionList = loadProductReaderTestDefinitions(); + + createGlobalProductList(); + + OpenJPEGInstaller.install(); + } + + @Test + public void testReadIntendedProductContent() throws IOException { + logger.info("Test product content for " + testDefinitionList.size() + " reader plugins."); + + for (TestDefinition testDefinition : testDefinitionList) { + logger.info("Start testing reader plugin: " + testDefinition.getProductReaderPlugin().getClass().getSimpleName() + "."); + long startTime = System.currentTimeMillis(); + + final List intendedProductIds = testDefinition.getDecodableProductIds(); + for (String productId : intendedProductIds) { + final TestProduct testProduct = testProductList.getById(productId); + String reason = "Test file not defined for ID=" + productId; + errorCollector.checkThat(reason, testProduct, is(notNullValue())); + if (testProduct == null) { + continue; + } + if (testProduct.exists()) { + final File testProductFile = getTestProductFile(testProduct); + + final ProductReader productReader = testDefinition.getProductReaderPlugin().createReaderInstance(); + + final Product product = productReader.readProductNodes(testProductFile, null); + try { + //assertExpectedContent(testDefinition, productId, product); + } catch (Throwable t) { + errorCollector.addError(new Throwable("[" + productId + "] " + t.getMessage(), t)); + } finally { + if (product != null) { + product.dispose(); + } + } + } else { + logProductNotExistent(2, testProduct); + } + } + double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; + logger.info("Finish testing reader plugin: " + testDefinition.getProductReaderPlugin().getClass().getSimpleName() + ". The time elapsed is " + elapsedSeconds + " seconds."); + } + } + + private static void createGlobalProductList() { + for (TestDefinition testDefinition : testDefinitionList) { + final List allPluginProducts = testDefinition.getAllProducts(); + for (TestProduct testProduct : allPluginProducts) { + if (!testIfIdAlreadyRegistered(testProduct)) { + testProductList.add(testProduct); + } + } + } + } + + private static boolean testIfIdAlreadyRegistered(TestProduct testProduct) { + final String id = testProduct.getId(); + final TestProduct storedProduct = testProductList.getById(id); + if (storedProduct != null) { + if (storedProduct.isDifferent(testProduct)) { + fail("Test file with ID=" + id + " already defined with different settings"); + } + + return true; + } + + return false; + } + + private void logProductNotExistent(int indention, TestProduct testProduct) { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < indention; i++) { + sb.append(INDENT); + } + logger.info(sb.toString() + "Not existent - " + testProduct.getId()); + } + + private static void assertExpectedContent(TestDefinition testDefinition, String productId, Product product) { + final ExpectedContent expectedContent = testDefinition.getExpectedContent(productId); + if (expectedContent == null) { + logger.info("No expected content for product id '" + productId+"'."); + } else { + long startTime = System.currentTimeMillis(); + logger.info("Start testing expected content: product id: " + productId+"."); + + final ContentAssert contentAssert = new ContentAssert(expectedContent, productId, product); + contentAssert.assertProductContent(); + + double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; + logger.info("Finish testing expected content: product id: " + productId+"'. The time elapsed is " + elapsedSeconds + " seconds."); + } + } + + private File getTestProductFile(TestProduct testProduct) { + final String relativePath = testProduct.getRelativePath(); + final File testProductFile = new File(dataRootDir, relativePath); + + errorCollector.checkThat("testProductFile exist " + testProduct.getId(), testProductFile.exists(), is(true)); + return testProductFile; + } + + private static void logInfoWithStars(final String text) { + final String msg = " " + text + " "; + final char[] stars = new char[msg.length()]; + Arrays.fill(stars, '*'); + final String starString = new String(stars); + logger.info(""); + logger.info(starString); + logger.info(msg); + logger.info(starString); + logger.info(""); + } + + private static TestDefinitionList loadProductReaderTestDefinitions() throws IOException { + final ObjectMapper mapper = new ObjectMapper(); + final Iterable readerPlugIns = SystemUtils.loadServices(ProductReaderPlugIn.class); + final String className = System.getProperty(PROPERTY_NAME_CASS_NAME); + + TestDefinitionList testDefinitionList = new TestDefinitionList(); + for (ProductReaderPlugIn readerPlugIn : readerPlugIns) { + final Class readerPlugInClass = readerPlugIn.getClass(); + if (className != null && !readerPlugInClass.getName().startsWith(className)) { + continue; + } + + final String path = readerPlugInClass.getName().replace(".", "/"); + final String dataResourceName = "/" + path + "-data.json"; + final URL dataResource = readerPlugInClass.getResource(dataResourceName); + if (dataResource == null) { + logger.warning(readerPlugInClass.getSimpleName() + " does not define test data"); + continue; + } + + final TestDefinition testDefinition = new TestDefinition(); + testDefinition.setProductReaderPlugin(readerPlugIn); + + final ProductList productList = mapper.readValue(dataResource, ProductList.class); + testIfProductFilesExists(productList); + testDefinition.addTestProducts(productList.getAll()); + + for (TestProduct product : productList.getAll()) { + final String fileResourceName = product.getId() + ".json"; + final URL fileResource = readerPlugInClass.getResource(fileResourceName); + if (fileResource == null) { + fail(readerPlugInClass.getSimpleName() + " resource file '" + fileResourceName + "' is missing"); + } + final ExpectedDataset expectedDataset = mapper.readValue(fileResource, ExpectedDataset.class); + testDefinition.addExpectedDataset(expectedDataset); + } + testDefinitionList.add(testDefinition); + } + return testDefinitionList; + } + + private static void assertTestDataDirectory() { + final String dataDirProperty = System.getProperty(PROPERTY_NAME_DATA_DIR); + if (dataDirProperty == null) { + fail("Data directory path not set"); + } + dataRootDir = new File(dataDirProperty); + if (!dataRootDir.isDirectory()) { + fail("Data directory is not valid: " + dataDirProperty); + } + } + + private static void testIfProductFilesExists(ProductList productList) { + for (TestProduct testProduct : productList) { + final String relativePath = testProduct.getRelativePath(); + final File productFile = new File(dataRootDir, relativePath); + if (!productFile.exists()) { + testProduct.exists(false); + if (FAIL_ON_MISSING_DATA) { + fail("Test product does not exist: " + productFile.getAbsolutePath()); + } + } + } + } + + private static void logFailOnMissingDataMessage() { + if (!FAIL_ON_MISSING_DATA) { + logger.warning("Tests will not fail if test data is missing!"); + } + } + + private static void initLogger() throws Exception { + // Suppress ugly (and harmless) JAI error messages saying that a JAI is going to continue in pure Java mode. + System.setProperty("com.sun.media.jai.disableMediaLib", "true"); // disable native libraries for JAI + + EngineConfig engineConfig = EngineConfig.instance(); + engineConfig.logLevel(Level.INFO); + engineConfig.loggerName("org.esa"); + + StringBuilder properties = new StringBuilder(); + Enumeration propertyNames = System.getProperties().propertyNames(); + while (propertyNames.hasMoreElements()) { + String systemPropertyName = (String)propertyNames.nextElement(); + if (systemPropertyName.endsWith(".level")) { + String systemPropertyValue = System.getProperty(systemPropertyName); + try { + Level level = Level.parse(systemPropertyValue); + if (properties.length() > 0) { + properties.append(LINE_SEPARATOR); + } + properties.append(systemPropertyName) + .append("=") + .append(level.getName()); + } catch (IllegalArgumentException exception) { + // ignore exception + } + } + } + if (properties.length() > 0) { + properties.append(LINE_SEPARATOR) + .append(".level = INFO"); + + ByteArrayInputStream inputStream = new ByteArrayInputStream(properties.toString().getBytes()); + LogManager logManager = LogManager.getLogManager(); + logManager.readConfiguration(inputStream); + } + + Logger rootLogger = Logger.getLogger(""); + for (Handler handler : rootLogger.getHandlers()) { + rootLogger.removeHandler(handler); + } + + Logger orgEsaLogger = Logger.getLogger("org.esa"); + for (Handler handler : rootLogger.getHandlers()) { + orgEsaLogger.removeHandler(handler); + } + ConsoleHandler consoleHandler = new ConsoleHandler() { + @Override + public synchronized void setLevel(Level newLevel) throws SecurityException { + super.setLevel(Level.FINEST); + } + }; + consoleHandler.setFormatter(new CustomLogFormatter()); + orgEsaLogger.addHandler(consoleHandler); + + logger = Logger.getLogger(RunGPTProductReaderTest.class.getName()); + + String logFilePath = System.getProperty(PROPERTY_NAME_LOG_FILE_PATH); + if (logFilePath != null) { + File logFile = new File(logFilePath); + FileOutputStream fos = new FileOutputStream(logFile); + StreamHandler streamHandler = new StreamHandler(fos, new CustomLogFormatter()); + logger.addHandler(streamHandler); + } + } + + private static class CustomLogFormatter extends Formatter { + + @Override + public synchronized String format(LogRecord record) { + StringBuilder sb = new StringBuilder(); + String message = formatMessage(record); + sb.append(record.getLevel().getName()); + sb.append(": "); + sb.append(message); + sb.append(LINE_SEPARATOR); + if (record.getThrown() != null) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + record.getThrown().printStackTrace(pw); + pw.close(); + sb.append(sw.toString()); + } + return sb.toString(); + } + } +} diff --git a/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java b/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java index 862584b..da427ee 100644 --- a/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java +++ b/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java @@ -11,6 +11,8 @@ import org.esa.snap.dataio.ExpectedDataset; import org.esa.snap.dataio.ProductReaderAcceptanceTest; import org.esa.snap.lib.openjpeg.activator.OpenJPEGInstaller; +import org.esa.snap.runtime.Config; +import org.esa.snap.runtime.EngineConfig; import org.junit.BeforeClass; import org.junit.Test; @@ -29,6 +31,8 @@ */ public class RunGPTProductReaderTest { + private final static String LINE_SEPARATOR = System.getProperty("line.separator", "\r\n"); + private static final String PROPERTY_NAME_LOG_FILE_PATH = "gpt.tests.log.file"; private static final String PROPERTY_NAME_GPT_TEST_RESOURCES_FOLDER_PATH = "gpt.test.resources.dir"; private static final String PROPERTY_NAME_GPT_SOURCE_PRODUCTS_FOLDER_PATH = "gpt.source.products.dir"; @@ -42,6 +46,9 @@ public class RunGPTProductReaderTest { private static Logger logger; + public RunGPTProductReaderTest() { + } + @BeforeClass public static void initialize() throws Exception { initLogger(); @@ -216,6 +223,7 @@ private static void processFile(File testFile, File sourceProductsFolder, File o String startPathToRemove, boolean failIfMissingSourceData) throws Exception { + long startTime = System.currentTimeMillis(); logger.info("Test the file '" + testFile.getName() + "' from the folder '" + testFile.getParent()+"'."); ObjectMapper mapper = new ObjectMapper(); @@ -278,6 +286,9 @@ private static void processFile(File testFile, File sourceProductsFolder, File o runGPT(graphFile, gptGraphParameters); + double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; + logger.info("Finish processing the graph file '" + graphFile.getAbsolutePath() +"'. The time elapsed is " + elapsedSeconds + " seconds."); + File productFile = new File(outputFolder, outputNameValue + ".dim"); Product product = ProductIO.readProduct(productFile); if (product == null) { @@ -302,6 +313,9 @@ private static void processFile(File testFile, File sourceProductsFolder, File o assertExpectedContent(product, expectedDataset.getExpectedContent(), assertMessagePrefix.toString()); } } + + double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; + logger.info("Finish testing the file '" + testFile.getName() + "' from the folder '" + testFile.getParent()+"'. The time elapsed is " + elapsedSeconds + " seconds."); } private static void runGPT(File graphFile, Map gptGraphParameters) throws Exception { @@ -336,11 +350,58 @@ private static void initLogger() throws Exception { // Suppress ugly (and harmless) JAI error messages saying that a JAI is going to continue in pure Java mode. System.setProperty("com.sun.media.jai.disableMediaLib", "true"); // disable native libraries for JAI - logger = Logger.getLogger(ProductReaderAcceptanceTest.class.getSimpleName()); - removeRootLogHandler(); - ConsoleHandler consoleHandler = new ConsoleHandler(); + EngineConfig engineConfig = EngineConfig.instance(); + engineConfig.logLevel(Level.INFO); + engineConfig.loggerName("org.esa"); + + StringBuilder properties = new StringBuilder(); + Enumeration propertyNames = System.getProperties().propertyNames(); + while (propertyNames.hasMoreElements()) { + String systemPropertyName = (String)propertyNames.nextElement(); + if (systemPropertyName.endsWith(".level")) { + String systemPropertyValue = System.getProperty(systemPropertyName); + try { + Level level = Level.parse(systemPropertyValue); + if (properties.length() > 0) { + properties.append(LINE_SEPARATOR); + } + properties.append(systemPropertyName) + .append("=") + .append(level.getName()); + } catch (IllegalArgumentException exception) { + // ignore exception + } + } + } + if (properties.length() > 0) { + properties.append(LINE_SEPARATOR) + .append(".level = INFO"); + + ByteArrayInputStream inputStream = new ByteArrayInputStream(properties.toString().getBytes()); + LogManager logManager = LogManager.getLogManager(); + logManager.readConfiguration(inputStream); + } + + Logger rootLogger = Logger.getLogger(""); + for (Handler handler : rootLogger.getHandlers()) { + rootLogger.removeHandler(handler); + } + + Logger orgEsaLogger = Logger.getLogger("org.esa"); + for (Handler handler : rootLogger.getHandlers()) { + orgEsaLogger.removeHandler(handler); + } + ConsoleHandler consoleHandler = new ConsoleHandler() { + @Override + public synchronized void setLevel(Level newLevel) throws SecurityException { + super.setLevel(Level.FINEST); + } + }; consoleHandler.setFormatter(new CustomLogFormatter()); - logger.addHandler(consoleHandler); + orgEsaLogger.addHandler(consoleHandler); + + logger = Logger.getLogger(RunGPTProductReaderTest.class.getName()); + String logFilePath = System.getProperty(PROPERTY_NAME_LOG_FILE_PATH); if (logFilePath != null) { File logFile = new File(logFilePath); @@ -350,14 +411,6 @@ private static void initLogger() throws Exception { } } - private static void removeRootLogHandler() { - Logger rootLogger = LogManager.getLogManager().getLogger(""); - Handler[] handlers = rootLogger.getHandlers(); - for (Handler handler : handlers) { - rootLogger.removeHandler(handler); - } - } - private static void validateFolderOnDisk(File folderToValidate) throws FileNotFoundException, NotDirectoryException { if (folderToValidate.exists()) { if (!folderToValidate.isDirectory()) { @@ -380,8 +433,6 @@ private static void validateFileOnDisk(File fileToValidate) throws FileNotFoundE private static class CustomLogFormatter extends Formatter { - private final static String LINE_SEPARATOR = System.getProperty("line.separator", "\r\n"); - @Override public synchronized String format(LogRecord record) { StringBuilder sb = new StringBuilder(); From e44ff026ab72ebba0e4464611731deafe7f29075 Mon Sep 17 00:00:00 2001 From: jeancoravu Date: Mon, 15 Jun 2020 17:14:49 +0300 Subject: [PATCH 2/7] Initialize the logger when running tests from classes: RunProductReaderTest, RunGPTProductReaderTest --- pom.xml | 7 ++ .../esa/snap/dataio/RunProductReaderTest.java | 103 ++---------------- .../esa/snap/gpt/RunGPTProductReaderTest.java | 94 +--------------- 3 files changed, 17 insertions(+), 187 deletions(-) diff --git a/pom.xml b/pom.xml index 88044f1..7dd76ec 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,13 @@ 1.10.17 test + + org.esa.snap + snap-runtime + ${snap.test.version} + test-jar + test + diff --git a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java index 5005d94..1f02d38 100644 --- a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java +++ b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java @@ -4,22 +4,21 @@ import org.esa.snap.core.dataio.ProductReader; import org.esa.snap.core.dataio.ProductReaderPlugIn; import org.esa.snap.core.datamodel.Product; -import org.esa.snap.core.util.StopWatch; import org.esa.snap.core.util.SystemUtils; import org.esa.snap.gpt.RunGPTProductReaderTest; import org.esa.snap.lib.openjpeg.activator.OpenJPEGInstaller; -import org.esa.snap.runtime.EngineConfig; +import org.esa.snap.runtime.LogUtils; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ErrorCollector; -import java.io.*; +import java.io.File; +import java.io.IOException; import java.net.URL; import java.util.Arrays; -import java.util.Enumeration; import java.util.List; -import java.util.logging.*; +import java.util.logging.Logger; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; @@ -27,12 +26,9 @@ public class RunProductReaderTest { - private final static String LINE_SEPARATOR = System.getProperty("line.separator", "\r\n"); - private static final String PROPERTYNAME_FAIL_ON_MISSING_DATA = "snap.reader.tests.failOnMissingData"; private static final String PROPERTY_NAME_DATA_DIR = "snap.reader.tests.data.dir"; - private static final String PROPERTY_NAME_LOG_FILE_PATH = "snap.reader.tests.log.file"; private static final String PROPERTY_NAME_CASS_NAME = "snap.reader.tests.class.name"; private static final boolean FAIL_ON_MISSING_DATA = Boolean.parseBoolean(System.getProperty(PROPERTYNAME_FAIL_ON_MISSING_DATA, "true")); private static final String INDENT = "\t"; @@ -40,7 +36,8 @@ public class RunProductReaderTest { private static TestDefinitionList testDefinitionList; private static File dataRootDir; - private static Logger logger; + private static final Logger logger = Logger.getLogger(RunProductReaderTest.class.getName()); + @Rule public ErrorCollector errorCollector = new ErrorCollector(); private static final ProductList testProductList = new ProductList(); @@ -50,7 +47,7 @@ public RunProductReaderTest() { @BeforeClass public static void initialize() throws Exception { - initLogger(); + LogUtils.initLogger(); logFailOnMissingDataMessage(); @@ -241,90 +238,4 @@ private static void logFailOnMissingDataMessage() { logger.warning("Tests will not fail if test data is missing!"); } } - - private static void initLogger() throws Exception { - // Suppress ugly (and harmless) JAI error messages saying that a JAI is going to continue in pure Java mode. - System.setProperty("com.sun.media.jai.disableMediaLib", "true"); // disable native libraries for JAI - - EngineConfig engineConfig = EngineConfig.instance(); - engineConfig.logLevel(Level.INFO); - engineConfig.loggerName("org.esa"); - - StringBuilder properties = new StringBuilder(); - Enumeration propertyNames = System.getProperties().propertyNames(); - while (propertyNames.hasMoreElements()) { - String systemPropertyName = (String)propertyNames.nextElement(); - if (systemPropertyName.endsWith(".level")) { - String systemPropertyValue = System.getProperty(systemPropertyName); - try { - Level level = Level.parse(systemPropertyValue); - if (properties.length() > 0) { - properties.append(LINE_SEPARATOR); - } - properties.append(systemPropertyName) - .append("=") - .append(level.getName()); - } catch (IllegalArgumentException exception) { - // ignore exception - } - } - } - if (properties.length() > 0) { - properties.append(LINE_SEPARATOR) - .append(".level = INFO"); - - ByteArrayInputStream inputStream = new ByteArrayInputStream(properties.toString().getBytes()); - LogManager logManager = LogManager.getLogManager(); - logManager.readConfiguration(inputStream); - } - - Logger rootLogger = Logger.getLogger(""); - for (Handler handler : rootLogger.getHandlers()) { - rootLogger.removeHandler(handler); - } - - Logger orgEsaLogger = Logger.getLogger("org.esa"); - for (Handler handler : rootLogger.getHandlers()) { - orgEsaLogger.removeHandler(handler); - } - ConsoleHandler consoleHandler = new ConsoleHandler() { - @Override - public synchronized void setLevel(Level newLevel) throws SecurityException { - super.setLevel(Level.FINEST); - } - }; - consoleHandler.setFormatter(new CustomLogFormatter()); - orgEsaLogger.addHandler(consoleHandler); - - logger = Logger.getLogger(RunGPTProductReaderTest.class.getName()); - - String logFilePath = System.getProperty(PROPERTY_NAME_LOG_FILE_PATH); - if (logFilePath != null) { - File logFile = new File(logFilePath); - FileOutputStream fos = new FileOutputStream(logFile); - StreamHandler streamHandler = new StreamHandler(fos, new CustomLogFormatter()); - logger.addHandler(streamHandler); - } - } - - private static class CustomLogFormatter extends Formatter { - - @Override - public synchronized String format(LogRecord record) { - StringBuilder sb = new StringBuilder(); - String message = formatMessage(record); - sb.append(record.getLevel().getName()); - sb.append(": "); - sb.append(message); - sb.append(LINE_SEPARATOR); - if (record.getThrown() != null) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - record.getThrown().printStackTrace(pw); - pw.close(); - sb.append(sw.toString()); - } - return sb.toString(); - } - } } diff --git a/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java b/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java index da427ee..46282c7 100644 --- a/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java +++ b/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java @@ -9,16 +9,14 @@ import org.esa.snap.dataio.ContentAssert; import org.esa.snap.dataio.ExpectedContent; import org.esa.snap.dataio.ExpectedDataset; -import org.esa.snap.dataio.ProductReaderAcceptanceTest; import org.esa.snap.lib.openjpeg.activator.OpenJPEGInstaller; -import org.esa.snap.runtime.Config; import org.esa.snap.runtime.EngineConfig; +import org.esa.snap.runtime.LogUtils; import org.junit.BeforeClass; import org.junit.Test; import java.io.*; import java.nio.file.NotDirectoryException; -import java.text.SimpleDateFormat; import java.util.*; import java.util.logging.*; import java.util.logging.Formatter; @@ -44,14 +42,14 @@ public class RunGPTProductReaderTest { private static final String PROPERTY_NAME_FAIL_ON_EXCEPTION = "gpt.tests.failOnException"; private static final String PROPERTY_NAME_LOG_EXCEPTION_STACK_TRACE = "gpt.tests.logExceptionStackTrace"; - private static Logger logger; + private static final Logger logger = Logger.getLogger(RunGPTProductReaderTest.class.getName()); public RunGPTProductReaderTest() { } @BeforeClass public static void initialize() throws Exception { - initLogger(); + LogUtils.initLogger(); OpenJPEGInstaller.install(); } @@ -346,71 +344,6 @@ private static void assertExpectedContent(Product product, ExpectedContent expec contentAssert.assertProductContent(); } - private static void initLogger() throws Exception { - // Suppress ugly (and harmless) JAI error messages saying that a JAI is going to continue in pure Java mode. - System.setProperty("com.sun.media.jai.disableMediaLib", "true"); // disable native libraries for JAI - - EngineConfig engineConfig = EngineConfig.instance(); - engineConfig.logLevel(Level.INFO); - engineConfig.loggerName("org.esa"); - - StringBuilder properties = new StringBuilder(); - Enumeration propertyNames = System.getProperties().propertyNames(); - while (propertyNames.hasMoreElements()) { - String systemPropertyName = (String)propertyNames.nextElement(); - if (systemPropertyName.endsWith(".level")) { - String systemPropertyValue = System.getProperty(systemPropertyName); - try { - Level level = Level.parse(systemPropertyValue); - if (properties.length() > 0) { - properties.append(LINE_SEPARATOR); - } - properties.append(systemPropertyName) - .append("=") - .append(level.getName()); - } catch (IllegalArgumentException exception) { - // ignore exception - } - } - } - if (properties.length() > 0) { - properties.append(LINE_SEPARATOR) - .append(".level = INFO"); - - ByteArrayInputStream inputStream = new ByteArrayInputStream(properties.toString().getBytes()); - LogManager logManager = LogManager.getLogManager(); - logManager.readConfiguration(inputStream); - } - - Logger rootLogger = Logger.getLogger(""); - for (Handler handler : rootLogger.getHandlers()) { - rootLogger.removeHandler(handler); - } - - Logger orgEsaLogger = Logger.getLogger("org.esa"); - for (Handler handler : rootLogger.getHandlers()) { - orgEsaLogger.removeHandler(handler); - } - ConsoleHandler consoleHandler = new ConsoleHandler() { - @Override - public synchronized void setLevel(Level newLevel) throws SecurityException { - super.setLevel(Level.FINEST); - } - }; - consoleHandler.setFormatter(new CustomLogFormatter()); - orgEsaLogger.addHandler(consoleHandler); - - logger = Logger.getLogger(RunGPTProductReaderTest.class.getName()); - - String logFilePath = System.getProperty(PROPERTY_NAME_LOG_FILE_PATH); - if (logFilePath != null) { - File logFile = new File(logFilePath); - FileOutputStream fos = new FileOutputStream(logFile); - StreamHandler streamHandler = new StreamHandler(fos, new CustomLogFormatter()); - logger.addHandler(streamHandler); - } - } - private static void validateFolderOnDisk(File folderToValidate) throws FileNotFoundException, NotDirectoryException { if (folderToValidate.exists()) { if (!folderToValidate.isDirectory()) { @@ -431,27 +364,6 @@ private static void validateFileOnDisk(File fileToValidate) throws FileNotFoundE } } - private static class CustomLogFormatter extends Formatter { - - @Override - public synchronized String format(LogRecord record) { - StringBuilder sb = new StringBuilder(); - String message = formatMessage(record); - sb.append(record.getLevel().getName()); - sb.append(": "); - sb.append(message); - sb.append(LINE_SEPARATOR); - if (record.getThrown() != null) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - record.getThrown().printStackTrace(pw); - pw.close(); - sb.append(sw.toString()); - } - return sb.toString(); - } - } - private static class InputGraphData { private String id; From 3930a8a4ff57e450a94b68c31ac7e7cee3f9dd74 Mon Sep 17 00:00:00 2001 From: jeancoravu Date: Wed, 17 Jun 2020 11:49:07 +0300 Subject: [PATCH 3/7] Create less objects when reading the JP2, S2 products and adding the MultiLevelSource object to each band of the product. --- .../org/esa/snap/dataio/RunProductReaderTest.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java index 1f02d38..5cfad7b 100644 --- a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java +++ b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java @@ -82,7 +82,7 @@ public void testReadIntendedProductContent() throws IOException { final Product product = productReader.readProductNodes(testProductFile, null); try { - //assertExpectedContent(testDefinition, productId, product); + assertExpectedContent(testDefinition, productId, product); } catch (Throwable t) { errorCollector.addError(new Throwable("[" + productId + "] " + t.getMessage(), t)); } finally { @@ -156,18 +156,6 @@ private File getTestProductFile(TestProduct testProduct) { return testProductFile; } - private static void logInfoWithStars(final String text) { - final String msg = " " + text + " "; - final char[] stars = new char[msg.length()]; - Arrays.fill(stars, '*'); - final String starString = new String(stars); - logger.info(""); - logger.info(starString); - logger.info(msg); - logger.info(starString); - logger.info(""); - } - private static TestDefinitionList loadProductReaderTestDefinitions() throws IOException { final ObjectMapper mapper = new ObjectMapper(); final Iterable readerPlugIns = SystemUtils.loadServices(ProductReaderPlugIn.class); From 000c6f429acc6a2bf6d8f3a3124581f748b29a85 Mon Sep 17 00:00:00 2001 From: jeancoravu Date: Thu, 18 Jun 2020 17:20:23 +0300 Subject: [PATCH 4/7] Update the log messages in the RunProductReaderTest class. --- .../esa/snap/dataio/RunProductReaderTest.java | 260 ++++++++++-------- 1 file changed, 143 insertions(+), 117 deletions(-) diff --git a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java index 5cfad7b..9bafc3e 100644 --- a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java +++ b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java @@ -5,7 +5,6 @@ import org.esa.snap.core.dataio.ProductReaderPlugIn; import org.esa.snap.core.datamodel.Product; import org.esa.snap.core.util.SystemUtils; -import org.esa.snap.gpt.RunGPTProductReaderTest; import org.esa.snap.lib.openjpeg.activator.OpenJPEGInstaller; import org.esa.snap.runtime.LogUtils; import org.junit.BeforeClass; @@ -16,8 +15,8 @@ import java.io.File; import java.io.IOException; import java.net.URL; -import java.util.Arrays; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import static org.hamcrest.CoreMatchers.is; @@ -26,35 +25,52 @@ public class RunProductReaderTest { - private static final String PROPERTYNAME_FAIL_ON_MISSING_DATA = "snap.reader.tests.failOnMissingData"; - private static final String PROPERTY_NAME_DATA_DIR = "snap.reader.tests.data.dir"; + private static final Logger logger = Logger.getLogger(RunProductReaderTest.class.getName()); + private static final String PROPERTY_NAME_FAIL_ON_MISSING_DATA = "snap.reader.tests.failOnMissingData"; + private static final String PROPERTY_NAME_DATA_DIR = "snap.reader.tests.data.dir"; private static final String PROPERTY_NAME_CASS_NAME = "snap.reader.tests.class.name"; - private static final boolean FAIL_ON_MISSING_DATA = Boolean.parseBoolean(System.getProperty(PROPERTYNAME_FAIL_ON_MISSING_DATA, "true")); - private static final String INDENT = "\t"; + private static final String PROPERTY_NAME_PRODUCT_ID = "snap.reader.tests.product.id"; + + private static final boolean FAIL_ON_MISSING_DATA = Boolean.parseBoolean(System.getProperty(PROPERTY_NAME_FAIL_ON_MISSING_DATA, "true")); private static TestDefinitionList testDefinitionList; private static File dataRootDir; - private static final Logger logger = Logger.getLogger(RunProductReaderTest.class.getName()); - @Rule public ErrorCollector errorCollector = new ErrorCollector(); - private static final ProductList testProductList = new ProductList(); public RunProductReaderTest() { } @BeforeClass public static void initialize() throws Exception { + assertRequiredTestDataDirectory(); + LogUtils.initLogger(); - logFailOnMissingDataMessage(); + String readerPlugInClassNameToTest = System.getProperty(PROPERTY_NAME_CASS_NAME); + String productIdToTest = System.getProperty(PROPERTY_NAME_PRODUCT_ID); + + if (logger.isLoggable(Level.FINE)) { + StringBuilder message = new StringBuilder(); + message.append("Input test parameters: data folder path: ") + .append(dataRootDir.getAbsolutePath()) + .append(", reader plugin class: ") + .append(readerPlugInClassNameToTest) + .append(", product id :") + .append(productIdToTest) + .append("."); + logger.log(Level.FINE, message.toString()); + } + + SystemUtils.init3rdPartyLibs(RunProductReaderTest.class); - assertTestDataDirectory(); - testDefinitionList = loadProductReaderTestDefinitions(); + if (!FAIL_ON_MISSING_DATA) { + logger.warning("Tests will not fail if test data is missing!"); + } - createGlobalProductList(); + testDefinitionList = loadProductReaderTestDefinitions(readerPlugInClassNameToTest, productIdToTest); OpenJPEGInstaller.install(); } @@ -64,25 +80,58 @@ public void testReadIntendedProductContent() throws IOException { logger.info("Test product content for " + testDefinitionList.size() + " reader plugins."); for (TestDefinition testDefinition : testDefinitionList) { - logger.info("Start testing reader plugin: " + testDefinition.getProductReaderPlugin().getClass().getSimpleName() + "."); - long startTime = System.currentTimeMillis(); - final List intendedProductIds = testDefinition.getDecodableProductIds(); + String readerPluginClassName = testDefinition.getProductReaderPlugin().getClass().getSimpleName(); + + logger.info("Test products content: reader plugin class: " + readerPluginClassName+ ", product count: " + intendedProductIds.size()+"."); + for (String productId : intendedProductIds) { - final TestProduct testProduct = testProductList.getById(productId); - String reason = "Test file not defined for ID=" + productId; - errorCollector.checkThat(reason, testProduct, is(notNullValue())); - if (testProduct == null) { - continue; + TestProduct foundTestProduct = null; + for (TestProduct testProduct : testDefinition.getAllProducts()) { + if (testProduct.getId().equalsIgnoreCase(productId)) { + foundTestProduct = testProduct; + break; + } } - if (testProduct.exists()) { - final File testProductFile = getTestProductFile(testProduct); + if (foundTestProduct == null) { + String message = "The test product with id '"+ productId+"' does not exist."; + if (FAIL_ON_MISSING_DATA) { + fail(message); + } else { + logger.info(message); + } + } else if (foundTestProduct.exists()) { + logger.info("Start testing the product content: reader plugin class: " + readerPluginClassName+ ", product id: " + foundTestProduct.getId()+", product relative path: " + foundTestProduct.getRelativePath()+"."); - final ProductReader productReader = testDefinition.getProductReaderPlugin().createReaderInstance(); + long startTime = System.currentTimeMillis(); + + final File testProductFile = new File(dataRootDir, foundTestProduct.getRelativePath()); + long startTimeReadProduct = System.currentTimeMillis(); + + final ProductReader productReader = testDefinition.getProductReaderPlugin().createReaderInstance(); final Product product = productReader.readProductNodes(testProductFile, null); try { - assertExpectedContent(testDefinition, productId, product); + if (logger.isLoggable(Level.FINE)) { + double elapsedSecondsReadProduct = (System.currentTimeMillis() - startTimeReadProduct) / 1000.0d; + StringBuilder message = new StringBuilder(); + message.append("Finish reading the product id: ") + .append(foundTestProduct.getId()) + .append(", relative path: ") + .append(foundTestProduct.getRelativePath()) + .append(", reader class: ") + .append(productReader.getClass().getName()) + .append(", size: ") + .append(product.getSceneRasterWidth()) + .append("x") + .append(product.getSceneRasterHeight()) + .append(", elapsed time: ") + .append(elapsedSecondsReadProduct) + .append(" seconds."); + logger.log(Level.FINE, message.toString()); + } + + assertExpectedContent(testDefinition, foundTestProduct, product); } catch (Throwable t) { errorCollector.addError(new Throwable("[" + productId + "] " + t.getMessage(), t)); } finally { @@ -90,81 +139,64 @@ public void testReadIntendedProductContent() throws IOException { product.dispose(); } } - } else { - logProductNotExistent(2, testProduct); - } - } - double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; - logger.info("Finish testing reader plugin: " + testDefinition.getProductReaderPlugin().getClass().getSimpleName() + ". The time elapsed is " + elapsedSeconds + " seconds."); - } - } - private static void createGlobalProductList() { - for (TestDefinition testDefinition : testDefinitionList) { - final List allPluginProducts = testDefinition.getAllProducts(); - for (TestProduct testProduct : allPluginProducts) { - if (!testIfIdAlreadyRegistered(testProduct)) { - testProductList.add(testProduct); + if (logger.isLoggable(Level.FINE)) { + double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; + StringBuilder message = new StringBuilder(); + message.append("Finish testing the product content id: ") + .append(foundTestProduct.getId()) + .append(", relative path: ") + .append(foundTestProduct.getRelativePath()) + .append(", elapsed time: ") + .append(elapsedSeconds) + .append(" seconds."); + logger.log(Level.FINE, message.toString()); + } + } else { + logger.info("The test product with id '"+ foundTestProduct.getId()+"' is missing from the disc."); } } } } - private static boolean testIfIdAlreadyRegistered(TestProduct testProduct) { - final String id = testProduct.getId(); - final TestProduct storedProduct = testProductList.getById(id); - if (storedProduct != null) { - if (storedProduct.isDifferent(testProduct)) { - fail("Test file with ID=" + id + " already defined with different settings"); - } - - return true; - } - - return false; - } - - private void logProductNotExistent(int indention, TestProduct testProduct) { - final StringBuilder sb = new StringBuilder(); - for (int i = 0; i < indention; i++) { - sb.append(INDENT); - } - logger.info(sb.toString() + "Not existent - " + testProduct.getId()); - } - - private static void assertExpectedContent(TestDefinition testDefinition, String productId, Product product) { - final ExpectedContent expectedContent = testDefinition.getExpectedContent(productId); + private static void assertExpectedContent(TestDefinition testDefinition, TestProduct testProduct, Product product) { + final ExpectedContent expectedContent = testDefinition.getExpectedContent(testProduct.getId()); if (expectedContent == null) { - logger.info("No expected content for product id '" + productId+"'."); + String message = "No expected content for product id '" + testProduct.getId()+"'."; + if (FAIL_ON_MISSING_DATA) { + fail(message); + } else { + logger.info(message); + } } else { long startTime = System.currentTimeMillis(); - logger.info("Start testing expected content: product id: " + productId+"."); - final ContentAssert contentAssert = new ContentAssert(expectedContent, productId, product); + final ContentAssert contentAssert = new ContentAssert(expectedContent, testProduct.getId(), product); contentAssert.assertProductContent(); - double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; - logger.info("Finish testing expected content: product id: " + productId+"'. The time elapsed is " + elapsedSeconds + " seconds."); + if (logger.isLoggable(Level.FINE)) { + double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; + StringBuilder message = new StringBuilder(); + message.append("Finish testing expected content: product id: ") + .append(testProduct.getId()) + .append(", relative path: ") + .append(testProduct.getRelativePath()) + .append(", elapsed time: ") + .append(elapsedSeconds) + .append(" seconds."); + logger.log(Level.FINE, message.toString()); + } } } - private File getTestProductFile(TestProduct testProduct) { - final String relativePath = testProduct.getRelativePath(); - final File testProductFile = new File(dataRootDir, relativePath); - - errorCollector.checkThat("testProductFile exist " + testProduct.getId(), testProductFile.exists(), is(true)); - return testProductFile; - } - - private static TestDefinitionList loadProductReaderTestDefinitions() throws IOException { + private static TestDefinitionList loadProductReaderTestDefinitions(String readerPlugInClassNameToTest, String productIdToTest) throws IOException { final ObjectMapper mapper = new ObjectMapper(); final Iterable readerPlugIns = SystemUtils.loadServices(ProductReaderPlugIn.class); - final String className = System.getProperty(PROPERTY_NAME_CASS_NAME); TestDefinitionList testDefinitionList = new TestDefinitionList(); for (ProductReaderPlugIn readerPlugIn : readerPlugIns) { final Class readerPlugInClass = readerPlugIn.getClass(); - if (className != null && !readerPlugInClass.getName().startsWith(className)) { + if (readerPlugInClassNameToTest != null && !readerPlugInClass.getName().startsWith(readerPlugInClassNameToTest)) { continue; } @@ -172,58 +204,52 @@ private static TestDefinitionList loadProductReaderTestDefinitions() throws IOEx final String dataResourceName = "/" + path + "-data.json"; final URL dataResource = readerPlugInClass.getResource(dataResourceName); if (dataResource == null) { - logger.warning(readerPlugInClass.getSimpleName() + " does not define test data"); + logger.warning("The reader plugin class '"+readerPlugInClass.getSimpleName() + "' does not define test data."); continue; } + final ProductList productList = mapper.readValue(dataResource, ProductList.class); + final TestDefinition testDefinition = new TestDefinition(); testDefinition.setProductReaderPlugin(readerPlugIn); - - final ProductList productList = mapper.readValue(dataResource, ProductList.class); - testIfProductFilesExists(productList); - testDefinition.addTestProducts(productList.getAll()); - - for (TestProduct product : productList.getAll()) { - final String fileResourceName = product.getId() + ".json"; - final URL fileResource = readerPlugInClass.getResource(fileResourceName); - if (fileResource == null) { - fail(readerPlugInClass.getSimpleName() + " resource file '" + fileResourceName + "' is missing"); + for (TestProduct testProduct : productList.getAll()) { + if (productIdToTest == null || productIdToTest.equalsIgnoreCase(testProduct.getId())) { + final File productFile = new File(dataRootDir, testProduct.getRelativePath()); + if (!productFile.exists()) { + testProduct.exists(false); + String message = "The product '" + productFile.getAbsolutePath()+"' of the reader plugin '"+ readerPlugInClass.getSimpleName()+"' does not exist."; + if (FAIL_ON_MISSING_DATA) { + fail(message); + } else { + logger.info(message); + } + } + testDefinition.getAllProducts().add(testProduct); + final String fileResourceName = testProduct.getId() + ".json"; + final URL fileResource = readerPlugInClass.getResource(fileResourceName); + if (fileResource == null) { + fail("The resource file '" + fileResourceName+"' of the reader plugin '"+ readerPlugInClass.getSimpleName()+"' does not exist."); + } + final ExpectedDataset expectedDataset = mapper.readValue(fileResource, ExpectedDataset.class); + testDefinition.addExpectedDataset(expectedDataset); } - final ExpectedDataset expectedDataset = mapper.readValue(fileResource, ExpectedDataset.class); - testDefinition.addExpectedDataset(expectedDataset); } testDefinitionList.add(testDefinition); } return testDefinitionList; } - private static void assertTestDataDirectory() { - final String dataDirProperty = System.getProperty(PROPERTY_NAME_DATA_DIR); - if (dataDirProperty == null) { - fail("Data directory path not set"); - } - dataRootDir = new File(dataDirProperty); - if (!dataRootDir.isDirectory()) { - fail("Data directory is not valid: " + dataDirProperty); + private static void assertRequiredTestDataDirectory() { + String dataDirPropertyValue = System.getProperty(PROPERTY_NAME_DATA_DIR); + if (dataDirPropertyValue == null) { + fail("The test data property '"+PROPERTY_NAME_DATA_DIR+"' representing the folder which contains the source products is not set."); } - } - - private static void testIfProductFilesExists(ProductList productList) { - for (TestProduct testProduct : productList) { - final String relativePath = testProduct.getRelativePath(); - final File productFile = new File(dataRootDir, relativePath); - if (!productFile.exists()) { - testProduct.exists(false); - if (FAIL_ON_MISSING_DATA) { - fail("Test product does not exist: " + productFile.getAbsolutePath()); - } - } + dataRootDir = new File(dataDirPropertyValue); + if (!dataRootDir.exists()) { + fail("The path '" + dataDirPropertyValue+"' representing the folder which contains the source products does not exist."); } - } - - private static void logFailOnMissingDataMessage() { - if (!FAIL_ON_MISSING_DATA) { - logger.warning("Tests will not fail if test data is missing!"); + if (!dataRootDir.isDirectory()) { + fail("The path '" + dataDirPropertyValue+"' exists and it does not denote a folder."); } } } From 885ee33cf8aa450a21276e0d92465799a96ff77a Mon Sep 17 00:00:00 2001 From: jeancoravu Date: Fri, 19 Jun 2020 10:58:52 +0300 Subject: [PATCH 5/7] Update the log messages in the RunProductReaderTest, RunGPTProductReaderTest classes. --- .../esa/snap/dataio/RunProductReaderTest.java | 195 +++++++++++------- .../esa/snap/gpt/RunGPTProductReaderTest.java | 103 ++++++--- 2 files changed, 196 insertions(+), 102 deletions(-) diff --git a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java index 9bafc3e..529f615 100644 --- a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java +++ b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java @@ -1,26 +1,27 @@ package org.esa.snap.dataio; import com.fasterxml.jackson.databind.ObjectMapper; +import org.esa.s2tbx.dataio.gdal.GDALLibraryInstaller; import org.esa.snap.core.dataio.ProductReader; import org.esa.snap.core.dataio.ProductReaderPlugIn; import org.esa.snap.core.datamodel.Product; import org.esa.snap.core.util.SystemUtils; -import org.esa.snap.lib.openjpeg.activator.OpenJPEGInstaller; +import org.esa.snap.dataio.netcdf.NetCdfActivator; +import org.esa.snap.jp2.reader.OpenJPEGLibraryInstaller; import org.esa.snap.runtime.LogUtils; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; import org.junit.rules.ErrorCollector; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.fail; public class RunProductReaderTest { @@ -30,56 +31,90 @@ public class RunProductReaderTest { private static final String PROPERTY_NAME_FAIL_ON_MISSING_DATA = "snap.reader.tests.failOnMissingData"; private static final String PROPERTY_NAME_DATA_DIR = "snap.reader.tests.data.dir"; private static final String PROPERTY_NAME_CASS_NAME = "snap.reader.tests.class.name"; - private static final String PROPERTY_NAME_PRODUCT_ID = "snap.reader.tests.product.id"; + private static final String PROPERTY_NAME_PRODUCT_IDS = "snap.reader.tests.product.ids"; - private static final boolean FAIL_ON_MISSING_DATA = Boolean.parseBoolean(System.getProperty(PROPERTY_NAME_FAIL_ON_MISSING_DATA, "true")); - - private static TestDefinitionList testDefinitionList; + private static List testDefinitionList; private static File dataRootDir; + private static boolean isFailOnMissingData; - @Rule - public ErrorCollector errorCollector = new ErrorCollector(); + private ErrorCollector errorCollector = new ErrorCollector(); public RunProductReaderTest() { } @BeforeClass public static void initialize() throws Exception { - assertRequiredTestDataDirectory(); - LogUtils.initLogger(); + String dataDirPropertyValue = System.getProperty(PROPERTY_NAME_DATA_DIR); + if (dataDirPropertyValue == null) { + fail("The test data property '"+PROPERTY_NAME_DATA_DIR+"' representing the folder which contains the source products is not set."); + } + String readerPlugInClassNameToTest = System.getProperty(PROPERTY_NAME_CASS_NAME); - String productIdToTest = System.getProperty(PROPERTY_NAME_PRODUCT_ID); + String productIdsToTest = System.getProperty(PROPERTY_NAME_PRODUCT_IDS); + String failOnMissingData = System.getProperty(PROPERTY_NAME_FAIL_ON_MISSING_DATA); if (logger.isLoggable(Level.FINE)) { StringBuilder message = new StringBuilder(); message.append("Input test parameters: data folder path: ") - .append(dataRootDir.getAbsolutePath()) + .append(dataDirPropertyValue) .append(", reader plugin class: ") .append(readerPlugInClassNameToTest) - .append(", product id :") - .append(productIdToTest) + .append(", product ids: ") + .append(productIdsToTest) + .append(", fail on missing data: ") + .append(failOnMissingData) .append("."); logger.log(Level.FINE, message.toString()); } + dataRootDir = new File(dataDirPropertyValue); + if (!dataRootDir.exists()) { + fail("The path '" + dataDirPropertyValue+"' representing the folder which contains the source products does not exist."); + } + if (!dataRootDir.isDirectory()) { + fail("The path '" + dataDirPropertyValue+"' exists and it does not denote a folder."); + } + + isFailOnMissingData = true; + if (failOnMissingData != null) { + isFailOnMissingData = Boolean.parseBoolean(failOnMissingData); + } + SystemUtils.init3rdPartyLibs(RunProductReaderTest.class); - if (!FAIL_ON_MISSING_DATA) { + if (!isFailOnMissingData) { logger.warning("Tests will not fail if test data is missing!"); } - testDefinitionList = loadProductReaderTestDefinitions(readerPlugInClassNameToTest, productIdToTest); + testDefinitionList = loadProductReaderTestDefinitions(readerPlugInClassNameToTest, productIdsToTest); - OpenJPEGInstaller.install(); + OpenJPEGLibraryInstaller.install(); + GDALLibraryInstaller.install(); + NetCdfActivator.activate(); } + /** + * The required parameters to run the method: + * -Dsnap.reader.tests.data.dir=\\CV-DEV-SRV01\Satellite_Imagery\TestingJUnitFiles + * + * Optional parameters: + * -Dsnap.reader.tests.class.name=org.esa.s2tbx.dataio.muscate.MuscateProductReaderPlugin + * -Dsnap.reader.tests.product.ids=MUSCATE-Zip-31TFK,MUSCATE-v18,MUSCATE-v20 + * -Dsnap.reader.tests.failOnMissingData=false + * + * @throws Exception + */ @Test public void testReadIntendedProductContent() throws IOException { logger.info("Test product content for " + testDefinitionList.size() + " reader plugins."); - for (TestDefinition testDefinition : testDefinitionList) { + for (int k=0; k 0) { + logger.info(""); // write an empty line + } final List intendedProductIds = testDefinition.getDecodableProductIds(); String readerPluginClassName = testDefinition.getProductReaderPlugin().getClass().getSimpleName(); @@ -95,7 +130,7 @@ public void testReadIntendedProductContent() throws IOException { } if (foundTestProduct == null) { String message = "The test product with id '"+ productId+"' does not exist."; - if (FAIL_ON_MISSING_DATA) { + if (isFailOnMissingData) { fail(message); } else { logger.info(message); @@ -111,46 +146,53 @@ public void testReadIntendedProductContent() throws IOException { final ProductReader productReader = testDefinition.getProductReaderPlugin().createReaderInstance(); final Product product = productReader.readProductNodes(testProductFile, null); - try { + if (product == null) { + String message = "The product can not be read from file '" + testProductFile.getAbsolutePath()+"' does not exist."; + if (isFailOnMissingData) { + fail(message); + } else { + logger.log(Level.SEVERE, message); + } + } else { + try { + if (logger.isLoggable(Level.FINE)) { + double elapsedSecondsReadProduct = (System.currentTimeMillis() - startTimeReadProduct) / 1000.0d; + StringBuilder message = new StringBuilder(); + message.append("Finish reading the product id: ") + .append(foundTestProduct.getId()) + .append(", relative path: ") + .append(foundTestProduct.getRelativePath()) + .append(", reader class: ") + .append(productReader.getClass().getName()) + .append(", size: ") + .append(product.getSceneRasterWidth()) + .append("x") + .append(product.getSceneRasterHeight()) + .append(", elapsed time: ") + .append(elapsedSecondsReadProduct) + .append(" seconds."); + logger.log(Level.FINE, message.toString()); + } + + assertExpectedContent(testDefinition, foundTestProduct, product); + } catch (Throwable t) { + errorCollector.addError(new Throwable("[" + productId + "] " + t.getMessage(), t)); + } finally { + product.dispose(); + } + if (logger.isLoggable(Level.FINE)) { - double elapsedSecondsReadProduct = (System.currentTimeMillis() - startTimeReadProduct) / 1000.0d; + double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; StringBuilder message = new StringBuilder(); - message.append("Finish reading the product id: ") + message.append("Finish testing the product content id: ") .append(foundTestProduct.getId()) .append(", relative path: ") .append(foundTestProduct.getRelativePath()) - .append(", reader class: ") - .append(productReader.getClass().getName()) - .append(", size: ") - .append(product.getSceneRasterWidth()) - .append("x") - .append(product.getSceneRasterHeight()) .append(", elapsed time: ") - .append(elapsedSecondsReadProduct) + .append(elapsedSeconds) .append(" seconds."); logger.log(Level.FINE, message.toString()); } - - assertExpectedContent(testDefinition, foundTestProduct, product); - } catch (Throwable t) { - errorCollector.addError(new Throwable("[" + productId + "] " + t.getMessage(), t)); - } finally { - if (product != null) { - product.dispose(); - } - } - - if (logger.isLoggable(Level.FINE)) { - double elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000.0d; - StringBuilder message = new StringBuilder(); - message.append("Finish testing the product content id: ") - .append(foundTestProduct.getId()) - .append(", relative path: ") - .append(foundTestProduct.getRelativePath()) - .append(", elapsed time: ") - .append(elapsedSeconds) - .append(" seconds."); - logger.log(Level.FINE, message.toString()); } } else { logger.info("The test product with id '"+ foundTestProduct.getId()+"' is missing from the disc."); @@ -163,10 +205,10 @@ private static void assertExpectedContent(TestDefinition testDefinition, TestPro final ExpectedContent expectedContent = testDefinition.getExpectedContent(testProduct.getId()); if (expectedContent == null) { String message = "No expected content for product id '" + testProduct.getId()+"'."; - if (FAIL_ON_MISSING_DATA) { + if (isFailOnMissingData) { fail(message); } else { - logger.info(message); + logger.log(Level.SEVERE, message); } } else { long startTime = System.currentTimeMillis(); @@ -189,11 +231,23 @@ private static void assertExpectedContent(TestDefinition testDefinition, TestPro } } - private static TestDefinitionList loadProductReaderTestDefinitions(String readerPlugInClassNameToTest, String productIdToTest) throws IOException { + private static List loadProductReaderTestDefinitions(String readerPlugInClassNameToTest, String productIdsToTest) throws IOException { final ObjectMapper mapper = new ObjectMapper(); final Iterable readerPlugIns = SystemUtils.loadServices(ProductReaderPlugIn.class); - TestDefinitionList testDefinitionList = new TestDefinitionList(); + String[] productIds = null; + if (productIdsToTest != null) { + productIds = productIdsToTest.split(","); + if (productIds.length == 0) { + throw new IllegalArgumentException("The product ids array is empty."); + } else { + for (int k=0; k testDefinitionList = new ArrayList<>(); for (ProductReaderPlugIn readerPlugIn : readerPlugIns) { final Class readerPlugInClass = readerPlugIn.getClass(); if (readerPlugInClassNameToTest != null && !readerPlugInClass.getName().startsWith(readerPlugInClassNameToTest)) { @@ -213,12 +267,23 @@ private static TestDefinitionList loadProductReaderTestDefinitions(String reader final TestDefinition testDefinition = new TestDefinition(); testDefinition.setProductReaderPlugin(readerPlugIn); for (TestProduct testProduct : productList.getAll()) { - if (productIdToTest == null || productIdToTest.equalsIgnoreCase(testProduct.getId())) { + boolean canTestProduct; + if (productIds == null) { + canTestProduct = true; + } else { + canTestProduct = false; + for (int k=0; k Date: Tue, 14 Jul 2020 10:10:29 +0200 Subject: [PATCH 6/7] fixed typo --- .../java/org/esa/snap/dataio/ProductReaderAcceptanceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/esa/snap/dataio/ProductReaderAcceptanceTest.java b/src/test/java/org/esa/snap/dataio/ProductReaderAcceptanceTest.java index 4d4aa34..4e9f3bb 100644 --- a/src/test/java/org/esa/snap/dataio/ProductReaderAcceptanceTest.java +++ b/src/test/java/org/esa/snap/dataio/ProductReaderAcceptanceTest.java @@ -72,7 +72,7 @@ public class ProductReaderAcceptanceTest { private static final String PROPERTYNAME_FAIL_ON_MISSING_DATA = "snap.reader.tests.failOnMissingData"; private static final String PROPERTYNAME_FAIL_ON_INTENDED = "snap.reader.tests.failOnMultipleIntendedReaders"; private static final String PROPERTYNAME_LOG_FILE_PATH = "snap.reader.tests.log.file"; - private static final String PROPERTYNAME_CASS_NAME = "snap.reader.tests.class.name"; + private static final String PROPERTYNAME_CLASS_NAME = "snap.reader.tests.class.name"; private static final boolean FAIL_ON_MISSING_DATA = Boolean.parseBoolean(System.getProperty(PROPERTYNAME_FAIL_ON_MISSING_DATA, "true")); private static final String INDENT = "\t"; private static final ProductList testProductList = new ProductList(); @@ -471,7 +471,7 @@ private static void loadProductReaderTestDefinitions() throws IOException { final Iterable readerPlugIns = SystemUtils.loadServices(ProductReaderPlugIn.class); testDefinitionList = new TestDefinitionList(); - final String className = System.getProperty(PROPERTYNAME_CASS_NAME); + final String className = System.getProperty(PROPERTYNAME_CLASS_NAME); for (ProductReaderPlugIn readerPlugIn : readerPlugIns) { final Class readerPlugInClass = readerPlugIn.getClass(); From 8095f249b61936a339420b1de714c16bb3d6d0da Mon Sep 17 00:00:00 2001 From: Marco Peters Date: Tue, 14 Jul 2020 17:33:02 +0200 Subject: [PATCH 7/7] some renamings --- .../java/org/esa/snap/dataio/RunProductReaderTest.java | 5 ++--- .../java/org/esa/snap/gpt/RunGPTProductReaderTest.java | 10 ++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java index 529f615..18eab8c 100644 --- a/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java +++ b/src/test/java/org/esa/snap/dataio/RunProductReaderTest.java @@ -8,13 +8,12 @@ import org.esa.snap.core.util.SystemUtils; import org.esa.snap.dataio.netcdf.NetCdfActivator; import org.esa.snap.jp2.reader.OpenJPEGLibraryInstaller; -import org.esa.snap.runtime.LogUtils; +import org.esa.snap.runtime.LogUtils4Tests; import org.junit.BeforeClass; import org.junit.Test; import org.junit.rules.ErrorCollector; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.net.URL; import java.util.ArrayList; @@ -44,7 +43,7 @@ public RunProductReaderTest() { @BeforeClass public static void initialize() throws Exception { - LogUtils.initLogger(); + LogUtils4Tests.initLogger(); String dataDirPropertyValue = System.getProperty(PROPERTY_NAME_DATA_DIR); if (dataDirPropertyValue == null) { diff --git a/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java b/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java index 24cdc68..c4fb5f7 100644 --- a/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java +++ b/src/test/java/org/esa/snap/gpt/RunGPTProductReaderTest.java @@ -13,11 +13,14 @@ import org.esa.snap.dataio.ExpectedDataset; import org.esa.snap.dataio.netcdf.NetCdfActivator; import org.esa.snap.jp2.reader.OpenJPEGLibraryInstaller; -import org.esa.snap.runtime.LogUtils; +import org.esa.snap.runtime.LogUtils4Tests; import org.junit.BeforeClass; import org.junit.Test; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.OutputStream; +import java.io.PrintStream; import java.nio.file.NotDirectoryException; import java.util.ArrayList; import java.util.HashMap; @@ -27,7 +30,6 @@ import java.util.logging.Logger; import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; /** * Created by jcoravu on 1/4/2020. @@ -51,7 +53,7 @@ public RunGPTProductReaderTest() { @BeforeClass public static void initialize() throws Exception { - LogUtils.initLogger(); + LogUtils4Tests.initLogger(); SystemUtils.init3rdPartyLibs(RunGPTProductReaderTest.class);