Merge pull request #5149 from hansva/5148

do not fail on missing metadata objects, some cleanup, fixes #5148

Author: hansva

core/src/main/java/org/apache/hop/core/plugins/BasePluginTypeExposer.java

@@ -28,8 +28,6 @@
  * way.
  *
  * <p>Unless you're running within OSGI, you should never use this class.
- *
- * <p>Created by nbaker on 2/11/15.
  */
 public class BasePluginTypeExposer {
   private BasePluginType pluginType;

core/src/main/java/org/apache/hop/core/plugins/SupplementalPlugin.java

@@ -23,11 +23,7 @@
 import java.util.concurrent.Callable;
 import org.apache.hop.core.exception.HopPluginException;
 
-/**
- * This is a holder of Plugin Class mappings which supplement those of the stock Plugin.
- *
- * <p>Created by nbaker on 3/17/17.
- */
+/** This is a holder of Plugin Class mappings which supplement those of the stock Plugin. */
 public class SupplementalPlugin extends Plugin implements IClassLoadingPlugin {
   Map<Class, Callable> factoryMap = new HashMap<>();
   private Class<? extends IPluginType> pluginClass;

core/src/main/java/org/apache/hop/core/row/value/ValueMetaConverter.java

@@ -32,8 +32,6 @@
  * This class is intended to facilitate any needed conversions of a IValueMeta field from one type
  * to another. It was initially implemented for Orc storage in the hadoop shims project. This class
  * is added here because the converstions are not dependendant on orc in any way.
- *
- * <p>Created by tkafalas on 12/8/2017.
  */
 public class ValueMetaConverter implements Serializable, IValueMetaConverter {
   private static final String DEFAULT_DATE_FORMAT = ValueMetaBase.DEFAULT_DATE_FORMAT_MASK;

core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataParser.java

@@ -27,6 +27,8 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hop.core.encryption.ITwoWayPasswordEncoder;
 import org.apache.hop.core.exception.HopException;
@@ -42,8 +44,9 @@
 
 public class JsonMetadataParser<T extends IHopMetadata> {
 
-  private Class<T> managedClass;
-  private IHopMetadataProvider metadataProvider;
+  private final Class<T> managedClass;
+
+  @Setter @Getter private IHopMetadataProvider metadataProvider;
 
   public JsonMetadataParser(Class<T> managedClass, IHopMetadataProvider metadataProvider) {
     this.managedClass = managedClass;
@@ -401,20 +404,4 @@ private JSONObject savePojoProperty(String key, Object fieldValue, Class<?> fiel
           e);
     }
   }
-
-  /**
-   * Gets provider
-   *
-   * @return value of provider
-   */
-  public IHopMetadataProvider getMetadataProvider() {
-    return metadataProvider;
-  }
-
-  /**
-   * @param metadataProvider The provider to set
-   */
-  public void setMetadataProvider(IHopMetadataProvider metadataProvider) {
-    this.metadataProvider = metadataProvider;
-  }
 }

core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java

@@ -101,9 +101,7 @@ public T load(String name) throws HopException {
     try {
       // Load the JSON in a streaming fashion so we can parse the properties one by one...
       //
-      InputStream fileInputStream = null;
-      try {
-        fileInputStream = HopVfs.getInputStream(filename);
+      try (InputStream fileInputStream = HopVfs.getInputStream(filename)) {
         JsonFactory jsonFactory = new JsonFactory();
         try (com.fasterxml.jackson.core.JsonParser jsonParser =
             jsonFactory.createParser(fileInputStream)) {
@@ -116,10 +114,6 @@ public T load(String name) throws HopException {
           t.setMetadataProviderName(metadataProvider.getDescription());
           return t;
         }
-      } finally {
-        if (fileInputStream != null) {
-          fileInputStream.close();
-        }
       }
     } catch (Exception e) {
       throw new HopException(
@@ -250,7 +244,7 @@ public T delete(String name) throws HopException {
   public List<String> listObjectNames() throws HopException {
     List<String> names = new ArrayList<>();
 
-    // Read only access doesn't require a folder;
+    // Read-only access doesn't require a folder
     validateBaseFolder(false);
     if (!baseFolderExists) {
       // This is not an error.  We simply don't have objects of the given type.
@@ -274,7 +268,7 @@ public List<String> listObjectNames() throws HopException {
 
   @Override
   public boolean exists(String name) throws HopException {
-    // Read only access doesn't require a folder;
+    // Read-only access doesn't require a folder
     validateBaseFolder(false);
     if (!baseFolderExists) {
       return false;

engine/src/main/java/org/apache/hop/core/metadata/SerializableMetadataProvider.java

@@ -20,8 +20,9 @@
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonToken;
 import java.io.ByteArrayInputStream;
-import org.apache.hop.core.Const;
+import java.nio.charset.StandardCharsets;
 import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.logging.LogChannel;
 import org.apache.hop.metadata.api.HopMetadata;
 import org.apache.hop.metadata.api.IHopMetadata;
 import org.apache.hop.metadata.api.IHopMetadataProvider;
@@ -106,20 +107,28 @@ public SerializableMetadataProvider(String storeJson) throws HopException {
     this.description = "Serializable metadata provider (source is JSON)";
     try {
 
-      ByteArrayInputStream inputStream = null;
-      try {
-        inputStream = new ByteArrayInputStream(storeJson.getBytes(Const.XML_ENCODING));
+      try (ByteArrayInputStream inputStream =
+          new ByteArrayInputStream(storeJson.getBytes(StandardCharsets.UTF_8))) {
 
         JsonFactory jsonFactory = new JsonFactory();
         com.fasterxml.jackson.core.JsonParser jsonParser = jsonFactory.createParser(inputStream);
 
         // Loop over the classes until there's no more left
         //
-        jsonParser.nextToken(); // skip {
+        jsonParser.nextToken(); // skip "{"
         while (jsonParser.nextToken() != JsonToken.END_OBJECT) {
 
           String classKey = jsonParser.getText();
-          Class<IHopMetadata> managedClass = getMetadataClassForKey(classKey);
+          Class<IHopMetadata> managedClass = getManagedClass(classKey);
+
+          if (managedClass == null) {
+            // Skip this JSON Array
+            jsonParser.nextToken(); // skip "{"
+            while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
+              // Go to END_ARRAY
+            }
+            continue;
+          }
 
           JsonMetadataParser<IHopMetadata> metadataParser =
               new JsonMetadataParser<>(managedClass, this);
@@ -128,19 +137,32 @@ public SerializableMetadataProvider(String storeJson) throws HopException {
 
           // Loop over the metadata objects in the JSON for the given class...
           //
-          jsonParser.nextToken(); // skip {
+          jsonParser.nextToken(); // skip "{"
           while (jsonParser.nextToken() != JsonToken.END_ARRAY) {
             IHopMetadata object = metadataParser.loadJsonObject(managedClass, jsonParser);
-            serializer.save(object);
+            if (object != null) {
+              serializer.save(object);
+            }
           }
         }
-      } finally {
-        if (inputStream != null) {
-          inputStream.close();
-        }
       }
     } catch (Exception e) {
       throw new HopException("Error reading metadata from JSON", e);
     }
   }
+
+  /**
+   * @param classKey
+   * @return
+   */
+  private Class<IHopMetadata> getManagedClass(String classKey) {
+    Class<IHopMetadata> managedClass;
+    try {
+      managedClass = getMetadataClassForKey(classKey);
+    } catch (HopException e) {
+      LogChannel.GENERAL.logError("Error loading class " + classKey, e);
+      managedClass = null;
+    }
+    return managedClass;
+  }
 }

plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileNameParserIT.java

@@ -24,7 +24,6 @@
 import org.apache.hop.vfs.s3.s3.vfs.S3FileNameParser;
 import org.junit.Test;
 
-/** created by: rfellows date: 5/25/12 */
 public class S3FileNameParserIT {
 
   @Test

plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileNameTest.java

@@ -24,7 +24,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-/** created by: rfellows date: 05/17/2012 */
 public class S3FileNameTest {
 
   private S3FileName fileName = null;

plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3FileObjectTest.java

@@ -70,7 +70,6 @@
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 
-/** created by: dzmitry_bahdanovich date: 10/18/13 */
 public class S3FileObjectTest {
 
   public static final String HOST = "S3";

plugins/tech/aws/src/test/java/org/apache/hop/vfs/s3/vfs/S3NFileNameTest.java

@@ -24,7 +24,6 @@
 import org.junit.Before;
 import org.junit.Test;
 
-/** created by: rfellows date: 05/17/2012 */
 public class S3NFileNameTest {
 
   private S3NFileName fileName = null;