Improved user interface for pipeline and workflow verification #4912

- Add the possibility of verifying workflows. - Display verification results in graph extra views. - Add keyboard shortcut F7 to verify a component. - Edit a component when double-clicking on a problem. - Replace Metrics icon.
apache · Feb 17, 2025 · 3d9a45b · 3d9a45b
1 parent 89611b3
commit 3d9a45b
Show file tree

Hide file tree

Showing 28 changed files with 796 additions and 420 deletions.
diff --git a/core/src/main/resources/org/apache/hop/i18n/messages/messages_en_US.properties b/core/src/main/resources/org/apache/hop/i18n/messages/messages_en_US.properties
@@ -108,6 +108,8 @@ System.Log.UnexpectedError=Unexpected error
 System.Tooltip.Browse=Browse folders or files...
 System.Tooltip.BrowseForDir=Browse for a directory
 System.Tooltip.BrowseForFileOrDirAndAdd=Browse for a file or directory & add to the list
+System.Tooltip.CollapseALl=Collapse all
+System.Tooltip.ExpandAll=Expand all
 System.Tooltip.GetFields=Get the fields as defined in previous transforms.
 System.Tooltip.Help=Display detailed information
 System.Tooltip.VariableToDir=Insert a variable in the directory

diff --git a/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java b/engine/src/main/java/org/apache/hop/pipeline/PipelineMeta.java
@@ -40,6 +40,7 @@
 import org.apache.hop.core.ICheckResult;
 import org.apache.hop.core.IProgressMonitor;
 import org.apache.hop.core.NotePadMeta;
+import org.apache.hop.core.ProgressNullMonitorListener;
 import org.apache.hop.core.Result;
 import org.apache.hop.core.SqlStatement;
 import org.apache.hop.core.attributes.AttributesUtil;
@@ -2782,42 +2783,41 @@ public void checkTransforms(
       IVariables variables,
       IHopMetadataProvider metadataProvider) {
     try {
-      remarks.clear(); // Start with a clean slate...
+      // Start with a clean slate...
+      remarks.clear();
 
-      Map<IValueMeta, String> values = new Hashtable<>();
-      String[] transformnames;
-      TransformMeta[] transforms;
-      List<TransformMeta> selectedTransforms = getSelectedTransforms();
-      if (!onlySelected || selectedTransforms.isEmpty()) {
-        transformnames = getTransformNames();
-        transforms = getTransformsArray();
-      } else {
-        transformnames = getSelectedTransformNames();
-        transforms = selectedTransforms.toArray(new TransformMeta[selectedTransforms.size()]);
+      if (monitor == null) {
+        monitor = new ProgressNullMonitorListener();
       }
 
+      Map<IValueMeta, String> values = new Hashtable<>();
+
+      List<TransformMeta> transforms = (onlySelected) ? getSelectedTransforms() : getTransforms();
+
+      TransformMeta[] transformArray = transforms.toArray(new TransformMeta[0]);
+
       ExtensionPointHandler.callExtensionPoint(
           LogChannel.GENERAL,
           variables,
           HopExtensionPoint.BeforeCheckTransforms.id,
-          new CheckTransformsExtension(remarks, variables, this, transforms, metadataProvider));
+          new CheckTransformsExtension(remarks, variables, this, transformArray, metadataProvider));
 
       boolean stopChecking = false;
 
-      if (monitor != null) {
-        monitor.beginTask(
-            BaseMessages.getString(PKG, "PipelineMeta.Monitor.VerifyingThisPipelineTask.Title"),
-            transforms.length + 2);
-      }
+      monitor.beginTask(
+          BaseMessages.getString(PKG, "PipelineMeta.Monitor.VerifyingThisPipelineTask.Title"),
+          transforms.size());
+
+      int worked = 1;
+      for (TransformMeta transformMeta : transforms) {
 
-      for (int i = 0; i < transforms.length && !stopChecking; i++) {
-        if (monitor != null) {
-          monitor.subTask(
-              BaseMessages.getString(
-                  PKG, "PipelineMeta.Monitor.VerifyingTransformTask.Title", transformnames[i]));
+        if (stopChecking) {
+          break;
         }
 
-        TransformMeta transformMeta = transforms[i];
+        monitor.subTask(
+            BaseMessages.getString(
+                PKG, "PipelineMeta.Monitor.VerifyingTransformTask.Title", transformMeta.getName()));
 
         int nrinfo = findNrInfoTransforms(transformMeta);
         TransformMeta[] infoTransform = null;
@@ -2978,21 +2978,18 @@ public void checkTransforms(
           remarks.add(cr);
         }
 
-        if (monitor != null) {
-          monitor.worked(1); // progress bar...
-          if (monitor.isCanceled()) {
-            stopChecking = true;
-          }
+        monitor.worked(worked++); // progress bar...
+        if (monitor.isCanceled()) {
+          stopChecking = true;
         }
       }
 
-      if (monitor != null) {
-        monitor.subTask(
-            BaseMessages.getString(
-                PKG,
-                "PipelineMeta.Monitor.CheckingForDatabaseUnfriendlyCharactersInFieldNamesTask.Title"));
-      }
-      if (values.size() > 0) {
+      monitor.subTask(
+          BaseMessages.getString(
+              PKG,
+              "PipelineMeta.Monitor.CheckingForDatabaseUnfriendlyCharactersInFieldNamesTask.Title"));
+
+      if (!values.isEmpty()) {
         for (IValueMeta v : values.keySet()) {
           String message = values.get(v);
           CheckResult cr =
@@ -3015,14 +3012,14 @@ public void checkTransforms(
                 null);
         remarks.add(cr);
       }
-      if (monitor != null) {
-        monitor.worked(1);
-      }
+
       ExtensionPointHandler.callExtensionPoint(
           LogChannel.GENERAL,
           variables,
           HopExtensionPoint.AfterCheckTransforms.id,
-          new CheckTransformsExtension(remarks, variables, this, transforms, metadataProvider));
+          new CheckTransformsExtension(remarks, variables, this, transformArray, metadataProvider));
+
+      monitor.done();
     } catch (Exception e) {
       throw new RuntimeException("Error checking transforms", e);
     }

diff --git a/engine/src/main/java/org/apache/hop/workflow/WorkflowMeta.java b/engine/src/main/java/org/apache/hop/workflow/WorkflowMeta.java
@@ -34,6 +34,7 @@
 import org.apache.hop.core.ICheckResult;
 import org.apache.hop.core.IProgressMonitor;
 import org.apache.hop.core.NotePadMeta;
+import org.apache.hop.core.ProgressNullMonitorListener;
 import org.apache.hop.core.SqlStatement;
 import org.apache.hop.core.attributes.AttributesUtil;
 import org.apache.hop.core.database.DatabaseMeta;
@@ -1720,7 +1721,7 @@ protected void setInternalEntryCurrentDirectory(IVariables variables) {
    * own settings.
    *
    * @param remarks List of CheckResult remarks inserted into by each Action
-   * @param onlySelected true if you only want to check the selected workflows
+   * @param onlySelected true if you only want to check the selected actions
    * @param monitor Progress monitor (not presently in use)
    */
   public void checkActions(
@@ -1729,39 +1730,40 @@ public void checkActions(
       IProgressMonitor monitor,
       IVariables variables,
       IHopMetadataProvider metadataProvider) {
-    remarks.clear(); // Empty remarks
-    if (monitor != null) {
-      monitor.beginTask(
-          BaseMessages.getString(PKG, "WorkflowMeta.Monitor.VerifyingThisActionTask.Title"),
-          workflowActions.size() + 2);
-    }
-    boolean stopChecking = false;
-    for (int i = 0; i < workflowActions.size() && !stopChecking; i++) {
-      ActionMeta copy = workflowActions.get(i); // get the action copy
-      if ((!onlySelected) || (onlySelected && copy.isSelected())) {
-        IAction action = copy.getAction();
-        if (action != null) {
-          if (monitor != null) {
-            monitor.subTask(
-                BaseMessages.getString(
-                    PKG, "WorkflowMeta.Monitor.VerifyingAction.Title", action.getName()));
-          }
-          action.check(remarks, this, variables, metadataProvider);
-          if (monitor != null) {
-            monitor.worked(1); // progress bar...
-            if (monitor.isCanceled()) {
-              stopChecking = true;
-            }
-          }
-        }
+
+    // Start with a clean slate...
+    remarks.clear();
+
+    if (monitor == null) {
+      monitor = new ProgressNullMonitorListener();
+    }
+
+    List<ActionMeta> actions = (onlySelected) ? this.getSelectedActions() : getActions();
+
+    monitor.beginTask(
+        BaseMessages.getString(PKG, "WorkflowMeta.Monitor.VerifyingThisActionTask.Title"),
+        actions.size());
+
+    int worked = 1;
+    for (ActionMeta actionMeta : actions) {
+      if (monitor.isCanceled()) {
+        break;
       }
-      if (monitor != null) {
-        monitor.worked(1);
+
+      IAction action = actionMeta.getAction();
+      if (action != null) {
+        monitor.subTask(
+            BaseMessages.getString(
+                PKG, "WorkflowMeta.Monitor.VerifyingAction.Title", action.getName()));
+
+        action.check(remarks, this, variables, metadataProvider);
+
+        // Progress bar...
+        monitor.worked(worked++);
       }
     }
-    if (monitor != null) {
-      monitor.done();
-    }
+
+    monitor.done();
   }
 
   /**

diff --git a/...ders/src/main/java/org/apache/hop/workflow/actions/deletefolders/ActionDeleteFolders.java b/...ders/src/main/java/org/apache/hop/workflow/actions/deletefolders/ActionDeleteFolders.java
@@ -25,6 +25,7 @@
 import org.apache.commons.vfs2.FileSelectInfo;
 import org.apache.commons.vfs2.FileSelector;
 import org.apache.commons.vfs2.FileType;
+import org.apache.hop.core.CheckResult;
 import org.apache.hop.core.Const;
 import org.apache.hop.core.ICheckResult;
 import org.apache.hop.core.Result;
@@ -43,8 +44,6 @@
 import org.apache.hop.resource.ResourceReference;
 import org.apache.hop.workflow.WorkflowMeta;
 import org.apache.hop.workflow.action.ActionBase;
-import org.apache.hop.workflow.action.validator.ActionValidatorUtils;
-import org.apache.hop.workflow.action.validator.AndValidator;
 
 /** This defines a 'delete folders' action. */
 @Action(
@@ -305,16 +304,22 @@ public void check(
       WorkflowMeta workflowMeta,
       IVariables variables,
       IHopMetadataProvider metadataProvider) {
-    boolean res =
-        ActionValidatorUtils.andValidator()
-            .validate(
-                this,
-                "fileItems",
-                remarks,
-                AndValidator.putValidators(ActionValidatorUtils.notNullValidator()));
-
-    if (!res) {
-      return;
+    //    boolean res =
+    //        ActionValidatorUtils.andValidator()
+    //            .validate(
+    //                this,
+    //                "fileItems",
+    //                remarks,
+    //                AndValidator.putValidators(ActionValidatorUtils.notNullValidator()));
+    //
+    //    if (!res) {
+    //      return;
+    //    }
+
+    if (fileItems == null || fileItems.isEmpty()) {
+      String message = BaseMessages.getString(PKG, "CloneRowMeta.CheckResult.NrClonesdMissing");
+      remarks.add(
+          new CheckResult(ICheckResult.TYPE_RESULT_WARNING, "Any folders to deletes", this));
     }
 
     /* TODO: If we enable action check

diff --git a/ui/src/main/java/org/apache/hop/ui/core/gui/GuiResource.java b/ui/src/main/java/org/apache/hop/ui/core/gui/GuiResource.java
@@ -219,6 +219,7 @@ public class GuiResource {
   private Image imageExpandAll;
   private Image imageCollapseAll;
   private Image imageAdd;
+  private Image imageCheck;
   private Image imageCopy;
   private Image imageCancel;
   private Image imageCut;
@@ -440,6 +441,7 @@ private void dispose() {
 
     // Common images
     imageLogo.dispose();
+    imageCheck.dispose();
     imageDisabledHop.dispose();
     imageDatabase.dispose();
     imageData.dispose();
@@ -718,6 +720,7 @@ private void loadCommonImages() {
     imageAddAll = loadAsResource(display, "ui/images/add_all.svg", ConstUi.SMALL_ICON_SIZE);
     imageAddSingle = loadAsResource(display, "ui/images/add_single.svg", ConstUi.SMALL_ICON_SIZE);
     imageCalendar = loadAsResource(display, "ui/images/calendar.svg", ConstUi.SMALL_ICON_SIZE);
+    imageCheck = loadAsResource(display, "ui/images/check.svg", ConstUi.SMALL_ICON_SIZE);
     imageClosePanel = loadAsResource(display, "ui/images/close-panel.svg", ConstUi.SMALL_ICON_SIZE);
     imageCollapseAll =
         loadAsResource(display, "ui/images/collapse-all.svg", ConstUi.SMALL_ICON_SIZE);
@@ -2100,6 +2103,10 @@ public Image getImageClose() {
     return imageClose;
   }
 
+  public Image getImageCheck() {
+    return imageCheck;
+  }
+
   /**
    * Gets imageToolbarPause
    *