Added a config option to double-check file staging (off by default).

config/config.go

@@ -56,6 +56,9 @@ type serviceConfig struct {
 	DeleteAfter int `json:"delete_after" yaml:"delete_after"`
 	// flag indicating whether debug logging and other tools are enabled
 	Debug bool `json:"debug" yaml:"debug"`
+	// flag indicating whether an endpoint double-checks that files are staged
+	// (if not set, the endpoint will trust a database for staging status)
+	DoubleCheckStaging bool `json:"double_check_staging" yaml:"double_check_staging"`
 }
 
 // global config variables

tasks/subtask.go

@@ -29,6 +29,7 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/kbase/dts/auth"
+	"github.com/kbase/dts/config"
 	"github.com/kbase/dts/databases"
 	"github.com/kbase/dts/endpoints"
 )
@@ -116,18 +117,20 @@ func (subtask *transferSubtask) checkStaging() error {
 	}
 
 	if subtask.StagingStatus == databases.StagingStatusSucceeded { // staged!
-		// the database thinks the files are staged. Does its endpoint agree?
-		endpoint, err := endpoints.NewEndpoint(subtask.SourceEndpoint)
-		if err != nil {
-			return err
-		}
-		staged, err := endpoint.FilesStaged(subtask.Resources)
-		if err != nil {
-			return err
-		}
-		if !staged {
-			return fmt.Errorf("Database %s reports staged files, but endpoint %s cannot see them. Is the endpoint's root set properly?",
-				subtask.Source, subtask.SourceEndpoint)
+		if config.Service.DoubleCheckStaging {
+			// the database thinks the files are staged. Does its endpoint agree?
+			endpoint, err := endpoints.NewEndpoint(subtask.SourceEndpoint)
+			if err != nil {
+				return err
+			}
+			staged, err := endpoint.FilesStaged(subtask.Resources)
+			if err != nil {
+				return err
+			}
+			if !staged {
+				return fmt.Errorf("Database %s reports staged files, but endpoint %s cannot see them. Is the endpoint's root set properly?",
+					subtask.Source, subtask.SourceEndpoint)
+			}
 		}
 		return subtask.beginTransfer() // move along
 	}