restructure integration test params

Signed-off-by: Avi Deitcher <[email protected]>

cmd/common_test.go

@@ -19,9 +19,9 @@ func newMockExecs() *mockExecs {
 	return m
 }
 
-func (m *mockExecs) Dump(opts core.DumpOptions) error {
+func (m *mockExecs) Dump(opts core.DumpOptions) (core.DumpResults, error) {
 	args := m.Called(opts)
-	return args.Error(0)
+	return core.DumpResults{}, args.Error(0)
 }
 
 func (m *mockExecs) Restore(opts core.RestoreOptions) error {

cmd/dump.go

@@ -199,7 +199,7 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er
 					Run:                 uid,
 					FilenamePattern:     filenamePattern,
 				}
-				err := executor.Dump(dumpOpts)
+				_, err := executor.Dump(dumpOpts)
 				if err != nil {
 					return fmt.Errorf("error running dump: %w", err)
 				}

cmd/root.go

@@ -19,7 +19,7 @@ import (
 type execs interface {
 	SetLogger(logger *log.Logger)
 	GetLogger() *log.Logger
-	Dump(opts core.DumpOptions) error
+	Dump(opts core.DumpOptions) (core.DumpResults, error)
 	Restore(opts core.RestoreOptions) error
 	Prune(opts core.PruneOptions) error
 	Timer(timerOpts core.TimerOptions, cmd func() error) error

pkg/core/dump.go

@@ -2,11 +2,11 @@ package core
 
 import (
 	"fmt"
-	"html/template"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"text/template"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -16,7 +16,10 @@ import (
 )
 
 // Dump run a single dump, based on the provided opts
-func (e *Executor) Dump(opts DumpOptions) error {
+func (e *Executor) Dump(opts DumpOptions) (DumpResults, error) {
+	results := DumpResults{Start: time.Now()}
+	defer func() { results.End = time.Now() }()
+
 	targets := opts.Targets
 	safechars := opts.Safechars
 	dbnames := opts.DBNames
@@ -29,35 +32,38 @@ func (e *Executor) Dump(opts DumpOptions) error {
 	logger := e.Logger.WithField("run", opts.Run.String())
 
 	now := time.Now()
+	results.Time = now
+
 	timepart := now.Format(time.RFC3339)
 	logger.Infof("beginning dump %s", timepart)
 	if safechars {
 		timepart = strings.ReplaceAll(timepart, ":", "-")
 	}
+	results.Timestamp = timepart
 
 	// sourceFilename: file that the uploader looks for when performing the upload
 	// targetFilename: the remote file that is actually uploaded
 	sourceFilename := fmt.Sprintf("db_backup_%s.%s", timepart, compressor.Extension())
-	targetFilename, err := processFilenamePattern(filenamePattern, now, timepart, compressor.Extension())
+	targetFilename, err := ProcessFilenamePattern(filenamePattern, now, timepart, compressor.Extension())
 	if err != nil {
-		return fmt.Errorf("failed to process filename pattern: %v", err)
+		return results, fmt.Errorf("failed to process filename pattern: %v", err)
 	}
 
 	// create a temporary working directory
 	tmpdir, err := os.MkdirTemp("", "databacker_backup")
 	if err != nil {
-		return fmt.Errorf("failed to make temporary working directory: %v", err)
+		return results, fmt.Errorf("failed to make temporary working directory: %v", err)
 	}
 	defer os.RemoveAll(tmpdir)
 	// execute pre-backup scripts if any
 	if err := preBackup(timepart, path.Join(tmpdir, sourceFilename), tmpdir, opts.PreBackupScripts, logger.Level == log.DebugLevel); err != nil {
-		return fmt.Errorf("error running pre-restore: %v", err)
+		return results, fmt.Errorf("error running pre-restore: %v", err)
 	}
 
 	// do the dump(s)
 	workdir, err := os.MkdirTemp("", "databacker_cache")
 	if err != nil {
-		return fmt.Errorf("failed to make temporary cache directory: %v", err)
+		return results, fmt.Errorf("failed to make temporary cache directory: %v", err)
 	}
 	defer os.RemoveAll(workdir)
 
@@ -66,62 +72,69 @@ func (e *Executor) Dump(opts DumpOptions) error {
 	// do we split the output by schema, or one big dump file?
 	if len(dbnames) == 0 {
 		if dbnames, err = database.GetSchemas(dbconn); err != nil {
-			return fmt.Errorf("failed to list database schemas: %v", err)
+			return results, fmt.Errorf("failed to list database schemas: %v", err)
 		}
 	}
 	for _, s := range dbnames {
 		outFile := path.Join(workdir, fmt.Sprintf("%s_%s.sql", s, timepart))
 		f, err := os.Create(outFile)
 		if err != nil {
-			return fmt.Errorf("failed to create dump file '%s': %v", outFile, err)
+			return results, fmt.Errorf("failed to create dump file '%s': %v", outFile, err)
 		}
 		dw = append(dw, database.DumpWriter{
 			Schemas: []string{s},
 			Writer:  f,
 		})
 	}
+	results.DumpStart = time.Now()
 	if err := database.Dump(dbconn, database.DumpOpts{
 		Compact:             compact,
 		SuppressUseDatabase: suppressUseDatabase,
 		MaxAllowedPacket:    maxAllowedPacket,
 	}, dw); err != nil {
-		return fmt.Errorf("failed to dump database: %v", err)
+		return results, fmt.Errorf("failed to dump database: %v", err)
 	}
+	results.DumpEnd = time.Now()
 
 	// create my tar writer to archive it all together
 	// WRONG: THIS WILL CAUSE IT TO TRY TO LOOP BACK ON ITSELF
 	outFile := path.Join(tmpdir, sourceFilename)
 	f, err := os.OpenFile(outFile, os.O_CREATE|os.O_WRONLY, 0o644)
 	if err != nil {
-		return fmt.Errorf("failed to open output file '%s': %v", outFile, err)
+		return results, fmt.Errorf("failed to open output file '%s': %v", outFile, err)
 	}
 	defer f.Close()
 	cw, err := compressor.Compress(f)
 	if err != nil {
-		return fmt.Errorf("failed to create compressor: %v", err)
+		return results, fmt.Errorf("failed to create compressor: %v", err)
 	}
 	if err := archive.Tar(workdir, cw); err != nil {
-		return fmt.Errorf("error creating the compressed archive: %v", err)
+		return results, fmt.Errorf("error creating the compressed archive: %v", err)
 	}
 	// we need to close it explicitly before moving ahead
 	f.Close()
 
 	// execute post-backup scripts if any
 	if err := postBackup(timepart, path.Join(tmpdir, sourceFilename), tmpdir, opts.PostBackupScripts, logger.Level == log.DebugLevel); err != nil {
-		return fmt.Errorf("error running pre-restore: %v", err)
+		return results, fmt.Errorf("error running pre-restore: %v", err)
 	}
 
 	// upload to each destination
 	for _, t := range targets {
-		logger.Debugf("uploading via protocol %s from %s to %s", t.Protocol(), sourceFilename, targetFilename)
-		copied, err := t.Push(targetFilename, filepath.Join(tmpdir, sourceFilename), logger)
+		uploadResult := UploadResult{Target: t.URL(), Start: time.Now()}
+		targetCleanFilename := t.Clean(targetFilename)
+		logger.Debugf("uploading via protocol %s from %s to %s", t.Protocol(), sourceFilename, targetCleanFilename)
+		copied, err := t.Push(targetCleanFilename, filepath.Join(tmpdir, sourceFilename), logger)
 		if err != nil {
-			return fmt.Errorf("failed to push file: %v", err)
+			return results, fmt.Errorf("failed to push file: %v", err)
 		}
 		logger.Debugf("completed copying %d bytes", copied)
+		uploadResult.Filename = targetCleanFilename
+		uploadResult.End = time.Now()
+		results.Uploads = append(results.Uploads, uploadResult)
 	}
 
-	return nil
+	return results, nil
 }
 
 // run pre-backup scripts, if they exist
@@ -147,9 +160,9 @@ func postBackup(timestamp, dumpfile, dumpdir, postBackupDir string, debug bool)
 	return runScripts(postBackupDir, env)
 }
 
-// processFilenamePattern takes a template pattern and processes it with the current time.
+// ProcessFilenamePattern takes a template pattern and processes it with the current time.
 // Passes the timestamp as a string, because it sometimes gets changed for safechars.
-func processFilenamePattern(pattern string, now time.Time, timestamp, ext string) (string, error) {
+func ProcessFilenamePattern(pattern string, now time.Time, timestamp, ext string) (string, error) {
 	if pattern == "" {
 		pattern = DefaultFilenamePattern
 	}

pkg/core/dumpresults.go

@@ -0,0 +1,22 @@
+package core
+
+import "time"
+
+// DumpResults lists results of the dump.
+type DumpResults struct {
+	Start     time.Time
+	End       time.Time
+	Time      time.Time
+	Timestamp string
+	DumpStart time.Time
+	DumpEnd   time.Time
+	Uploads   []UploadResult
+}
+
+// UploadResult lists results of an individual upload
+type UploadResult struct {
+	Target string
+	Filename string
+	Start time.Time
+	End time.Time
+}

pkg/storage/file/file.go

@@ -28,6 +28,10 @@ func (f *File) Push(target, source string, logger *log.Entry) (int64, error) {
 	return copyFile(source, filepath.Join(f.path, target))
 }
 
+func (f *File) Clean(filename string) string {
+	return filename
+}
+
 func (f *File) Protocol() string {
 	return "file"
 }

pkg/storage/s3/s3.go

@@ -125,6 +125,10 @@ func (s *S3) Push(target, source string, logger *log.Entry) (int64, error) {
 	return 0, nil
 }
 
+func (s *S3) Clean(filename string) string {
+	return filename
+}
+
 func (s *S3) Protocol() string {
 	return "s3"
 }

pkg/storage/smb/smb.go

@@ -80,7 +80,7 @@ func (s *SMB) Push(target, source string, logger *log.Entry) (int64, error) {
 		err    error
 	)
 	err = s.exec(s.url, func(fs *smb2.Share, sharepath string) error {
-		smbFilename := fmt.Sprintf("%s%c%s", sharepath, smb2.PathSeparator, filepath.Base(strings.ReplaceAll(target, ":", "-")))
+		smbFilename := fmt.Sprintf("%s%c%s", sharepath, smb2.PathSeparator, target)
 		from, err := os.Open(source)
 		if err != nil {
 			return err
@@ -97,6 +97,10 @@ func (s *SMB) Push(target, source string, logger *log.Entry) (int64, error) {
 	return copied, err
 }
 
+func (s *SMB) Clean(filename string) string {
+	return strings.ReplaceAll(filename, ":", "-")
+}
+
 func (s *SMB) Protocol() string {
 	return "smb"
 }

pkg/storage/storage.go

@@ -9,6 +9,7 @@ import (
 type Storage interface {
 	Protocol() string
 	URL() string
+	Clean(filename string) string
 	Push(target, source string, logger *log.Entry) (int64, error)
 	Pull(source, target string, logger *log.Entry) (int64, error)
 	ReadDir(dirname string, logger *log.Entry) ([]fs.FileInfo, error)