fix filename-pattern

Signed-off-by: Avi Deitcher <[email protected]>

cmd/dump.go

@@ -18,6 +18,7 @@ const (
 	defaultBegin            = "+0"
 	defaultFrequency        = 1440
 	defaultMaxAllowedPacket = 4194304
+	defaultFilenamePattern  = core.DefaultFilenamePattern
 )
 
 func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
@@ -140,6 +141,14 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er
 			if retention == "" && cmdConfig.configuration != nil {
 				retention = cmdConfig.configuration.Prune.Retention
 			}
+			filenamePattern := v.GetString("filename-pattern")
+
+			if !v.IsSet("filename-pattern") && cmdConfig.configuration != nil {
+				filenamePattern = cmdConfig.configuration.Dump.FilenamePattern
+			}
+			if filenamePattern == "" {
+				filenamePattern = defaultFilenamePattern
+			}
 
 			// timer options
 			once := v.GetBool("once")
@@ -188,6 +197,7 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er
 					Compact:             compact,
 					MaxAllowedPacket:    maxAllowedPacket,
 					Run:                 uid,
+					FilenamePattern:     filenamePattern,
 				}
 				err := executor.Dump(dumpOpts)
 				if err != nil {
@@ -247,7 +257,7 @@ S3: If it is a URL of the format s3://bucketname/path then it will connect via S
 	flags.String("compression", defaultCompression, "Compression to use. Supported are: `gzip`, `bzip2`")
 
 	// source filename pattern
-	flags.String("filename-pattern", "db_backup_{{ .now }}.{{ .compression }}", "Pattern to use for filename in target. See documentation.")
+	flags.String("filename-pattern", defaultFilenamePattern, "Pattern to use for filename in target. See documentation.")
 
 	// pre-backup scripts
 	flags.String("pre-backup-scripts", "", "Directory wherein any file ending in `.sh` will be run pre-backup.")

cmd/dump_test.go

@@ -38,12 +38,14 @@ func TestDumpCmd(t *testing.T) {
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"file URL with prune", []string{"--server", "abc", "--target", "file:///foo/bar", "--retention", "1h"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 
 		// database name and port
@@ -52,12 +54,14 @@ func TestDumpCmd(t *testing.T) {
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"database explicit name with explicit port", []string{"--server", "abc", "--port", "3307", "--target", "file:///foo/bar"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: 3307},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 
 		// config file
@@ -66,12 +70,21 @@ func TestDumpCmd(t *testing.T) {
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 		{"config file with port override", []string{"--config-file", "testdata/config.yml", "--port", "3307"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
+		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
+		{"config file with filename pattern override", []string{"--config-file", "testdata/pattern.yml", "--port", "3307"}, "", false, core.DumpOptions{
+			Targets:          []storage.Storage{file.New(*fileTargetURL)},
+			MaxAllowedPacket: defaultMaxAllowedPacket,
+			Compressor:       &compression.GzipCompressor{},
+			DBConn:           database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"},
+			FilenamePattern:  "foo_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 
 		// timer options
@@ -80,24 +93,28 @@ func TestDumpCmd(t *testing.T) {
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Once: true, Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"cron flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Cron: "0 0 * * *"}, nil},
 		{"begin flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--begin", "1234"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: "1234"}, nil},
 		{"frequency flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--frequency", "10"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: 10, Begin: defaultBegin}, nil},
 		{"incompatible flags: once/cron", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--cron", "0 0 * * *"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
 		{"incompatible flags: once/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
@@ -114,13 +131,15 @@ func TestDumpCmd(t *testing.T) {
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc", Port: defaultPort},
 			PreBackupScripts: "/prebackup",
+			FilenamePattern:  "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"postbackup scripts", []string{"--server", "abc", "--target", "file:///foo/bar", "--post-backup-scripts", "/postbackup"}, "", false, core.DumpOptions{
 			Targets:           []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket:  defaultMaxAllowedPacket,
 			Compressor:        &compression.GzipCompressor{},
 			DBConn:            database.Connection{Host: "abc", Port: defaultPort},
 			PostBackupScripts: "/postbackup",
+			FilenamePattern:   "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"prebackup and postbackup scripts", []string{"--server", "abc", "--target", "file:///foo/bar", "--post-backup-scripts", "/postbackup", "--pre-backup-scripts", "/prebackup"}, "", false, core.DumpOptions{
 			Targets:           []storage.Storage{file.New(*fileTargetURL)},
@@ -129,6 +148,7 @@ func TestDumpCmd(t *testing.T) {
 			DBConn:            database.Connection{Host: "abc", Port: defaultPort},
 			PreBackupScripts:  "/prebackup",
 			PostBackupScripts: "/postbackup",
+			FilenamePattern:   "db_backup_{{ .now }}.{{ .compression }}",
 		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 	}
 

cmd/testdata/pattern.yml

@@ -0,0 +1,26 @@
+version: config.databack.io/v1
+kind: local
+
+spec: 
+  database:
+    server: abcd
+    port: 3306
+    credentials:
+      username: user2
+      password: xxxx2
+
+  targets:
+    local:
+      type: file
+      url: file:///foo/bar
+    other:
+      type: file
+      url: /foo/bar
+
+  dump:
+    filenamePattern: "foo_{{ .now }}.{{ .compression }}"
+    targets:
+    - local
+
+  prune:
+    retention: "1h"

pkg/core/const.go

@@ -0,0 +1,5 @@
+package core
+
+const (
+	DefaultFilenamePattern = "db_backup_{{ .now }}.{{ .compression }}"
+)

pkg/core/dump.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"fmt"
+	"html/template"
 	"os"
 	"path"
 	"path/filepath"
@@ -24,6 +25,7 @@ func (e *Executor) Dump(opts DumpOptions) error {
 	compact := opts.Compact
 	suppressUseDatabase := opts.SuppressUseDatabase
 	maxAllowedPacket := opts.MaxAllowedPacket
+	filenamePattern := opts.FilenamePattern
 	logger := e.Logger.WithField("run", opts.Run.String())
 
 	now := time.Now()
@@ -36,7 +38,10 @@ func (e *Executor) Dump(opts DumpOptions) error {
 	// sourceFilename: file that the uploader looks for when performing the upload
 	// targetFilename: the remote file that is actually uploaded
 	sourceFilename := fmt.Sprintf("db_backup_%s.%s", timepart, compressor.Extension())
-	targetFilename := sourceFilename
+	targetFilename, err := processFilenamePattern(filenamePattern, now, timepart, compressor.Extension())
+	if err != nil {
+		return fmt.Errorf("failed to process filename pattern: %v", err)
+	}
 
 	// create a temporary working directory
 	tmpdir, err := os.MkdirTemp("", "databacker_backup")
@@ -45,7 +50,7 @@ func (e *Executor) Dump(opts DumpOptions) error {
 	}
 	defer os.RemoveAll(tmpdir)
 	// execute pre-backup scripts if any
-	if err := preBackup(timepart, path.Join(tmpdir, targetFilename), tmpdir, opts.PreBackupScripts, logger.Level == log.DebugLevel); err != nil {
+	if err := preBackup(timepart, path.Join(tmpdir, sourceFilename), tmpdir, opts.PreBackupScripts, logger.Level == log.DebugLevel); err != nil {
 		return fmt.Errorf("error running pre-restore: %v", err)
 	}
 
@@ -102,13 +107,13 @@ func (e *Executor) Dump(opts DumpOptions) error {
 	f.Close()
 
 	// execute post-backup scripts if any
-	if err := postBackup(timepart, path.Join(tmpdir, targetFilename), tmpdir, opts.PostBackupScripts, logger.Level == log.DebugLevel); err != nil {
+	if err := postBackup(timepart, path.Join(tmpdir, sourceFilename), tmpdir, opts.PostBackupScripts, logger.Level == log.DebugLevel); err != nil {
 		return fmt.Errorf("error running pre-restore: %v", err)
 	}
 
 	// upload to each destination
 	for _, t := range targets {
-		logger.Debugf("uploading via protocol %s from %s", t.Protocol(), targetFilename)
+		logger.Debugf("uploading via protocol %s from %s to %s", t.Protocol(), sourceFilename, targetFilename)
 		copied, err := t.Push(targetFilename, filepath.Join(tmpdir, sourceFilename), logger)
 		if err != nil {
 			return fmt.Errorf("failed to push file: %v", err)
@@ -141,3 +146,29 @@ func postBackup(timestamp, dumpfile, dumpdir, postBackupDir string, debug bool)
 	}
 	return runScripts(postBackupDir, env)
 }
+
+// processFilenamePattern takes a template pattern and processes it with the current time.
+// Passes the timestamp as a string, because it sometimes gets changed for safechars.
+func processFilenamePattern(pattern string, now time.Time, timestamp, ext string) (string, error) {
+	if pattern == "" {
+		pattern = DefaultFilenamePattern
+	}
+	tmpl, err := template.New("filename").Parse(pattern)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse filename pattern: %v", err)
+	}
+	var buf strings.Builder
+	if err := tmpl.Execute(&buf, map[string]string{
+		"now":         timestamp,
+		"year":        now.Format("2006"),
+		"month":       now.Format("01"),
+		"day":         now.Format("02"),
+		"hour":        now.Format("15"),
+		"minute":      now.Format("04"),
+		"second":      now.Format("05"),
+		"compression": ext,
+	}); err != nil {
+		return "", fmt.Errorf("failed to execute filename pattern: %v", err)
+	}
+	return buf.String(), nil
+}

pkg/core/dumpoptions.go

@@ -20,4 +20,5 @@ type DumpOptions struct {
 	SuppressUseDatabase bool
 	MaxAllowedPacket    int
 	Run                 uuid.UUID
+	FilenamePattern     string
 }