diff --git a/cmd/dump.go b/cmd/dump.go index 14742cf2..326289f4 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -18,6 +18,7 @@ const ( defaultBegin = "+0" defaultFrequency = 1440 defaultMaxAllowedPacket = 4194304 + defaultFilenamePattern = "db_backup_{{ .now }}.{{ .compression }}" ) func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) { @@ -140,6 +141,14 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er if retention == "" && cmdConfig.configuration != nil { retention = cmdConfig.configuration.Prune.Retention } + filenamePattern := v.GetString("filename-pattern") + + if !v.IsSet("filename-pattern") && cmdConfig.configuration != nil { + filenamePattern = cmdConfig.configuration.Dump.FilenamePattern + } + if filenamePattern == "" { + filenamePattern = defaultFilenamePattern + } // timer options once := v.GetBool("once") @@ -188,6 +197,7 @@ func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, er Compact: compact, MaxAllowedPacket: maxAllowedPacket, Run: uid, + FilenamePattern: filenamePattern, } err := executor.Dump(dumpOpts) if err != nil { @@ -247,7 +257,7 @@ S3: If it is a URL of the format s3://bucketname/path then it will connect via S flags.String("compression", defaultCompression, "Compression to use. Supported are: `gzip`, `bzip2`") // source filename pattern - flags.String("filename-pattern", "db_backup_{{ .now }}.{{ .compression }}", "Pattern to use for filename in target. See documentation.") + flags.String("filename-pattern", defaultFilenamePattern, "Pattern to use for filename in target. See documentation.") // pre-backup scripts flags.String("pre-backup-scripts", "", "Directory wherein any file ending in `.sh` will be run pre-backup.") diff --git a/cmd/dump_test.go b/cmd/dump_test.go index 7fa882a1..8b885f92 100644 --- a/cmd/dump_test.go +++ b/cmd/dump_test.go @@ -38,12 +38,14 @@ func TestDumpCmd(t *testing.T) { MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil}, {"file URL with prune", []string{"--server", "abc", "--target", "file:///foo/bar", "--retention", "1h"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}}, // database name and port @@ -52,12 +54,14 @@ func TestDumpCmd(t *testing.T) { MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil}, {"database explicit name with explicit port", []string{"--server", "abc", "--port", "3307", "--target", "file:///foo/bar"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: 3307}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil}, // config file @@ -66,12 +70,21 @@ func TestDumpCmd(t *testing.T) { MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}}, {"config file with port override", []string{"--config-file", "testdata/config.yml", "--port", "3307"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", + }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}}, + {"config file with filename pattern override", []string{"--config-file", "testdata/pattern.yml", "--port", "3307"}, "", false, core.DumpOptions{ + Targets: []storage.Storage{file.New(*fileTargetURL)}, + MaxAllowedPacket: defaultMaxAllowedPacket, + Compressor: &compression.GzipCompressor{}, + DBConn: database.Connection{Host: "abcd", Port: 3307, User: "user2", Pass: "xxxx2"}, + FilenamePattern: "foo_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}}, // timer options @@ -80,24 +93,28 @@ func TestDumpCmd(t *testing.T) { MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Once: true, Frequency: defaultFrequency, Begin: defaultBegin}, nil}, {"cron flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Cron: "0 0 * * *"}, nil}, {"begin flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--begin", "1234"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: "1234"}, nil}, {"frequency flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--frequency", "10"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, MaxAllowedPacket: defaultMaxAllowedPacket, Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: 10, Begin: defaultBegin}, nil}, {"incompatible flags: once/cron", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--cron", "0 0 * * *"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil}, {"incompatible flags: once/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil}, @@ -114,6 +131,7 @@ func TestDumpCmd(t *testing.T) { Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, PreBackupScripts: "/prebackup", + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil}, {"postbackup scripts", []string{"--server", "abc", "--target", "file:///foo/bar", "--post-backup-scripts", "/postbackup"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, @@ -121,6 +139,7 @@ func TestDumpCmd(t *testing.T) { Compressor: &compression.GzipCompressor{}, DBConn: database.Connection{Host: "abc", Port: defaultPort}, PostBackupScripts: "/postbackup", + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil}, {"prebackup and postbackup scripts", []string{"--server", "abc", "--target", "file:///foo/bar", "--post-backup-scripts", "/postbackup", "--pre-backup-scripts", "/prebackup"}, "", false, core.DumpOptions{ Targets: []storage.Storage{file.New(*fileTargetURL)}, @@ -129,6 +148,7 @@ func TestDumpCmd(t *testing.T) { DBConn: database.Connection{Host: "abc", Port: defaultPort}, PreBackupScripts: "/prebackup", PostBackupScripts: "/postbackup", + FilenamePattern: "db_backup_{{ .now }}.{{ .compression }}", }, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil}, } diff --git a/cmd/testdata/pattern.yml b/cmd/testdata/pattern.yml new file mode 100644 index 00000000..1db344c0 --- /dev/null +++ b/cmd/testdata/pattern.yml @@ -0,0 +1,26 @@ +version: config.databack.io/v1 +kind: local + +spec: + database: + server: abcd + port: 3306 + credentials: + username: user2 + password: xxxx2 + + targets: + local: + type: file + url: file:///foo/bar + other: + type: file + url: /foo/bar + + dump: + filenamePattern: "foo_{{ .now }}.{{ .compression }}" + targets: + - local + + prune: + retention: "1h" \ No newline at end of file diff --git a/pkg/core/dump.go b/pkg/core/dump.go index f428d8db..06cb9009 100644 --- a/pkg/core/dump.go +++ b/pkg/core/dump.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "html/template" "os" "path" "path/filepath" @@ -24,6 +25,7 @@ func (e *Executor) Dump(opts DumpOptions) error { compact := opts.Compact suppressUseDatabase := opts.SuppressUseDatabase maxAllowedPacket := opts.MaxAllowedPacket + filenamePattern := opts.FilenamePattern logger := e.Logger.WithField("run", opts.Run.String()) now := time.Now() @@ -36,7 +38,10 @@ func (e *Executor) Dump(opts DumpOptions) error { // sourceFilename: file that the uploader looks for when performing the upload // targetFilename: the remote file that is actually uploaded sourceFilename := fmt.Sprintf("db_backup_%s.%s", timepart, compressor.Extension()) - targetFilename := sourceFilename + targetFilename, err := processFilenamePattern(filenamePattern, now, timepart, compressor.Extension()) + if err != nil { + return fmt.Errorf("failed to process filename pattern: %v", err) + } // create a temporary working directory tmpdir, err := os.MkdirTemp("", "databacker_backup") @@ -45,7 +50,7 @@ func (e *Executor) Dump(opts DumpOptions) error { } defer os.RemoveAll(tmpdir) // execute pre-backup scripts if any - if err := preBackup(timepart, path.Join(tmpdir, targetFilename), tmpdir, opts.PreBackupScripts, logger.Level == log.DebugLevel); err != nil { + if err := preBackup(timepart, path.Join(tmpdir, sourceFilename), tmpdir, opts.PreBackupScripts, logger.Level == log.DebugLevel); err != nil { return fmt.Errorf("error running pre-restore: %v", err) } @@ -102,13 +107,13 @@ func (e *Executor) Dump(opts DumpOptions) error { f.Close() // execute post-backup scripts if any - if err := postBackup(timepart, path.Join(tmpdir, targetFilename), tmpdir, opts.PostBackupScripts, logger.Level == log.DebugLevel); err != nil { + if err := postBackup(timepart, path.Join(tmpdir, sourceFilename), tmpdir, opts.PostBackupScripts, logger.Level == log.DebugLevel); err != nil { return fmt.Errorf("error running pre-restore: %v", err) } // upload to each destination for _, t := range targets { - logger.Debugf("uploading via protocol %s from %s", t.Protocol(), targetFilename) + logger.Debugf("uploading via protocol %s from %s to %s", t.Protocol(), sourceFilename, targetFilename) copied, err := t.Push(targetFilename, filepath.Join(tmpdir, sourceFilename), logger) if err != nil { return fmt.Errorf("failed to push file: %v", err) @@ -141,3 +146,29 @@ func postBackup(timestamp, dumpfile, dumpdir, postBackupDir string, debug bool) } return runScripts(postBackupDir, env) } + +// processFilenamePattern takes a template pattern and processes it with the current time. +// Passes the timestamp as a string, because it sometimes gets changed for safechars. +func processFilenamePattern(pattern string, now time.Time, timestamp, ext string) (string, error) { + if pattern == "" { + return "", fmt.Errorf("filename pattern is empty") + } + tmpl, err := template.New("filename").Parse(pattern) + if err != nil { + return "", fmt.Errorf("failed to parse filename pattern: %v", err) + } + var buf strings.Builder + if err := tmpl.Execute(&buf, map[string]string{ + "now": timestamp, + "year": now.Format("2006"), + "month": now.Format("01"), + "day": now.Format("02"), + "hour": now.Format("15"), + "minute": now.Format("04"), + "second": now.Format("05"), + "compression": ext, + }); err != nil { + return "", fmt.Errorf("failed to execute filename pattern: %v", err) + } + return buf.String(), nil +} diff --git a/pkg/core/dumpoptions.go b/pkg/core/dumpoptions.go index 4a41cba9..ae71ad9c 100644 --- a/pkg/core/dumpoptions.go +++ b/pkg/core/dumpoptions.go @@ -20,4 +20,5 @@ type DumpOptions struct { SuppressUseDatabase bool MaxAllowedPacket int Run uuid.UUID + FilenamePattern string }