Skip to content

Commit

Permalink
implement remote logging
Browse files Browse the repository at this point in the history
Signed-off-by: Avi Deitcher <[email protected]>
  • Loading branch information
deitch committed May 2, 2024
1 parent f8938b6 commit 15a96e4
Show file tree
Hide file tree
Showing 36 changed files with 742 additions and 273 deletions.
57 changes: 49 additions & 8 deletions cmd/common_test.go
Original file line number Diff line number Diff line change
@@ -1,41 +1,82 @@
package cmd

import (
"github.com/databacker/mysql-backup/pkg/compression"
"reflect"

"github.com/databacker/mysql-backup/pkg/core"
"github.com/databacker/mysql-backup/pkg/database"
"github.com/databacker/mysql-backup/pkg/storage"
"github.com/go-test/deep"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/mock"
)

type mockExecs struct {
mock.Mock
logger *log.Logger
}

func newMockExecs() *mockExecs {
m := &mockExecs{}
return m
}

func (m *mockExecs) dump(opts core.DumpOptions) error {
func (m *mockExecs) Dump(opts core.DumpOptions) error {
args := m.Called(opts)
return args.Error(0)
}

func (m *mockExecs) restore(target storage.Storage, targetFile string, dbconn database.Connection, databasesMap map[string]string, compressor compression.Compressor) error {
args := m.Called(target, targetFile, dbconn, databasesMap, compressor)
func (m *mockExecs) Restore(opts core.RestoreOptions) error {
args := m.Called(opts)
return args.Error(0)
}

func (m *mockExecs) prune(opts core.PruneOptions) error {
func (m *mockExecs) Prune(opts core.PruneOptions) error {
args := m.Called(opts)
return args.Error(0)
}
func (m *mockExecs) timer(timerOpts core.TimerOptions, cmd func() error) error {
func (m *mockExecs) Timer(timerOpts core.TimerOptions, cmd func() error) error {
args := m.Called(timerOpts)
err := args.Error(0)
if err != nil {
return err
}
return cmd()
}

func (m *mockExecs) SetLogger(logger *log.Logger) {
m.logger = logger
}

func (m *mockExecs) GetLogger() *log.Logger {
return m.logger
}

func equalIgnoreFields(a, b interface{}, fields []string) bool {
va := reflect.ValueOf(a)
vb := reflect.ValueOf(b)

// Check if both a and b are struct types
if va.Kind() != reflect.Struct || vb.Kind() != reflect.Struct {
return false
}

// Make a map of fields to ignore for quick lookup
ignoreMap := make(map[string]bool)
for _, f := range fields {
ignoreMap[f] = true
}

// Compare fields that are not in the ignore list
for i := 0; i < va.NumField(); i++ {
field := va.Type().Field(i).Name
if !ignoreMap[field] {
vaField := va.Field(i).Interface()
vbField := vb.Field(i).Interface()
diff := deep.Equal(vaField, vbField)
if diff != nil {
return false
}
}
}

return true
}
55 changes: 28 additions & 27 deletions cmd/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"fmt"
"strings"

log "github.com/sirupsen/logrus"
"github.com/google/uuid"
"github.com/spf13/cobra"
"github.com/spf13/viper"

Expand All @@ -20,7 +20,7 @@ const (
defaultMaxAllowedPacket = 4194304
)

func dumpCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
func dumpCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
if cmdConfig == nil {
return nil, fmt.Errorf("cmdConfig is nil")
}
Expand All @@ -37,7 +37,7 @@ func dumpCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
bindFlags(cmd, v)
},
RunE: func(cmd *cobra.Command, args []string) error {
log.Debug("starting dump")
cmdConfig.logger.Debug("starting dump")
// check targets
targetURLs := v.GetStringSlice("target")
var (
Expand Down Expand Up @@ -130,19 +130,6 @@ func dumpCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
return fmt.Errorf("failure to get compression '%s': %v", compressionAlgo, err)
}
}
dumpOpts := core.DumpOptions{
Targets: targets,
Safechars: safechars,
DBNames: include,
DBConn: cmdConfig.dbconn,
Compressor: compressor,
Exclude: exclude,
PreBackupScripts: preBackupScripts,
PostBackupScripts: preBackupScripts,
SuppressUseDatabase: noDatabaseName,
Compact: compact,
MaxAllowedPacket: maxAllowedPacket,
}

// retention, if enabled
retention := v.GetString("retention")
Expand Down Expand Up @@ -173,31 +160,45 @@ func dumpCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
Begin: begin,
Frequency: frequency,
}
dump := core.Dump
prune := core.Prune
timer := core.TimerCommand
if execs != nil {
dump = execs.dump
prune = execs.prune
timer = execs.timer
var executor execs
executor = &core.Executor{}
if passedExecs != nil {
executor = passedExecs
}
executor.SetLogger(cmdConfig.logger)

// at this point, any errors should not have usage
cmd.SilenceUsage = true
if err := timer(timerOpts, func() error {
err := dump(dumpOpts)
if err := executor.Timer(timerOpts, func() error {
uid := uuid.New()
dumpOpts := core.DumpOptions{
Targets: targets,
Safechars: safechars,
DBNames: include,
DBConn: cmdConfig.dbconn,
Compressor: compressor,
Exclude: exclude,
PreBackupScripts: preBackupScripts,
PostBackupScripts: preBackupScripts,
SuppressUseDatabase: noDatabaseName,
Compact: compact,
MaxAllowedPacket: maxAllowedPacket,
Run: uid,
}
err := executor.Dump(dumpOpts)
if err != nil {
return fmt.Errorf("error running dump: %w", err)
}
if retention != "" {
if err := prune(core.PruneOptions{Targets: targets, Retention: retention}); err != nil {
if err := executor.Prune(core.PruneOptions{Targets: targets, Retention: retention}); err != nil {
return fmt.Errorf("error running prune: %w", err)
}
}
return nil
}); err != nil {
return fmt.Errorf("error running command: %w", err)
}
log.Info("Backup complete")
executor.GetLogger().Info("Backup complete")
return nil
},
}
Expand Down
11 changes: 5 additions & 6 deletions cmd/dump_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,15 +110,14 @@ func TestDumpCmd(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := newMockExecs()
m.On("dump", mock.MatchedBy(func(dumpOpts core.DumpOptions) bool {
diff := deep.Equal(dumpOpts, tt.expectedDumpOptions)
if diff == nil {
m.On("Dump", mock.MatchedBy(func(dumpOpts core.DumpOptions) bool {
if equalIgnoreFields(dumpOpts, tt.expectedDumpOptions, []string{"Run"}) {
return true
}
t.Errorf("dumpOpts compare failed: %v", diff)
t.Errorf("dumpOpts compare failed: %#v %#v", dumpOpts, tt.expectedDumpOptions)
return false
})).Return(nil)
m.On("timer", mock.MatchedBy(func(timerOpts core.TimerOptions) bool {
m.On("Timer", mock.MatchedBy(func(timerOpts core.TimerOptions) bool {
diff := deep.Equal(timerOpts, tt.expectedTimerOptions)
if diff == nil {
return true
Expand All @@ -127,7 +126,7 @@ func TestDumpCmd(t *testing.T) {
return false
})).Return(nil)
if tt.expectedPruneOptions != nil {
m.On("prune", mock.MatchedBy(func(pruneOpts core.PruneOptions) bool {
m.On("Prune", mock.MatchedBy(func(pruneOpts core.PruneOptions) bool {
diff := deep.Equal(pruneOpts, *tt.expectedPruneOptions)
if diff == nil {
return true
Expand Down
24 changes: 13 additions & 11 deletions cmd/prune.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ import (
"fmt"
"strings"

log "github.com/sirupsen/logrus"
"github.com/google/uuid"
"github.com/spf13/cobra"
"github.com/spf13/viper"

"github.com/databacker/mysql-backup/pkg/core"
"github.com/databacker/mysql-backup/pkg/storage"
)

func pruneCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
func pruneCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
if cmdConfig == nil {
return nil, fmt.Errorf("cmdConfig is nil")
}
Expand All @@ -31,7 +31,7 @@ func pruneCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error)
bindFlags(cmd, v)
},
RunE: func(cmd *cobra.Command, args []string) error {
log.Debug("starting prune")
cmdConfig.logger.Debug("starting prune")
retention := v.GetString("retention")
targetURLs := v.GetStringSlice("target")
var (
Expand Down Expand Up @@ -96,18 +96,20 @@ func pruneCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error)
Frequency: frequency,
}

prune := core.Prune
timer := core.TimerCommand
if execs != nil {
prune = execs.prune
timer = execs.timer
var executor execs
executor = &core.Executor{}
if passedExecs != nil {
executor = passedExecs
}
if err := timer(timerOpts, func() error {
return prune(core.PruneOptions{Targets: targets, Retention: retention})
executor.SetLogger(cmdConfig.logger)

if err := executor.Timer(timerOpts, func() error {
uid := uuid.New()
return executor.Prune(core.PruneOptions{Targets: targets, Retention: retention, Run: uid})
}); err != nil {
return fmt.Errorf("error running prune: %w", err)
}
log.Info("Pruning complete")
executor.GetLogger().Info("Pruning complete")
return nil
},
}
Expand Down
10 changes: 4 additions & 6 deletions cmd/prune_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"github.com/databacker/mysql-backup/pkg/core"
"github.com/databacker/mysql-backup/pkg/storage"
"github.com/databacker/mysql-backup/pkg/storage/file"
"github.com/go-test/deep"
"github.com/stretchr/testify/mock"
)

Expand All @@ -32,15 +31,14 @@ func TestPruneCmd(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := newMockExecs()
m.On("prune", mock.MatchedBy(func(pruneOpts core.PruneOptions) bool {
diff := deep.Equal(pruneOpts, tt.expectedPruneOptions)
if diff == nil {
m.On("Prune", mock.MatchedBy(func(pruneOpts core.PruneOptions) bool {
if equalIgnoreFields(pruneOpts, tt.expectedPruneOptions, []string{"Run"}) {
return true
}
t.Errorf("pruneOpts compare failed: %v", diff)
t.Errorf("pruneOpts compare failed: %#v %#v", pruneOpts, tt.expectedPruneOptions)
return false
})).Return(nil)
m.On("timer", tt.expectedTimerOptions).Return(nil)
m.On("Timer", tt.expectedTimerOptions).Return(nil)
cmd, err := rootCmd(m)
if err != nil {
t.Fatal(err)
Expand Down
28 changes: 20 additions & 8 deletions cmd/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"fmt"
"strings"

log "github.com/sirupsen/logrus"
"github.com/google/uuid"
"github.com/spf13/cobra"
"github.com/spf13/viper"

Expand All @@ -14,7 +14,7 @@ import (
"github.com/databacker/mysql-backup/pkg/util"
)

func restoreCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
func restoreCmd(passedExecs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
if cmdConfig == nil {
return nil, fmt.Errorf("cmdConfig is nil")
}
Expand All @@ -28,7 +28,7 @@ func restoreCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error
},
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
log.Debug("starting restore")
cmdConfig.logger.Debug("starting restore")
targetFile := args[0]
target := v.GetString("target")
// get databases namesand mappings
Expand Down Expand Up @@ -94,16 +94,28 @@ func restoreCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error
return fmt.Errorf("invalid target url: %v", err)
}
}
restore := core.Restore
if execs != nil {
restore = execs.restore
var executor execs
executor = &core.Executor{}
if passedExecs != nil {
executor = passedExecs
}
executor.SetLogger(cmdConfig.logger)

// at this point, any errors should not have usage
cmd.SilenceUsage = true
if err := restore(store, targetFile, cmdConfig.dbconn, databasesMap, compressor); err != nil {
uid := uuid.New()
restoreOpts := core.RestoreOptions{
Target: store,
TargetFile: targetFile,
Compressor: compressor,
DatabasesMap: databasesMap,
DBConn: cmdConfig.dbconn,
Run: uid,
}
if err := executor.Restore(restoreOpts); err != nil {
return fmt.Errorf("error restoring: %v", err)
}
log.Info("Restore complete")
passedExecs.GetLogger().Info("Restore complete")
return nil
},
}
Expand Down
Loading

0 comments on commit 15a96e4

Please sign in to comment.