Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Demo robustness #3130

Draft
wants to merge 17 commits into
base: master
Choose a base branch
from
Draft

Demo robustness #3130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
c6ed34b
collect stats
chaitalisg Jul 6, 2023
160164e
recover from error
chaitalisg Jul 6, 2023
5eb2380
Merge branch 'master' into demo-robustness
chaitalisg Jul 6, 2023
5b6b879
recover from error
chaitalisg Jul 6, 2023
9ae19bc
Merge branch 'demo-robustness' of github.com:kastenhq/kopia into demo…
chaitalisg Jul 6, 2023
e0fa4ee
add more stats
chaitalisg Jul 7, 2023
b92ef49
add more stats
chaitalisg Jul 10, 2023
90f8177
Merge branch 'master' into demo-robustness
chaitalisg Jul 12, 2023
3817849
Merge branch 'master' into demo-robustness
chaitalisg Jul 13, 2023
7d59748
get stats after no space left error
chaitalisg Jul 13, 2023
9470506
reduce cache size
chaitalisg Jul 14, 2023
5280f5a
add logs to functions
chaitalisg Jul 14, 2023
5e51347
add logs to functions
chaitalisg Jul 14, 2023
b58b81d
add logs to functions
chaitalisg Jul 14, 2023
55d6e48
Merge branch 'master' into demo-robustness
chaitalisg Jul 18, 2023
b8f4a3e
Merge branch 'master' into demo-robustness
chaitalisg Aug 14, 2023
781017b
add debug message
chaitalisg Aug 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 153 additions & 0 deletions tests/robustness/engine/action.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@
"bytes"
"context"
"errors"
"fmt"
"log"
"math/rand"
"os/exec"
"path/filepath"
"strconv"
"strings"
Expand Down Expand Up @@ -86,6 +88,8 @@
// CheckErrRecovery tries to recover from no space left error
// by deleting data directories.
func (e *Engine) CheckErrRecovery(ctx context.Context, incomingErr error, actionOpts ActionOpts) (outgoingErr error) {
log.Printf("Inside CheckErrRecovery")

outgoingErr = incomingErr

if incomingErr == nil {
Expand All @@ -96,6 +100,8 @@

if errIsNotEnoughSpace(incomingErr) && ctrl[ThrowNoSpaceOnDeviceErrField] == "" {
// no space left on device
e.gatherStats()

// Delete everything in the data directory
outgoingErr = e.FileWriter.DeleteEverything(ctx)
if outgoingErr != nil {
Expand Down Expand Up @@ -257,6 +263,14 @@

b := &bytes.Buffer{}

// Get source dir size from snapshot being restored
// may have to implement another method to get snapshot ID storage stats
// snapshots, err := e.TestRepo.ListSnapshots(ctx)

// Check volume capacity of e.FileWriter.DataDirectory
// before restoring the snapshot
// Restore only if available empty space > source size

if err := e.Checker.RestoreSnapshotToPath(ctx, snapID, e.FileWriter.DataDirectory(ctx), b, opts); err != nil {
log.Print(b.String())

Expand Down Expand Up @@ -332,3 +346,142 @@

return snapIDList[rand.Intn(len(snapIDList))], nil //nolint:gosec
}

func (e *Engine) gatherStats() {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Collect metrics

log.Printf("Inside gatherStats")
// Gather evidence
// df -ah
app := "df"
arg0 := "-ah"
fmt.Println("df -ah")

Check failure on line 356 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

only cuddled expressions if assigning variable or using from line above (wsl)

Check failure on line 356 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (macos-latest) 

only cuddled expressions if assigning variable or using from line above (wsl)
cmd := exec.Command(app, arg0)

Check failure on line 357 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

assignments should only be cuddled with other assignments (wsl)

Check failure on line 357 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (macos-latest) 

assignments should only be cuddled with other assignments (wsl)
stdout, err := cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// temp dirs
// du -sh /tmp/
// /tmp/engine-data-*
app = "du"
arg0 = "-sh"
arg1 := "/tmp/"
fmt.Println("du -sh /tmp/engine-data-*")

Check failure on line 374 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

only cuddled expressions if assigning variable or using from line above (wsl)

Check failure on line 374 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (macos-latest) 

only cuddled expressions if assigning variable or using from line above (wsl)
cmd = exec.Command(app, arg0, arg1)

Check failure on line 375 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

assignments should only be cuddled with other assignments (wsl)

Check failure on line 375 in tests/robustness/engine/action.go

View workflow job for this annotation

GitHub Actions / Lint (macos-latest) 

assignments should only be cuddled with other assignments (wsl)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// filesystem
// du -sh /codefresh/volume/test-repo/robustness-metadata/
arg1 = "/codefresh/volume/test-repo/robustness-metadata/"
fmt.Println("du -sh ", arg1)
cmd = exec.Command(app, arg0, arg1)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// robustness-data
// du -sh /codefresh/volume/test-repo/robustness-data/
arg1 = "/codefresh/volume/test-repo/robustness-data/"
fmt.Println("du -sh ", arg1)
cmd = exec.Command(app, arg0, arg1)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// du -sh /codefresh/volume/fio-data-root/fio-data-*
arg1 = "/codefresh/volume/fio-data-root/"
fmt.Println("du -sh ", arg1)
cmd = exec.Command(app, arg0, arg1)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// du -sh /codefresh/volume/kopia/
arg1 = "/codefresh/volume/kopia/"
fmt.Println("du -sh ", arg1)
cmd = exec.Command(app, arg0, arg1)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// du -sh /root/.cache/kopia/cli-logs
arg1 = "/root/.cache/kopia/cli-logs"
fmt.Println("du -sh ", arg1)
cmd = exec.Command(app, arg0, arg1)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// du -sh /codefresh/volume/k10/
arg1 = "/codefresh/volume/k10/"
fmt.Println("du -sh ", arg1)
cmd = exec.Command(app, arg0, arg1)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))

// logs, cache
// du -sh /root/.cache/kopia/*/server-contents/
// the above one doesn't work
arg1 = "/root/.cache/kopia/"
fmt.Println("du -sh ", arg1)
cmd = exec.Command(app, arg0, arg1)
stdout, err = cmd.Output()

if err != nil {
fmt.Println(err.Error())
return
}

// Print the output
fmt.Println(string(stdout))
}
6 changes: 6 additions & 0 deletions tests/robustness/multiclient_test/framework/snapshotter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ func (mcs *MultiClientSnapshotter) ConnectOrCreateRepo(repoPath string) error {
return err
}

_, _, err1 := mcs.server.Run("cache", "info")

if err1 != nil {
return err1
}

_, _, err := mcs.server.Run("policy", "set", "--global", "--keep-latest", strconv.Itoa(1<<31-1), "--compression", "s2-default")

return err
Expand Down
24 changes: 24 additions & 0 deletions tests/robustness/multiclient_test/multiclient_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,24 @@ func TestManySmallFiles(t *testing.T) {

f := func(ctx context.Context, t *testing.T) { //nolint:thelper
err := tryRestoreIntoDataDirectory(ctx, t)
err2 := eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

_, err = eng.ExecAction(ctx, engine.WriteRandomFilesActionKey, fileWriteOpts)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

snapOut, err := eng.ExecAction(ctx, engine.SnapshotDirActionKey, nil)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

_, err = eng.ExecAction(ctx, engine.RestoreSnapshotActionKey, snapOut)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)
}

ctx := testlogging.ContextWithLevel(t, testlogging.LevelInfo)
Expand All @@ -75,16 +83,24 @@ func TestOneLargeFile(t *testing.T) {

f := func(ctx context.Context, t *testing.T) { //nolint:thelper
err := tryRestoreIntoDataDirectory(ctx, t)
err2 := eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

_, err = eng.ExecAction(ctx, engine.WriteRandomFilesActionKey, fileWriteOpts)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

snapOut, err := eng.ExecAction(ctx, engine.SnapshotDirActionKey, nil)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

_, err = eng.ExecAction(ctx, engine.RestoreSnapshotActionKey, snapOut)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)
}

ctx := testlogging.ContextWithLevel(t, testlogging.LevelInfo)
Expand Down Expand Up @@ -112,16 +128,24 @@ func TestManySmallFilesAcrossDirecoryTree(t *testing.T) {

f := func(ctx context.Context, t *testing.T) { //nolint:thelper
err := tryRestoreIntoDataDirectory(ctx, t)
err2 := eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

_, err = eng.ExecAction(ctx, engine.WriteRandomFilesActionKey, fileWriteOpts)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

snapOut, err := eng.ExecAction(ctx, engine.SnapshotDirActionKey, nil)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)

_, err = eng.ExecAction(ctx, engine.RestoreSnapshotActionKey, snapOut)
err2 = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
require.NoError(t, err2)
}

ctx := testlogging.ContextWithLevel(t, testlogging.LevelInfo)
Expand Down
2 changes: 2 additions & 0 deletions tests/robustness/robustness_test/robustness_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,14 @@ func TestOneLargeFile(t *testing.T) {
ctx := testlogging.ContextWithLevel(t, testlogging.LevelInfo)

_, err := eng.ExecAction(ctx, engine.WriteRandomFilesActionKey, fileWriteOpts)
err = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)

snapOut, err := eng.ExecAction(ctx, engine.SnapshotDirActionKey, nil)
require.NoError(t, err)

_, err = eng.ExecAction(ctx, engine.RestoreSnapshotActionKey, snapOut)
err = eng.CheckErrRecovery(ctx, err, engine.ActionOpts{})
require.NoError(t, err)
}

Expand Down
3 changes: 3 additions & 0 deletions tests/tools/kopiaclient/kopiaclient.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import (
"bytes"
"context"
"fmt"
"io"
"log"
"os"
Expand Down Expand Up @@ -192,6 +193,7 @@
SecretAccessKey: os.Getenv(awsSecretAccessKeyEnvKey),
}
st, err = s3.New(ctx, s3Opts, false)
fmt.Println("In getStorage: created new dir in S3 bucket.")

Check failure on line 196 in tests/tools/kopiaclient/kopiaclient.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

only cuddled expressions if assigning variable or using from line above (wsl)

Check failure on line 196 in tests/tools/kopiaclient/kopiaclient.go

View workflow job for this annotation

GitHub Actions / Lint (macos-latest) 

only cuddled expressions if assigning variable or using from line above (wsl)
} else {
if iErr := os.MkdirAll(repoDir, 0o700); iErr != nil {
return nil, errors.Wrap(iErr, "cannot create directory")
Expand All @@ -201,6 +203,7 @@
Path: repoDir,
}
st, err = filesystem.New(ctx, fsOpts, false)
fmt.Println("In getStorage: created new dir in local filesystem.")
}

return st, errors.Wrap(err, "unable to get storage")
Expand Down
4 changes: 2 additions & 2 deletions tests/tools/kopiarunner/kopia_snapshotter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ const (
serverControlPassword = "abcdef"

// Flag value settings.
contentCacheSizeSettingMB = 500
metadataCacheSizeSettingMB = 500
contentCacheSizeSettingMB = 50 // 500
metadataCacheSizeSettingMB = 50 // 500
parallelSetting = 8

aclEnabledMatchStr = "ACLs already enabled"
Expand Down
Loading