Skip to content

Commit 08b321c

Browse files
filkeithfilkeith
authored
FMWK-620 Add directory list support for restore (#188)
1 parent 4f493ba commit 08b321c

26 files changed

+1086
-303
lines changed

‎cmd/asrestore/readme.md

Lines changed: 39 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -67,39 +67,45 @@ Restore Flags:
6767
--socket-timeout int Socket timeout in milliseconds. If this value is 0, it's set to total-timeout. If both are 0,
6868
there is no socket idle time limit (default 10000)
6969
-N, --nice int The limits for read/write storage bandwidth in MiB/s
70-
-i, --input-file string Restore from a single backup file. Use - for stdin.
71-
Required, unless --directory or --directory-list is used.
72-
73-
-u, --unique Skip records that already exist in the namespace;
74-
Don't touch them.
75-
76-
-r, --replace Fully replace records that already exist in the namespace;
77-
Don't update them.
78-
79-
-g, --no-generation Don't check the generation of records that already exist in the namespace.
80-
--ignore-record-error Ignore permanent record specific error. e.g AEROSPIKE_RECORD_TOO_BIG.
81-
By default such errors are not ignored and asrestore terminates.
82-
Optional: Use verbose mode to see errors in detail.
83-
--disable-batch-writes Disables the use of batch writes when restoring records to the Aerospike cluster.
84-
By default, the cluster is checked for batch write support, so only set this flag if you explicitly
85-
don't want
86-
batch writes to be used or asrestore is failing to recognize that batch writes are disabled
87-
and is failing to work because of it.
88-
--max-async-batches int The max number of outstanding async record batch write calls at a time.
89-
For pre-6.0 servers, 'batches' are only a logical grouping of
90-
records, and each record is uploaded individually. The true max
91-
number of async aerospike calls would then be
92-
<max-async-batches> * <batch-size>. (default 32)
93-
--batch-size int The max allowed number of records to simultaneously upload
94-
in an async batch write calls to make to aerospike at a time.
95-
Default is 128 with batch writes enabled, or 16 without batch writes. (default 128)
96-
--extra-ttl int For records with expirable void-times, add N seconds of extra-ttl to the
97-
recorded void-time.
98-
-T, --timeout int Set the timeout (ms) for info commands. (default 10000)
99-
--retry-base-timeout int Set the initial delay between retry attempts in milliseconds (default 1000)
100-
--retry-multiplier float retry-multiplier is used to increase the delay between subsequent retry attempts.
101-
The actual delay is calculated as: retry-base-timeout * (retry-multiplier ^ attemptNumber) (default 1)
102-
--retry-max-retries uint Set the maximum number of retry attempts that will be made. If set to 0, no retries will be performed.
70+
-i, --input-file string Restore from a single backup file. Use - for stdin.
71+
Required, unless --directory or --directory-list is used.
72+
73+
--directory-list string A comma separated list of paths to directories that hold the backup files. Required,
74+
unless -i or -d is used. The paths may not contain commas
75+
Example: `asrestore --directory-list /path/to/dir1/,/path/to/dir2
76+
--parent-directory string A common root path for all paths used in --directory-list.
77+
This path is prepended to all entries in --directory-list.
78+
Example: `asrestore --parent-directory /common/root/path --directory-list /path/to/dir1/,/path/to/dir2
79+
-u, --unique Skip records that already exist in the namespace;
80+
Don't touch them.
81+
82+
-r, --replace Fully replace records that already exist in the namespace;
83+
Don't update them.
84+
85+
-g, --no-generation Don't check the generation of records that already exist in the namespace.
86+
--ignore-record-error Ignore permanent record specific error. e.g AEROSPIKE_RECORD_TOO_BIG.
87+
By default such errors are not ignored and asrestore terminates.
88+
Optional: Use verbose mode to see errors in detail.
89+
--disable-batch-writes Disables the use of batch writes when restoring records to the Aerospike cluster.
90+
By default, the cluster is checked for batch write support, so only set this flag if you explicitly
91+
don't want
92+
batch writes to be used or asrestore is failing to recognize that batch writes are disabled
93+
and is failing to work because of it.
94+
--max-async-batches int The max number of outstanding async record batch write calls at a time.
95+
For pre-6.0 servers, 'batches' are only a logical grouping of
96+
records, and each record is uploaded individually. The true max
97+
number of async aerospike calls would then be
98+
<max-async-batches> * <batch-size>. (default 32)
99+
--batch-size int The max allowed number of records to simultaneously upload
100+
in an async batch write calls to make to aerospike at a time.
101+
Default is 128 with batch writes enabled, or 16 without batch writes. (default 128)
102+
--extra-ttl int For records with expirable void-times, add N seconds of extra-ttl to the
103+
recorded void-time.
104+
-T, --timeout int Set the timeout (ms) for info commands. (default 10000)
105+
--retry-base-timeout int Set the initial delay between retry attempts in milliseconds (default 1000)
106+
--retry-multiplier float retry-multiplier is used to increase the delay between subsequent retry attempts.
107+
The actual delay is calculated as: retry-base-timeout * (retry-multiplier ^ attemptNumber) (default 1)
108+
--retry-max-retries uint Set the maximum number of retry attempts that will be made. If set to 0, no retries will be performed.
103109
104110
Compression Flags:
105111
-z, --compress string Enables decompressing of backup files using the specified compression algorithm.
@@ -167,13 +173,6 @@ Any Azure parameter can be retrieved from secret agent.
167173

168174
## Unsupported flags
169175
```
170-
--directory-list A comma seperated list of paths to directories that hold the backup files. Required,
171-
unless -i or -d is used. The paths may not contain commas
172-
Example: `asrestore --directory-list /path/to/dir1/,/path/to/dir2
173-
174-
--parent-directory A common root path for all paths used in --directory-list.
175-
This path is prepended to all entries in --directory-list.
176-
Example: `asrestore --parent-directory /common/root/path --directory-list /path/to/dir1/,/path/to/dir2
177176
178177
-m, --machine <path> Output machine-readable status updates to the given path,
179178
typically a FIFO.

‎cmd/internal/app/asrestore.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ func NewASRestore(
5454
return nil, err
5555
}
5656

57+
if err := validateRestoreParams(restoreParams, commonParams); err != nil {
58+
return nil, err
59+
}
60+
5761
restoreConfig, err := mapRestoreConfig(
5862
restoreParams,
5963
commonParams,

‎cmd/internal/app/readers.go

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package app
1717
import (
1818
"context"
1919
"fmt"
20+
"path"
2021

2122
"github.com/aerospike/backup-go"
2223
"github.com/aerospike/backup-go/cmd/internal/models"
@@ -62,24 +63,28 @@ func getReader(
6263
}
6364

6465
func newLocalReader(r *models.Restore, c *models.Common, b *models.Backup) (backup.StreamingReader, error) {
65-
var opts []local.Opt
66+
opts := make([]local.Opt, 0)
6667

67-
if c.Directory != "" && r.InputFile == "" {
68+
// As we validate this fields in validation function, we can switch here.
69+
switch {
70+
case c.Directory != "":
6871
opts = append(opts, local.WithDir(c.Directory))
6972
// Append Validator only if backup params are not set.
7073
// That means we don't need to check that we are saving a state file.
7174
if b == nil {
7275
opts = append(opts, local.WithValidator(asb.NewValidator()))
7376
}
74-
}
75-
76-
if r.InputFile != "" && c.Directory == "" {
77+
case r.InputFile != "":
7778
opts = append(opts, local.WithFile(r.InputFile))
79+
case r.DirectoryList != "":
80+
dirList := prepareDirectoryList(r.ParentDirectory, r.DirectoryList)
81+
opts = append(opts, local.WithDirList(dirList))
7882
}
7983

8084
return local.NewReader(opts...)
8185
}
8286

87+
//nolint:dupl // This code is not duplicated, it is a different initialization.
8388
func newS3Reader(
8489
ctx context.Context,
8590
a *models.AwsS3,
@@ -94,22 +99,26 @@ func newS3Reader(
9499

95100
opts := make([]s3.Opt, 0)
96101

97-
if c.Directory != "" && r.InputFile == "" {
102+
// As we validate this fields in validation function, we can switch here.
103+
switch {
104+
case c.Directory != "":
98105
opts = append(opts, s3.WithDir(c.Directory))
99106
// Append Validator only if backup params are not set.
100107
// That means we don't need to check that we are saving a state file.
101108
if b == nil {
102109
opts = append(opts, s3.WithValidator(asb.NewValidator()))
103110
}
104-
}
105-
106-
if r.InputFile != "" && c.Directory == "" {
111+
case r.InputFile != "":
107112
opts = append(opts, s3.WithFile(r.InputFile))
113+
case r.DirectoryList != "":
114+
dirList := prepareDirectoryList(r.ParentDirectory, r.DirectoryList)
115+
opts = append(opts, s3.WithDirList(dirList))
108116
}
109117

110118
return s3.NewReader(ctx, client, a.BucketName, opts...)
111119
}
112120

121+
//nolint:dupl // This code is not duplicated, it is a different initialization.
113122
func newGcpReader(
114123
ctx context.Context,
115124
g *models.GcpStorage,
@@ -124,17 +133,20 @@ func newGcpReader(
124133

125134
opts := make([]storage.Opt, 0)
126135

127-
if c.Directory != "" && r.InputFile == "" {
136+
// As we validate this fields in validation function, we can switch here.
137+
switch {
138+
case c.Directory != "":
128139
opts = append(opts, storage.WithDir(c.Directory))
129140
// Append Validator only if backup params are not set.
130141
// That means we don't need to check that we are saving a state file.
131142
if b == nil {
132143
opts = append(opts, storage.WithValidator(asb.NewValidator()))
133144
}
134-
}
135-
136-
if r.InputFile != "" && c.Directory == "" {
145+
case r.InputFile != "":
137146
opts = append(opts, storage.WithFile(r.InputFile))
147+
case r.DirectoryList != "":
148+
dirList := prepareDirectoryList(r.ParentDirectory, r.DirectoryList)
149+
opts = append(opts, storage.WithDirList(dirList))
138150
}
139151

140152
return storage.NewReader(ctx, client, g.BucketName, opts...)
@@ -154,18 +166,33 @@ func newAzureReader(
154166

155167
opts := make([]blob.Opt, 0)
156168

157-
if c.Directory != "" && r.InputFile == "" {
169+
// As we validate this fields in validation function, we can switch here.
170+
switch {
171+
case c.Directory != "":
158172
opts = append(opts, blob.WithDir(c.Directory))
159173
// Append Validator only if backup params are not set.
160174
// That means we don't need to check that we are saving a state file.
161175
if b == nil {
162176
opts = append(opts, blob.WithValidator(asb.NewValidator()))
163177
}
164-
}
165-
166-
if r.InputFile != "" && c.Directory == "" {
178+
case r.InputFile != "":
167179
opts = append(opts, blob.WithFile(r.InputFile))
180+
case r.DirectoryList != "":
181+
dirList := prepareDirectoryList(r.ParentDirectory, r.DirectoryList)
182+
opts = append(opts, blob.WithDirList(dirList))
168183
}
169184

170185
return blob.NewReader(ctx, client, a.ContainerName, opts...)
171186
}
187+
188+
// prepareDirectoryList parses command line parameters and return slice of strings.
189+
func prepareDirectoryList(parentDir, dirList string) []string {
190+
result := splitByComma(dirList)
191+
if parentDir != "" {
192+
for i := range result {
193+
result[i] = path.Join(parentDir, result[i])
194+
}
195+
}
196+
197+
return result
198+
}

‎cmd/internal/app/readers_test.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package app
1616

1717
import (
1818
"context"
19+
"path"
1920
"testing"
2021

2122
"github.com/aerospike/backup-go/cmd/internal/models"
@@ -158,3 +159,98 @@ func TestNewAzureReader(t *testing.T) {
158159
assert.NotNil(t, writer)
159160
assert.Equal(t, testAzureType, writer.GetType())
160161
}
162+
163+
func TestPrepareDirectoryList(t *testing.T) {
164+
tests := []struct {
165+
name string
166+
parentDir string
167+
dirList string
168+
expected []string
169+
}{
170+
{
171+
name: "Empty input",
172+
parentDir: "",
173+
dirList: "",
174+
expected: nil,
175+
},
176+
{
177+
name: "Single directory without parentDir",
178+
parentDir: "",
179+
dirList: "dir1",
180+
expected: []string{"dir1"},
181+
},
182+
{
183+
name: "Multiple directories without parentDir",
184+
parentDir: "",
185+
dirList: "dir1,dir2,dir3",
186+
expected: []string{"dir1", "dir2", "dir3"},
187+
},
188+
{
189+
name: "Single directory with parentDir",
190+
parentDir: "parent",
191+
dirList: "dir1",
192+
expected: []string{path.Join("parent", "dir1")},
193+
},
194+
{
195+
name: "Multiple directories with parentDir",
196+
parentDir: "parent",
197+
dirList: "dir1,dir2,dir3",
198+
expected: []string{
199+
path.Join("parent", "dir1"),
200+
path.Join("parent", "dir2"),
201+
path.Join("parent", "dir3"),
202+
},
203+
},
204+
{
205+
name: "Trailing commas in dirList",
206+
parentDir: "parent",
207+
dirList: "dir1,dir2,",
208+
expected: []string{
209+
path.Join("parent", "dir1"),
210+
path.Join("parent", "dir2"),
211+
path.Join("parent", ""),
212+
},
213+
},
214+
{
215+
name: "Whitespace in dirList",
216+
parentDir: "parent",
217+
dirList: " dir1 , dir2 ,dir3 ",
218+
expected: []string{
219+
path.Join("parent", " dir1 "),
220+
path.Join("parent", " dir2 "),
221+
path.Join("parent", "dir3 "),
222+
},
223+
},
224+
{
225+
name: "ParentDir is empty but dirList has valid directories",
226+
parentDir: "",
227+
dirList: "dir1,dir2",
228+
expected: []string{"dir1", "dir2"},
229+
},
230+
{
231+
name: "ParentDir has trailing slash",
232+
parentDir: "parent/",
233+
dirList: "dir1,dir2",
234+
expected: []string{
235+
path.Join("parent/", "dir1"),
236+
path.Join("parent/", "dir2"),
237+
},
238+
},
239+
{
240+
name: "ParentDir with absolute path",
241+
parentDir: "/absolute/path",
242+
dirList: "dir1,dir2",
243+
expected: []string{
244+
path.Join("/absolute/path", "dir1"),
245+
path.Join("/absolute/path", "dir2"),
246+
},
247+
},
248+
}
249+
250+
for _, tt := range tests {
251+
t.Run(tt.name, func(t *testing.T) {
252+
actual := prepareDirectoryList(tt.parentDir, tt.dirList)
253+
assert.Equal(t, tt.expected, actual)
254+
})
255+
}
256+
}

‎cmd/internal/app/validation.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ func validateStorages(
5252

5353
//nolint:gocyclo // It is a long validation function.
5454
func validateBackupParams(backupParams *models.Backup, commonParams *models.Common) error {
55+
if commonParams.Directory != "" && backupParams.OutputFile != "" {
56+
return fmt.Errorf("only one of output-file and directory may be configured at the same time")
57+
}
58+
5559
// Only one filter is allowed.
5660
if backupParams.AfterDigest != "" && backupParams.PartitionList != "" {
5761
return fmt.Errorf("only one of after-digest or partition-list can be configured")
@@ -143,3 +147,19 @@ func validatePartitionFilters(partitionFilters []*aerospike.PartitionFilter) err
143147

144148
return nil
145149
}
150+
151+
func validateRestoreParams(restoreParams *models.Restore, commonParams *models.Common) error {
152+
if commonParams.Directory != "" && restoreParams.InputFile != "" {
153+
return fmt.Errorf("only one of directory and input-file may be configured at the same time")
154+
}
155+
156+
if restoreParams.DirectoryList != "" && (commonParams.Directory != "" || restoreParams.InputFile != "") {
157+
return fmt.Errorf("only one of directory, input-file and directory-list may be configured at the same time")
158+
}
159+
160+
if restoreParams.ParentDirectory != "" && restoreParams.DirectoryList == "" {
161+
return fmt.Errorf("must specify directory-list list")
162+
}
163+
164+
return nil
165+
}

0 commit comments

Comments
 (0)