From fc7adee5d18954669e945e57ec6493ae083e1c24 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Thu, 21 Dec 2023 23:01:58 +0530 Subject: [PATCH 01/12] wait for repair and increase numBlocks (#1338) --- wasmsdk/allocation.go | 4 ++++ zboxcore/sdk/allocation.go | 2 -- zboxcore/sdk/sdk.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/wasmsdk/allocation.go b/wasmsdk/allocation.go index 3812cf199..44b9500c0 100644 --- a/wasmsdk/allocation.go +++ b/wasmsdk/allocation.go @@ -194,6 +194,10 @@ func updateAllocationWithRepair(allocationID string, hash, err := allocationObj.UpdateWithRepair(size, extend, uint64(lock), addBlobberId, removeBlobberId, false, &sdk.FileOptionsParameters{}, statusBar) if err == nil { clearAllocation(allocationID) + wg.Wait() + if statusBar.err != nil { + return hash, statusBar.err + } } return hash, err diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 922350615..ea68b4304 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -388,7 +388,6 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string, if ref.EncryptedKey != "" { opts = []ChunkedUploadOption{ WithMask(mask), - WithChunkNumber(10), WithStatusCallback(status), WithEncrypt(true), WithEncryptedPoint(ref.EncryptedKeyPoint), @@ -396,7 +395,6 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string, } else { opts = []ChunkedUploadOption{ WithMask(mask), - WithChunkNumber(10), WithStatusCallback(status), } } diff --git a/zboxcore/sdk/sdk.go b/zboxcore/sdk/sdk.go index 04601f146..414a95e5e 100644 --- a/zboxcore/sdk/sdk.go +++ b/zboxcore/sdk/sdk.go @@ -53,7 +53,7 @@ type StatusCallback interface { RepairCompleted(filesRepaired int) } -var numBlockDownloads = 10 +var numBlockDownloads = 100 var sdkInitialized = false var networkWorkerTimerInHours = 1 var shouldVerifyHash = true From 5697527646f720827d76bc3a179c52d30639a903 Mon Sep 17 00:00:00 2001 From: Ebrahim Gomaa Date: Fri, 22 Dec 2023 07:55:16 +0200 Subject: [PATCH 02/12] reorder wait group done (#1340) --- core/transaction/entity.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/transaction/entity.go b/core/transaction/entity.go index 6c06356b0..70782c439 100644 --- a/core/transaction/entity.go +++ b/core/transaction/entity.go @@ -267,6 +267,7 @@ func SendTransactionSync(txn *Transaction, miners []string) error { if err != nil { fails <- err } + wg.Done() }() //nolint } wg.Wait() @@ -297,9 +298,6 @@ func SendTransactionSync(txn *Transaction, miners []string) error { } func sendTransactionToURL(url string, txn *Transaction, wg *sync.WaitGroup) ([]byte, error) { - if wg != nil { - defer wg.Done() - } postReq, err := util.NewHTTPPostRequest(url, txn) if err != nil { //Logger.Error("Error in serializing the transaction", txn, err.Error()) From c4073cde471331cebc6f45c52fe9244ed93fd661 Mon Sep 17 00:00:00 2001 From: Ebrahim Gomaa Date: Mon, 25 Dec 2023 22:03:43 +0200 Subject: [PATCH 03/12] hotfix / remove hard coded prefix handling of encrypted upload (#1344) * remove hard coded prefix handling of encrypted upload * fix for other parts of the file --- wasmsdk/blobber.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index cd9d59338..53fcab0b3 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -589,9 +589,7 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) { wg.Add(1) encrypt := option.Encrypt remotePath := option.RemotePath - if strings.HasPrefix(remotePath, "/Encrypted") { - encrypt = true - } + fileReader := jsbridge.NewFileReader(option.ReadChunkFuncName, option.FileSize) mimeType, err := zboxutil.GetFileContentType(fileReader) if err != nil { @@ -678,9 +676,7 @@ func uploadWithJsFuncs(allocationID, remotePath string, readChunkFuncName string } } wg.Add(1) - if strings.HasPrefix(remotePath, "/Encrypted") { - encrypt = true - } + fileReader := jsbridge.NewFileReader(readChunkFuncName, fileSize) @@ -760,9 +756,6 @@ func upload(allocationID, remotePath string, fileBytes, thumbnailBytes []byte, w wg := &sync.WaitGroup{} statusBar := &StatusBar{wg: wg} wg.Add(1) - if strings.HasPrefix(remotePath, "/Encrypted") { - encrypt = true - } fileReader := bytes.NewReader(fileBytes) From ea5f217dd0eeeda83fa4b584f1b616278cb79541 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Sun, 31 Dec 2023 21:37:44 +0530 Subject: [PATCH 04/12] increase batch size (#1349) --- zboxcore/sdk/allocation.go | 3 +-- zboxcore/sdk/chunked_upload.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index ea68b4304..1e833b8e1 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -42,7 +42,7 @@ var ( noBLOBBERS = errors.New("", "No Blobbers set in this allocation") notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object") IsWasm = false - MultiOpBatchSize = 10 + MultiOpBatchSize = 50 ) const ( @@ -242,7 +242,6 @@ func GetWritePriceRange() (PriceRange, error) { func SetWasm() { IsWasm = true BatchSize = 5 - MultiOpBatchSize = 7 } func getPriceRange(name string) (PriceRange, error) { diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index e6846cc75..223522def 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -752,7 +752,7 @@ func (su *ChunkedUpload) uploadProcessor() { su.consensus.Reset() var pos uint64 var errCount int32 - swg := sizedwaitgroup.New(BatchSize) + swg := sizedwaitgroup.New(BatchSize * 2) for i := su.uploadMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) swg.Add() From d5aa3ea03fae0a1959bf8296f9e5f0e8fd98dc64 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Fri, 5 Jan 2024 02:16:40 +0530 Subject: [PATCH 05/12] Sprint changes (#1355) * Update sprint 1.12 (#1341) * wait for repair and increase numBlocks (#1338) * reorder wait group done (#1340) --------- Co-authored-by: Ebrahim Gomaa * fix trailing whitespace (#1343) * Merge staging changes (#1346) * wait for repair and increase numBlocks (#1338) * reorder wait group done (#1340) * hotfix / remove hard coded prefix handling of encrypted upload (#1344) * remove hard coded prefix handling of encrypted upload * fix for other parts of the file --------- Co-authored-by: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Co-authored-by: Ebrahim Gomaa * fix upload select (#1351) * fix workdir in mobile sdk (#1345) * fix mobile workdir * set multi op batch size * set option to download to disk in wasm (#1348) * fix panic in hash chan (#1352) * Fix merge conflict in sprint-1.12 (#1354) * wait for repair and increase numBlocks (#1338) * reorder wait group done (#1340) * hotfix / remove hard coded prefix handling of encrypted upload (#1344) * remove hard coded prefix handling of encrypted upload * fix for other parts of the file * increase batch size (#1349) --------- Co-authored-by: Ebrahim Gomaa * repair in batches (#1347) * repair in batches * fix lint * fix unit test * fix batch size --------- Co-authored-by: Yury --------- Co-authored-by: Ebrahim Gomaa Co-authored-by: peterlimg <54137706+peterlimg@users.noreply.github.com> Co-authored-by: Yury --- core/version/version.go | 2 +- mobilesdk/zbox/storage.go | 12 +++ wasmsdk/blobber.go | 18 +++- wasmsdk/demo/index.html | 5 +- wasmsdk/jsbridge/file_writer.go | 88 +++++++++++++++ zboxcore/sdk/allocation.go | 62 +++++++---- zboxcore/sdk/allocation_file_test.go | 91 +++++++++------- zboxcore/sdk/allocation_test.go | 1 + zboxcore/sdk/chunked_upload.go | 6 +- zboxcore/sdk/commitworker.go | 11 +- zboxcore/sdk/downloadworker.go | 29 +++-- zboxcore/sdk/multi_operation_worker.go | 71 ++++++++----- zboxcore/sdk/repairworker.go | 142 +++++++++++++------------ zboxcore/sdk/upload_worker.go | 17 ++- 14 files changed, 375 insertions(+), 180 deletions(-) create mode 100644 wasmsdk/jsbridge/file_writer.go diff --git a/core/version/version.go b/core/version/version.go index 6655d402c..0c9ca3d6c 100644 --- a/core/version/version.go +++ b/core/version/version.go @@ -2,5 +2,5 @@ //====== THIS IS AUTOGENERATED FILE. DO NOT MODIFY ======== package version -const VERSIONSTR = "v1.10.0-141-gf81a4b84" +const VERSIONSTR = "v1.11-3-gad762e49" diff --git a/mobilesdk/zbox/storage.go b/mobilesdk/zbox/storage.go index e0b1bb855..37136652e 100644 --- a/mobilesdk/zbox/storage.go +++ b/mobilesdk/zbox/storage.go @@ -850,3 +850,15 @@ func GetRemoteFileMap(allocationID string) (string, error) { return string(retBytes), nil } + +// SetWorkingDir set working dir +// +// ## Inputs +// - workDir +func SetWorkingDir(workDir string) { + sdk.Workdir = workDir +} + +func SetMultiOpBatchSize(size int) { + sdk.SetMultiOpBatchSize(size) +} diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index 53fcab0b3..f6dbc87df 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -368,7 +368,7 @@ func multiDownload(allocationID, jsonMultiDownloadOptions, authTicket, callbackF fileName := strings.Replace(path.Base(option.RemotePath), "/", "-", -1) localPath := allocationID + "_" + fileName option.LocalPath = localPath - statusBar := &StatusBar{wg: wg, localPath: localPath} + statusBar := &StatusBar{wg: wg} allStatusBar[ind] = statusBar if useCallback { callback := js.Global().Get(callbackFuncName) @@ -376,8 +376,18 @@ func multiDownload(allocationID, jsonMultiDownloadOptions, authTicket, callbackF callback.Invoke(totalBytes, completedBytes, filename, objURL, err) } } - fs, _ := sys.Files.Open(localPath) - mf, _ := fs.(*sys.MemFile) + var mf sys.File + if option.DownloadToDisk { + mf, err = jsbridge.NewFileWriter(fileName) + if err != nil { + PrintError(err.Error()) + return "", err + } + } else { + statusBar.localPath = localPath + fs, _ := sys.Files.Open(localPath) + mf, _ = fs.(*sys.MemFile) + } var downloader sdk.Downloader if option.DownloadOp == 1 { @@ -467,6 +477,7 @@ type MultiDownloadOption struct { NumBlocks int `json:"numBlocks"` RemoteFileName string `json:"remoteFileName"` //Required only for file download with auth ticket RemoteLookupHash string `json:"remoteLookupHash,omitempty"` //Required only for file download with auth ticket + DownloadToDisk bool `json:"downloadToDisk"` } // MultiOperation - do copy, move, delete and createdir operation together @@ -677,7 +688,6 @@ func uploadWithJsFuncs(allocationID, remotePath string, readChunkFuncName string } wg.Add(1) - fileReader := jsbridge.NewFileReader(readChunkFuncName, fileSize) mimeType, err := zboxutil.GetFileContentType(fileReader) diff --git a/wasmsdk/demo/index.html b/wasmsdk/demo/index.html index d7eee1b5d..c7ece4f46 100644 --- a/wasmsdk/demo/index.html +++ b/wasmsdk/demo/index.html @@ -3,7 +3,7 @@ - + @@ -211,7 +211,7 @@

please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates let network = query.get('network') if (!network || network == 'undefined') { - network = "dev.zus.network" + network = "test.zus.network" } const blockWorker = 'https://' + network + '/dns'; @@ -802,6 +802,7 @@

please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates remotePath: path, downloadOp: 1, numBlocks: 0, + downloadToDisk: true, }) let stringifiedArray = JSON.stringify(objects); diff --git a/wasmsdk/jsbridge/file_writer.go b/wasmsdk/jsbridge/file_writer.go new file mode 100644 index 000000000..9a2ef955c --- /dev/null +++ b/wasmsdk/jsbridge/file_writer.go @@ -0,0 +1,88 @@ +//go:build js && wasm +// +build js,wasm + +package jsbridge + +import ( + "errors" + "io/fs" + "sync" + "syscall/js" +) + +var jsFileWriterMutex sync.Mutex + +type FileWriter struct { + writableStream js.Value +} + +func (w *FileWriter) Write(p []byte) (int, error) { + //js.Value doesn't work in parallel invoke + jsFileWriterMutex.Lock() + defer jsFileWriterMutex.Unlock() + + uint8Array := js.Global().Get("Uint8Array").New(len(p)) + js.CopyBytesToJS(uint8Array, p) + _, err := Await(w.writableStream.Call("write", uint8Array)) + if len(err) > 0 && !err[0].IsNull() { + return 0, errors.New("file_writer: " + err[0].String()) + } + return len(p), nil +} + +func (w *FileWriter) Close() error { + _, err := Await(w.writableStream.Call("close")) + if len(err) > 0 && !err[0].IsNull() { + return errors.New("file_writer: " + err[0].String()) + } + return nil +} + +func (w *FileWriter) Read(p []byte) (int, error) { + return 0, errors.New("file_writer: not supported") +} + +func (w *FileWriter) Seek(offset int64, whence int) (int64, error) { + return 0, nil +} + +func (w *FileWriter) Sync() error { + return nil +} + +func (w *FileWriter) Stat() (fs.FileInfo, error) { + return nil, nil +} + +func NewFileWriter(filename string) (*FileWriter, error) { + // writableStream := js.Global().Get("writableStream") + // stream, err := Await(writableStream.Call("create", filename)) + // if len(err) > 0 && !err[0].IsNull() { + // return nil, errors.New("file_writer: " + err[0].String()) + // } + // return &FileWriter{ + // writableStream: stream[0], + // }, nil + if !js.Global().Get("window").Get("showSaveFilePicker").Truthy() || !js.Global().Get("window").Get("WritableStream").Truthy() { + return nil, errors.New("file_writer: not supported") + } + + showSaveFilePicker := js.Global().Get("window").Get("showSaveFilePicker") + //create options with suggested name + options := js.Global().Get("Object").New() + options.Set("suggestedName", filename) + + //request a file handle + fileHandle, err := Await(showSaveFilePicker.Invoke(options)) + if len(err) > 0 && !err[0].IsNull() { + return nil, errors.New("file_writer: " + err[0].String()) + } + //create a writable stream + writableStream, err := Await(fileHandle[0].Call("createWritable")) + if len(err) > 0 && !err[0].IsNull() { + return nil, errors.New("file_writer: " + err[0].String()) + } + return &FileWriter{ + writableStream: writableStream[0], + }, nil +} diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 1e833b8e1..35ef7377c 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -43,6 +43,7 @@ var ( notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object") IsWasm = false MultiOpBatchSize = 50 + Workdir string ) const ( @@ -223,13 +224,16 @@ type OperationRequest struct { DestName string // Required only for rename operation DestPath string // Required for copy and move operation IsUpdate bool + IsRepair bool // Required for repair operation IsWebstreaming bool // Required for uploads - Workdir string - FileMeta FileMeta - FileReader io.Reader - Opts []ChunkedUploadOption + Workdir string + FileMeta FileMeta + FileReader io.Reader + Mask *zboxutil.Uint128 // Required for delete repair operation + DownloadFile bool // Required for upload repair operation + Opts []ChunkedUploadOption } func GetReadPriceRange() (PriceRange, error) { @@ -239,6 +243,10 @@ func GetWritePriceRange() (PriceRange, error) { return getPriceRange("max_write_price") } +func SetMultiOpBatchSize(size int) { + MultiOpBatchSize = size +} + func SetWasm() { IsWasm = true BatchSize = 5 @@ -372,10 +380,11 @@ func (a *Allocation) UploadFile(workdir, localpath string, remotepath string, return a.StartChunkedUpload(workdir, localpath, remotepath, status, false, false, "", false, false) } -func (a *Allocation) RepairFile(file sys.File, remotepath string, - status StatusCallback, mask zboxutil.Uint128, ref *fileref.FileRef) error { - +func (a *Allocation) RepairFile(file sys.File, remotepath string, statusCallback StatusCallback, mask zboxutil.Uint128, ref *fileref.FileRef) *OperationRequest { idr, _ := homedir.Dir() + if Workdir != "" { + idr = Workdir + } mask = mask.Not().And(zboxutil.NewUint128(1).Lsh(uint64(len(a.Blobbers))).Sub64(1)) fileMeta := FileMeta{ ActualSize: ref.ActualFileSize, @@ -387,22 +396,26 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string, if ref.EncryptedKey != "" { opts = []ChunkedUploadOption{ WithMask(mask), - WithStatusCallback(status), WithEncrypt(true), + WithStatusCallback(statusCallback), WithEncryptedPoint(ref.EncryptedKeyPoint), } } else { opts = []ChunkedUploadOption{ WithMask(mask), - WithStatusCallback(status), + WithStatusCallback(statusCallback), } } - connectionID := zboxutil.NewConnectionId() - chunkedUpload, err := CreateChunkedUpload(idr, a, fileMeta, file, false, true, false, connectionID, opts...) - if err != nil { - return err + op := &OperationRequest{ + OperationType: constants.FileOperationInsert, + IsRepair: true, + RemotePath: remotepath, + Workdir: idr, + FileMeta: fileMeta, + Opts: opts, + FileReader: file, } - return chunkedUpload.Start() + return op } // UpdateFileWithThumbnail [Deprecated]please use CreateChunkedUpload @@ -787,7 +800,7 @@ func (a *Allocation) RepairRequired(remotepath string) (zboxutil.Uint128, zboxut return found, deleteMask, !found.Equals(uploadMask), fileRef, nil } -func (a *Allocation) DoMultiOperation(operations []OperationRequest) error { +func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...MultiOperationOption) error { if len(operations) == 0 { return nil } @@ -808,6 +821,9 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest) error { consensusThresh: a.consensusThreshold, fullconsensus: a.fullconsensus, } + for _, opt := range opts { + opt(&mo) + } previousPaths := make(map[string]bool) connectionErrors := make([]error, len(mo.allocationObj.Blobbers)) @@ -844,6 +860,11 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest) error { break } op := operations[i] + op.RemotePath = strings.TrimSpace(op.RemotePath) + if op.FileMeta.RemotePath != "" { + op.FileMeta.RemotePath = strings.TrimSpace(op.FileMeta.RemotePath) + op.FileMeta.RemoteName = strings.TrimSpace(op.FileMeta.RemoteName) + } remotePath := op.RemotePath parentPaths := GenerateParentPaths(remotePath) @@ -870,13 +891,16 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest) error { operation = NewMoveOperation(op.RemotePath, op.DestPath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) case constants.FileOperationInsert: - operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, false, op.IsWebstreaming, op.Opts...) + operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, false, op.IsWebstreaming, op.IsRepair, op.DownloadFile, op.Opts...) case constants.FileOperationDelete: - operation = NewDeleteOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) - + if op.Mask != nil { + operation = NewDeleteOperation(op.RemotePath, *op.Mask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) + } else { + operation = NewDeleteOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) + } case constants.FileOperationUpdate: - operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, true, op.IsWebstreaming, op.Opts...) + operation, newConnectionID, err = NewUploadOperation(op.Workdir, mo.allocationObj, mo.connectionID, op.FileMeta, op.FileReader, true, op.IsWebstreaming, op.IsRepair, op.DownloadFile, op.Opts...) case constants.FileOperationCreateDir: operation = NewDirOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) diff --git a/zboxcore/sdk/allocation_file_test.go b/zboxcore/sdk/allocation_file_test.go index fd72925f6..91ff49239 100644 --- a/zboxcore/sdk/allocation_file_test.go +++ b/zboxcore/sdk/allocation_file_test.go @@ -647,32 +647,32 @@ func TestAllocation_RepairFile(t *testing.T) { ClientKey: mockClientKey, } - setupHttpResponses := func(t *testing.T, testName string, numBlobbers, numCorrect int) { - require.True(t, numBlobbers >= numCorrect) - for i := 0; i < numBlobbers; i++ { - var hash string - if i < numCorrect { - hash = mockActualHash - } - frName := mockFileRefName + strconv.Itoa(i) - url := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), url) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: func(fileRefName, hash string) io.ReadCloser { - jsonFR, err := json.Marshal(&fileref.FileRef{ - ActualFileHash: hash, - Ref: fileref.Ref{ - Name: fileRefName, - }, - }) - require.NoError(t, err) - return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) - }(frName, hash), - }, nil) - } - } + // setupHttpResponses := func(t *testing.T, testName string, numBlobbers, numCorrect int) { + // require.True(t, numBlobbers >= numCorrect) + // for i := 0; i < numBlobbers; i++ { + // var hash string + // if i < numCorrect { + // hash = mockActualHash + // } + // frName := mockFileRefName + strconv.Itoa(i) + // url := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" + // mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { + // return strings.HasPrefix(req.URL.String(), url) + // })).Return(&http.Response{ + // StatusCode: http.StatusOK, + // Body: func(fileRefName, hash string) io.ReadCloser { + // jsonFR, err := json.Marshal(&fileref.FileRef{ + // ActualFileHash: hash, + // Ref: fileref.Ref{ + // Name: fileRefName, + // }, + // }) + // require.NoError(t, err) + // return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) + // }(frName, hash), + // }, nil) + // } + // } setupHttpResponsesWithUpload := func(t *testing.T, testName string, numBlobbers, numCorrect int) { require.True(t, numBlobbers >= numCorrect) @@ -783,6 +783,18 @@ func TestAllocation_RepairFile(t *testing.T) { return ioutil.NopCloser(bytes.NewReader(respBuf)) }(), }, nil) + + urlCreateConnection := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + zboxutil.CREATE_CONNECTION_ENDPOINT + urlCreateConnection = strings.TrimRight(urlCreateConnection, "/") + mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { + return strings.HasPrefix(req.URL.String(), urlCreateConnection) + })).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: func() io.ReadCloser { + respBuf, _ := json.Marshal("connection_id") + return ioutil.NopCloser(bytes.NewReader(respBuf)) + }(), + }, nil) } } @@ -801,17 +813,17 @@ func TestAllocation_RepairFile(t *testing.T) { wantRepair bool errMsg string }{ - { - name: "Test_Repair_Not_Required_Failed", - parameters: parameters{ - localPath: mockLocalPath, - remotePath: "/", - }, - numBlobbers: 4, - numCorrect: 4, - setup: setupHttpResponses, - wantRepair: false, - }, + // { + // name: "Test_Repair_Not_Required_Failed", + // parameters: parameters{ + // localPath: mockLocalPath, + // remotePath: "/", + // }, + // numBlobbers: 4, + // numCorrect: 4, + // setup: setupHttpResponses, + // wantRepair: false, + // }, { name: "Test_Repair_Required_Success", parameters: parameters{ @@ -845,13 +857,13 @@ func TestAllocation_RepairFile(t *testing.T) { a.mutex = &sync.Mutex{} a.initialized = true sdkInitialized = true - setupMockAllocation(t, a) for i := 0; i < tt.numBlobbers; i++ { a.Blobbers = append(a.Blobbers, &blockchain.StorageNode{ ID: mockBlobberId + strconv.Itoa(i), Baseurl: "http://TestAllocation_RepairFile" + tt.name + mockBlobberUrl + strconv.Itoa(i), }) } + setupMockAllocation(t, a) tt.setup(t, tt.name, tt.numBlobbers, tt.numCorrect) found, _, isRequired, ref, err := a.RepairRequired(tt.parameters.remotePath) require.Nil(err) @@ -865,7 +877,8 @@ func TestAllocation_RepairFile(t *testing.T) { require.Nil(err) require.NotNil(sz) ref.ActualSize = sz.Size() - err = a.RepairFile(f, tt.parameters.remotePath, tt.parameters.status, found, ref) + op := a.RepairFile(f, tt.parameters.remotePath, tt.parameters.status, found, ref) + err = a.DoMultiOperation([]OperationRequest{*op}, WithRepair()) if tt.wantErr { require.NotNil(err) } else { diff --git a/zboxcore/sdk/allocation_test.go b/zboxcore/sdk/allocation_test.go index ccf7ab5c3..d72e155cc 100644 --- a/zboxcore/sdk/allocation_test.go +++ b/zboxcore/sdk/allocation_test.go @@ -2338,6 +2338,7 @@ func setupMockAllocation(t *testing.T, a *Allocation) { a.downloadProgressMap = make(map[string]*DownloadRequest) a.mutex = &sync.Mutex{} a.FileOptions = uint16(63) // 0011 1111 All allowed + InitCommitWorker(a.Blobbers) a.initialized = true if a.DataShards != 0 { a.fullconsensus, a.consensusThreshold = a.getConsensuses() diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index 223522def..c04add7ba 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -34,7 +34,7 @@ import ( ) const ( - DefaultUploadTimeOut = 2 * time.Minute + DefaultUploadTimeOut = 45 * time.Second ) var ( @@ -646,9 +646,9 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, select { case <-su.ctx.Done(): return context.Cause(su.ctx) - default: + case su.uploadChan <- blobberUpload: } - su.uploadChan <- blobberUpload + if isFinal { su.uploadWG.Wait() select { diff --git a/zboxcore/sdk/commitworker.go b/zboxcore/sdk/commitworker.go index 2f3f22bb8..b5bd31988 100644 --- a/zboxcore/sdk/commitworker.go +++ b/zboxcore/sdk/commitworker.go @@ -98,6 +98,11 @@ func (commitreq *CommitRequest) processCommit() { for _, change := range commitreq.changes { paths = append(paths, change.GetAffectedPath()...) } + if len(paths) == 0 { + l.Logger.Info("Nothing to commit") + commitreq.result = SuccessCommitResult() + return + } var req *http.Request var lR ReferencePathResult req, err := zboxutil.NewReferencePathRequest(commitreq.blobber.Baseurl, commitreq.allocationID, commitreq.allocationTx, paths) @@ -105,12 +110,6 @@ func (commitreq *CommitRequest) processCommit() { l.Logger.Error("Creating ref path req", err) return } - if len(paths) == 0 { - l.Logger.Info("Nothing to commit") - commitreq.result = SuccessCommitResult() - return - } - ctx, cncl := context.WithTimeout(context.Background(), (time.Second * 30)) err = zboxutil.HttpDo(ctx, cncl, req, func(resp *http.Response, err error) error { if err != nil { diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 3281a83e8..6f643c3d6 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -447,12 +447,25 @@ func (req *DownloadRequest) processDownload(ctx context.Context) { isPREAndWholeFile = true } + writeCtx, writeCancel := context.WithCancel(ctx) + defer writeCancel() var wg sync.WaitGroup wg.Add(1) + // Handle writing the blocks in order as soon as they are downloaded go func() { buffer := make(map[int][]byte) for i := 0; i < n; i++ { + select { + case <-writeCtx.Done(): + if isPREAndWholeFile { + closeOnce.Do(func() { + close(hashDataChan) + }) + } + goto breakLoop + default: + } if data, ok := buffer[i]; ok { // If the block we need to write next is already in the buffer, write it numBytes := int64(math.Min(float64(remainingSize), float64(len(data)))) @@ -527,6 +540,7 @@ func (req *DownloadRequest) processDownload(ctx context.Context) { } } } + breakLoop: req.fileHandler.Sync() //nolint wg.Done() }() @@ -552,11 +566,7 @@ func (req *DownloadRequest) processDownload(ctx context.Context) { }) } if err := eg.Wait(); err != nil { - if isPREAndWholeFile { - closeOnce.Do(func() { - close(hashDataChan) - }) - } + writeCancel() req.errorCB(err, remotePathCB) return } @@ -830,9 +840,12 @@ func (req *DownloadRequest) calculateShardsParams( if err != nil { return 0, err } - _, err = req.Seek(info.Size(), io.SeekStart) - if err != nil { - return 0, err + // Can be nil when using file writer in wasm + if info != nil { + _, err = req.Seek(info.Size(), io.SeekStart) + if err != nil { + return 0, err + } } effectiveChunkSize := effectiveBlockSize * int64(req.datashards) diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index b24f25992..8cdf19ba9 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -31,6 +31,15 @@ const ( var BatchSize = 6 +type MultiOperationOption func(mo *MultiOperation) + +func WithRepair() MultiOperationOption { + return func(mo *MultiOperation) { + mo.Consensus.consensusThresh = 0 + mo.isRepair = true + } +} + type Operationer interface { Process(allocObj *Allocation, connectionID string) ([]fileref.RefEntity, zboxutil.Uint128, error) buildChange(refs []fileref.RefEntity, uid uuid.UUID) []allocationchange.AllocationChange @@ -48,7 +57,8 @@ type MultiOperation struct { operationMask zboxutil.Uint128 maskMU *sync.Mutex Consensus - changes [][]allocationchange.AllocationChange + changes [][]allocationchange.AllocationChange + isRepair bool } func (mo *MultiOperation) createConnectionObj(blobberIdx int) (err error) { @@ -220,37 +230,40 @@ func (mo *MultiOperation) Process() error { } logger.Logger.Info("[writemarkerLocked]", time.Since(start).Milliseconds()) start = time.Now() - status, err := mo.allocationObj.CheckAllocStatus() - if err != nil { - logger.Logger.Error("Error checking allocation status", err) - writeMarkerMutex.Unlock(mo.ctx, mo.operationMask, mo.allocationObj.Blobbers, time.Minute, mo.connectionID) //nolint: errcheck - return fmt.Errorf("Check allocation status failed: %s", err.Error()) - } - if status == Repair { - logger.Logger.Info("Repairing allocation") - writeMarkerMutex.Unlock(mo.ctx, mo.operationMask, mo.allocationObj.Blobbers, time.Minute, mo.connectionID) //nolint: errcheck - statusBar := NewRepairBar(mo.allocationObj.ID) - if statusBar == nil { + status := Commit + if !mo.isRepair { + status, err = mo.allocationObj.CheckAllocStatus() + if err != nil { + logger.Logger.Error("Error checking allocation status", err) + writeMarkerMutex.Unlock(mo.ctx, mo.operationMask, mo.allocationObj.Blobbers, time.Minute, mo.connectionID) //nolint: errcheck + return fmt.Errorf("Check allocation status failed: %s", err.Error()) + } + if status == Repair { + logger.Logger.Info("Repairing allocation") + writeMarkerMutex.Unlock(mo.ctx, mo.operationMask, mo.allocationObj.Blobbers, time.Minute, mo.connectionID) //nolint: errcheck + statusBar := NewRepairBar(mo.allocationObj.ID) + if statusBar == nil { + for _, op := range mo.operations { + op.Error(mo.allocationObj, 0, ErrRetryOperation) + } + return ErrRetryOperation + } + statusBar.wg.Add(1) + err = mo.allocationObj.RepairAlloc(statusBar) + if err != nil { + return err + } + statusBar.wg.Wait() + if statusBar.success { + l.Logger.Info("Repair success") + } else { + l.Logger.Error("Repair failed") + } for _, op := range mo.operations { op.Error(mo.allocationObj, 0, ErrRetryOperation) } return ErrRetryOperation } - statusBar.wg.Add(1) - err = mo.allocationObj.RepairAlloc(statusBar) - if err != nil { - return err - } - statusBar.wg.Wait() - if statusBar.success { - l.Logger.Info("Repair success") - } else { - l.Logger.Error("Repair failed") - } - for _, op := range mo.operations { - op.Error(mo.allocationObj, 0, ErrRetryOperation) - } - return ErrRetryOperation } defer writeMarkerMutex.Unlock(mo.ctx, mo.operationMask, mo.allocationObj.Blobbers, time.Minute, mo.connectionID) //nolint: errcheck if status != Commit { @@ -293,7 +306,9 @@ func (mo *MultiOperation) Process() error { if commitReq.result != nil { if commitReq.result.Success { l.Logger.Info("Commit success", commitReq.blobber.Baseurl) - rollbackMask = rollbackMask.Or(zboxutil.NewUint128(1).Lsh(commitReq.blobberInd)) + if !mo.isRepair { + rollbackMask = rollbackMask.Or(zboxutil.NewUint128(1).Lsh(commitReq.blobberInd)) + } mo.consensus++ } else { l.Logger.Info("Commit failed", commitReq.blobber.Baseurl, commitReq.result.ErrorMessage) diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go index 57c279db2..92eba7efc 100644 --- a/zboxcore/sdk/repairworker.go +++ b/zboxcore/sdk/repairworker.go @@ -2,8 +2,10 @@ package sdk import ( "context" + "io" "sync" + "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/core/sys" "github.com/0chain/gosdk/zboxcore/fileref" l "github.com/0chain/gosdk/zboxcore/logger" @@ -69,7 +71,8 @@ func (r *RepairRequest) processRepair(ctx context.Context, a *Allocation) { } } -func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) { +func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) []OperationRequest { + ops := make([]OperationRequest, 0) switch dir.Type { case fileref.DIRECTORY: if len(dir.Children) == 0 { @@ -77,7 +80,7 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) { dir, err = a.ListDir(dir.Path, true) if err != nil { l.Logger.Error("Failed to get listDir for path ", zap.Any("path", dir.Path), zap.Error(err)) - return + return nil } } if len(dir.Children) == 0 { @@ -92,7 +95,7 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) { if r.statusCB != nil { r.statusCB.Error(a.ID, dir.Path, OpRepair, err) } - return + return nil } r.filesRepaired++ } else if consensus < len(a.Blobbers) { @@ -103,7 +106,7 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) { if r.statusCB != nil { r.statusCB.Error(a.ID, dir.Path, OpRepair, err) } - return + return nil } r.filesRepaired++ } @@ -111,28 +114,41 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) { } for _, childDir := range dir.Children { if r.checkForCancel(a) { - return + return nil + } + ops = append(ops, r.iterateDir(a, childDir)...) + if len(ops) >= MultiOpBatchSize/2 { + r.repairOperation(a, ops) + ops = nil } - r.iterateDir(a, childDir) } - + if len(ops) > 0 { + r.repairOperation(a, ops) + ops = nil + } case fileref.FILE: - r.repairFile(a, dir) + // this returns op object and mask + op := r.repairFile(a, dir) + if op != nil { + ops = append(ops, *op) + } default: l.Logger.Info("Invalid directory type", zap.Any("type", dir.Type)) } + return ops } -func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) { +func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) *OperationRequest { + var op *OperationRequest if r.checkForCancel(a) { - return + return nil } l.Logger.Info("Checking file for the path :", zap.Any("path", file.Path)) found, deleteMask, repairRequired, ref, err := a.RepairRequired(file.Path) if err != nil { l.Logger.Error("repair_required_failed", zap.Error(err)) - return + return nil } if repairRequired { l.Logger.Info("Repair required for the path :", zap.Any("path", file.Path)) @@ -146,89 +162,79 @@ func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) { if deleteMask.CountOnes() > 0 { l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path)) - consensus := deleteMask.CountOnes() - err := a.deleteFile(file.Path, 0, consensus, deleteMask) - if err != nil { - l.Logger.Error("delete_file_failed", zap.Error(err)) - return + // consensus := deleteMask.CountOnes() + // err := a.deleteFile(file.Path, 0, consensus, deleteMask) + op = &OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: file.Path, + Mask: &deleteMask, } } - + wg.Add(1) localPath := r.getLocalPath(file) if !checkFileExists(localPath) { if r.checkForCancel(a) { - return + return nil } - l.Logger.Info("Downloading file for the path :", zap.Any("path", file.Path)) - wg.Add(1) memFile := &sys.MemChanFile{ Buffer: make(chan []byte, 10), ChunkWriteSize: int(a.GetChunkReadSize(ref.EncryptedKey != "")), } - err = a.DownloadFileToFileHandler(memFile, ref.Path, false, statusCB, true) - if err != nil { - l.Logger.Error("download_file_failed", zap.Error(err)) - return - } - wg.Add(1) - uploadStatusCB := &RepairStatusCB{ - wg: &wg, - statusCB: r.statusCB, - } - l.Logger.Info("Repairing file for the path :", zap.Any("path", file.Path)) - go func(memFile *sys.MemChanFile) { - err = a.RepairFile(memFile, file.Path, uploadStatusCB, found, ref) - if err != nil { - l.Logger.Error("repair_file_failed", zap.Error(err)) - _ = a.CancelDownload(file.Path) - } - }(memFile) - wg.Wait() - if !uploadStatusCB.success { - l.Logger.Error("Failed to upload file, Status call back failed", - zap.Any("localpath", localPath), zap.Any("remotepath", file.Path)) - return - } - l.Logger.Info("Download file and upload success for repair", zap.Any("localpath", localPath), zap.Any("remotepath", file.Path)) + op = a.RepairFile(memFile, file.Path, statusCB, found, ref) + op.DownloadFile = true } else { - l.Logger.Info("FILE EXISTS", zap.Any("bool", true)) f, err := sys.Files.Open(localPath) - if err != nil { - l.Logger.Error("open_file_failed", zap.Error(err)) - return - } - wg.Add(1) - err = a.RepairFile(f, file.Path, statusCB, found, ref) if err != nil { l.Logger.Error("repair_file_failed", zap.Error(err)) - return + return nil } - defer f.Close() + op = a.RepairFile(f, file.Path, statusCB, found, ref) } if r.checkForCancel(a) { - return + return nil } } else { l.Logger.Info("Repair by delete", zap.Any("path", file.Path)) - consensus := found.CountOnes() - err := a.deleteFile(file.Path, 1, consensus, found) - if err != nil { - l.Logger.Error("repair_file_failed", zap.Error(err)) - return + op = &OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: file.Path, + Mask: &found, } } - l.Logger.Info("Repair file success", zap.Any("remotepath", file.Path)) - r.filesRepaired++ } else if deleteMask.CountOnes() > 0 { l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path)) - consensus := deleteMask.CountOnes() - err := a.deleteFile(file.Path, 0, consensus, deleteMask) - if err != nil { - l.Logger.Error("repair_file_failed", zap.Error(err)) - return + op = &OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: file.Path, + Mask: &deleteMask, + } + } + return op +} + +func (r *RepairRequest) repairOperation(a *Allocation, ops []OperationRequest) { + err := a.DoMultiOperation(ops, WithRepair()) + if err != nil { + l.Logger.Error("repair_file_failed", zap.Error(err)) + status := r.statusCB != nil + for _, op := range ops { + if op.DownloadFile { + _ = a.CancelDownload(op.RemotePath) + } + if status { + r.statusCB.Error(a.ID, op.RemotePath, OpRepair, err) + } + } + } else { + r.filesRepaired += len(ops) + } + for _, op := range ops { + if op.FileReader != nil && !op.DownloadFile { + if f, ok := op.FileReader.(io.Closer); ok { + f.Close() + } } - r.filesRepaired++ } } diff --git a/zboxcore/sdk/upload_worker.go b/zboxcore/sdk/upload_worker.go index b14552361..94e22a0f3 100644 --- a/zboxcore/sdk/upload_worker.go +++ b/zboxcore/sdk/upload_worker.go @@ -6,6 +6,7 @@ import ( thrown "github.com/0chain/errors" "github.com/0chain/gosdk/constants" + "github.com/0chain/gosdk/core/sys" "github.com/0chain/gosdk/zboxcore/allocationchange" "github.com/0chain/gosdk/zboxcore/fileref" l "github.com/0chain/gosdk/zboxcore/logger" @@ -19,9 +20,19 @@ type UploadOperation struct { opCode int chunkedUpload *ChunkedUpload isUpdate bool + isDownload bool } func (uo *UploadOperation) Process(allocObj *Allocation, connectionID string) ([]fileref.RefEntity, zboxutil.Uint128, error) { + if uo.isDownload { + if f, ok := uo.chunkedUpload.fileReader.(*sys.MemChanFile); ok { + err := allocObj.DownloadFileToFileHandler(f, uo.chunkedUpload.fileMeta.RemotePath, false, nil, true) + if err != nil { + l.Logger.Error("DownloadFileToFileHandler Failed", zap.String("path", uo.chunkedUpload.fileMeta.RemotePath), zap.Error(err)) + return nil, uo.chunkedUpload.uploadMask, err + } + } + } err := uo.chunkedUpload.process() if err != nil { l.Logger.Error("UploadOperation Failed", zap.String("name", uo.chunkedUpload.fileMeta.RemoteName), zap.Error(err)) @@ -117,7 +128,7 @@ func (uo *UploadOperation) Error(allocObj *Allocation, consensus int, err error) } } -func NewUploadOperation(workdir string, allocObj *Allocation, connectionID string, fileMeta FileMeta, fileReader io.Reader, isUpdate, isWebstreaming bool, opts ...ChunkedUploadOption) (*UploadOperation, string, error) { +func NewUploadOperation(workdir string, allocObj *Allocation, connectionID string, fileMeta FileMeta, fileReader io.Reader, isUpdate, isWebstreaming, isRepair, isMemoryDownload bool, opts ...ChunkedUploadOption) (*UploadOperation, string, error) { uo := &UploadOperation{} if fileMeta.ActualSize == 0 { byteReader := bytes.NewReader([]byte( @@ -126,7 +137,8 @@ func NewUploadOperation(workdir string, allocObj *Allocation, connectionID strin opts = append(opts, WithActualHash(emptyFileDataHash)) fileMeta.ActualSize = int64(len(emptyFileDataHash)) } - cu, err := CreateChunkedUpload(workdir, allocObj, fileMeta, fileReader, isUpdate, false, isWebstreaming, connectionID, opts...) + + cu, err := CreateChunkedUpload(workdir, allocObj, fileMeta, fileReader, isUpdate, isRepair, isWebstreaming, connectionID, opts...) if err != nil { return nil, "", err } @@ -134,5 +146,6 @@ func NewUploadOperation(workdir string, allocObj *Allocation, connectionID strin uo.chunkedUpload = cu uo.opCode = cu.opCode uo.isUpdate = isUpdate + uo.isDownload = isMemoryDownload return uo, cu.progress.ConnectionID, nil } From 553416a9feb2e64ad0b33454917ae04625532b22 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Sat, 6 Jan 2024 00:40:54 +0530 Subject: [PATCH 06/12] fix repair after update (#1357) --- zboxcore/sdk/allocation_file_test.go | 74 ++++++++++++++-------------- zboxcore/sdk/listworker.go | 5 +- zboxcore/sdk/repairworker.go | 26 +++++----- 3 files changed, 55 insertions(+), 50 deletions(-) diff --git a/zboxcore/sdk/allocation_file_test.go b/zboxcore/sdk/allocation_file_test.go index 91ff49239..38da3068c 100644 --- a/zboxcore/sdk/allocation_file_test.go +++ b/zboxcore/sdk/allocation_file_test.go @@ -647,32 +647,32 @@ func TestAllocation_RepairFile(t *testing.T) { ClientKey: mockClientKey, } - // setupHttpResponses := func(t *testing.T, testName string, numBlobbers, numCorrect int) { - // require.True(t, numBlobbers >= numCorrect) - // for i := 0; i < numBlobbers; i++ { - // var hash string - // if i < numCorrect { - // hash = mockActualHash - // } - // frName := mockFileRefName + strconv.Itoa(i) - // url := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" - // mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - // return strings.HasPrefix(req.URL.String(), url) - // })).Return(&http.Response{ - // StatusCode: http.StatusOK, - // Body: func(fileRefName, hash string) io.ReadCloser { - // jsonFR, err := json.Marshal(&fileref.FileRef{ - // ActualFileHash: hash, - // Ref: fileref.Ref{ - // Name: fileRefName, - // }, - // }) - // require.NoError(t, err) - // return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) - // }(frName, hash), - // }, nil) - // } - // } + setupHttpResponses := func(t *testing.T, testName string, numBlobbers, numCorrect int) { + require.True(t, numBlobbers >= numCorrect) + for i := 0; i < numBlobbers; i++ { + var hash string + if i < numCorrect { + hash = mockActualHash + } + frName := mockFileRefName + strconv.Itoa(i) + url := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" + mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { + return strings.HasPrefix(req.URL.String(), url) + })).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: func(fileRefName, hash string) io.ReadCloser { + jsonFR, err := json.Marshal(&fileref.FileRef{ + ActualFileHash: hash, + Ref: fileref.Ref{ + Name: fileRefName, + }, + }) + require.NoError(t, err) + return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) + }(frName, hash), + }, nil) + } + } setupHttpResponsesWithUpload := func(t *testing.T, testName string, numBlobbers, numCorrect int) { require.True(t, numBlobbers >= numCorrect) @@ -813,17 +813,17 @@ func TestAllocation_RepairFile(t *testing.T) { wantRepair bool errMsg string }{ - // { - // name: "Test_Repair_Not_Required_Failed", - // parameters: parameters{ - // localPath: mockLocalPath, - // remotePath: "/", - // }, - // numBlobbers: 4, - // numCorrect: 4, - // setup: setupHttpResponses, - // wantRepair: false, - // }, + { + name: "Test_Repair_Not_Required_Failed", + parameters: parameters{ + localPath: mockLocalPath, + remotePath: "/", + }, + numBlobbers: 4, + numCorrect: 4, + setup: setupHttpResponses, + wantRepair: false, + }, { name: "Test_Repair_Required_Success", parameters: parameters{ diff --git a/zboxcore/sdk/listworker.go b/zboxcore/sdk/listworker.go index b7ae67cf3..fea81df3e 100644 --- a/zboxcore/sdk/listworker.go +++ b/zboxcore/sdk/listworker.go @@ -242,7 +242,10 @@ func (req *ListRequest) GetListFromBlobbers() (*ListResult, error) { if req.forRepair { for _, child := range childResultMap { if child.consensus < child.fullconsensus { - result.Children = append(result.Children, child) + if _, ok := selected[child.LookupHash]; !ok { + result.Children = append(result.Children, child) + selected[child.LookupHash] = child + } } } } diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go index 92eba7efc..02387fe30 100644 --- a/zboxcore/sdk/repairworker.go +++ b/zboxcore/sdk/repairworker.go @@ -128,9 +128,9 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) []OperationRe } case fileref.FILE: // this returns op object and mask - op := r.repairFile(a, dir) - if op != nil { - ops = append(ops, *op) + repairOps := r.repairFile(a, dir) + if repairOps != nil { + ops = append(ops, repairOps...) } default: @@ -139,8 +139,8 @@ func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) []OperationRe return ops } -func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) *OperationRequest { - var op *OperationRequest +func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) []OperationRequest { + ops := make([]OperationRequest, 0) if r.checkForCancel(a) { return nil } @@ -162,16 +162,16 @@ func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) *OperationRe if deleteMask.CountOnes() > 0 { l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path)) - // consensus := deleteMask.CountOnes() - // err := a.deleteFile(file.Path, 0, consensus, deleteMask) - op = &OperationRequest{ + op := OperationRequest{ OperationType: constants.FileOperationDelete, RemotePath: file.Path, Mask: &deleteMask, } + ops = append(ops, op) } wg.Add(1) localPath := r.getLocalPath(file) + var op *OperationRequest if !checkFileExists(localPath) { if r.checkForCancel(a) { return nil @@ -190,27 +190,29 @@ func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) *OperationRe } op = a.RepairFile(f, file.Path, statusCB, found, ref) } - + ops = append(ops, *op) if r.checkForCancel(a) { return nil } } else { l.Logger.Info("Repair by delete", zap.Any("path", file.Path)) - op = &OperationRequest{ + op := OperationRequest{ OperationType: constants.FileOperationDelete, RemotePath: file.Path, Mask: &found, } + ops = append(ops, op) } } else if deleteMask.CountOnes() > 0 { l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path)) - op = &OperationRequest{ + op := OperationRequest{ OperationType: constants.FileOperationDelete, RemotePath: file.Path, Mask: &deleteMask, } + ops = append(ops, op) } - return op + return ops } func (r *RepairRequest) repairOperation(a *Allocation, ops []OperationRequest) { From 8125003dff7b4b8ae9d8520edcacdc5ee2cb8560 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Wed, 10 Jan 2024 00:46:36 +0530 Subject: [PATCH 07/12] fix err msg (#1361) * fix err msg * calc root hash once --- zboxcore/allocationchange/copyobject.go | 1 - zboxcore/allocationchange/createdir.go | 1 - zboxcore/allocationchange/deletefile.go | 2 -- zboxcore/allocationchange/moveobject.go | 1 - zboxcore/allocationchange/newfile.go | 1 - zboxcore/allocationchange/renameobject.go | 1 - zboxcore/allocationchange/updatefile.go | 1 - zboxcore/sdk/commitworker.go | 1 + zboxcore/sdk/downloadworker.go | 2 +- 9 files changed, 2 insertions(+), 9 deletions(-) diff --git a/zboxcore/allocationchange/copyobject.go b/zboxcore/allocationchange/copyobject.go index a30020779..2454cf3b6 100644 --- a/zboxcore/allocationchange/copyobject.go +++ b/zboxcore/allocationchange/copyobject.go @@ -76,7 +76,6 @@ func (ch *CopyFileChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[str ch.processChildren(affectedRef, fileIDMeta) dirRef.AddChild(ch.ObjectTree) - rootRef.CalculateHash() return } diff --git a/zboxcore/allocationchange/createdir.go b/zboxcore/allocationchange/createdir.go index 8737ca9dd..345e5cbaf 100644 --- a/zboxcore/allocationchange/createdir.go +++ b/zboxcore/allocationchange/createdir.go @@ -60,7 +60,6 @@ func (d *DirCreateChange) ProcessChange(rootRef *fileref.Ref, FileIDMeta map[str } } - rootRef.CalculateHash() return } diff --git a/zboxcore/allocationchange/deletefile.go b/zboxcore/allocationchange/deletefile.go index c4365174a..ba64f3f8a 100644 --- a/zboxcore/allocationchange/deletefile.go +++ b/zboxcore/allocationchange/deletefile.go @@ -17,7 +17,6 @@ func (ch *DeleteFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str if ch.ObjectTree.GetPath() == "/" { rootRef.Children = nil - rootRef.CalculateHash() return } @@ -48,7 +47,6 @@ func (ch *DeleteFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str for i, child := range dirRef.Children { if child.GetName() == ch.ObjectTree.GetName() { dirRef.RemoveChild(i) - rootRef.CalculateHash() return } } diff --git a/zboxcore/allocationchange/moveobject.go b/zboxcore/allocationchange/moveobject.go index 5f7ed9b63..b913f527b 100644 --- a/zboxcore/allocationchange/moveobject.go +++ b/zboxcore/allocationchange/moveobject.go @@ -112,7 +112,6 @@ func (ch *MoveFileChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[str dirRef.AddChild(ch.ObjectTree) - rootRef.CalculateHash() return } diff --git a/zboxcore/allocationchange/newfile.go b/zboxcore/allocationchange/newfile.go index 9bebd2900..d7961c505 100644 --- a/zboxcore/allocationchange/newfile.go +++ b/zboxcore/allocationchange/newfile.go @@ -92,7 +92,6 @@ func (ch *NewFileChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[stri fileIDMeta[ch.File.GetPath()] = ch.File.FileID dirRef.AddChild(ch.File) - rootRef.CalculateHash() return } diff --git a/zboxcore/allocationchange/renameobject.go b/zboxcore/allocationchange/renameobject.go index 64f5851d9..dc5471a65 100644 --- a/zboxcore/allocationchange/renameobject.go +++ b/zboxcore/allocationchange/renameobject.go @@ -64,7 +64,6 @@ func (ch *RenameFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str return } ch.processChildren(affectedRef) - rootRef.CalculateHash() return } diff --git a/zboxcore/allocationchange/updatefile.go b/zboxcore/allocationchange/updatefile.go index 5182dc246..f3b89ae8d 100644 --- a/zboxcore/allocationchange/updatefile.go +++ b/zboxcore/allocationchange/updatefile.go @@ -86,7 +86,6 @@ func (ch *UpdateFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str ch.NewFile.HashToBeComputed = true ch.NewFile.FileID = ch.OldFile.FileID dirRef.Children[idx] = ch.NewFile - rootRef.CalculateHash() return } diff --git a/zboxcore/sdk/commitworker.go b/zboxcore/sdk/commitworker.go index b5bd31988..e6cc22daf 100644 --- a/zboxcore/sdk/commitworker.go +++ b/zboxcore/sdk/commitworker.go @@ -181,6 +181,7 @@ func (commitreq *CommitRequest) processCommit() { } size += change.GetSize() } + rootRef.CalculateHash() err = commitreq.commitBlobber(rootRef, lR.LatestWM, size, fileIDMeta) if err != nil { commitreq.result = ErrorCommitResult(err.Error()) diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 6f643c3d6..06ac4acb4 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -285,7 +285,7 @@ func (req *DownloadRequest) fillShards(shards [][][]byte, result *downloadBlock) data = result.BlockChunks[i] } if i >= len(shards) || len(shards[i]) <= result.idx { - l.Logger.Error("Invalid shard index", result.idx, len(shards), len(shards[i])) + l.Logger.Error("Invalid shard index", result.idx, len(shards)) return errors.New("invalid_shard_index", fmt.Sprintf("Invalid shard index %d shard len: %d shard block len: %d", result.idx, len(shards), i)) } shards[i][result.idx] = data From a415a6bef76bdb7ce444e6e570e027aea2f948c1 Mon Sep 17 00:00:00 2001 From: Yury Date: Mon, 15 Jan 2024 17:34:23 +0200 Subject: [PATCH 08/12] Hotfix/add fork (#1364) * remove path header, fix ws tag * add hardfork * add hardfork --------- Co-authored-by: din-mukhammed --- core/transaction/entity.go | 1 + zboxcore/mocks/AllocationChange.go | 11 ++++---- zboxcore/mocks/EncryptionScheme.go | 11 ++++---- zboxcore/mocks/FileNameBuilder.go | 11 ++++---- zboxcore/mocks/Hasher.go | 11 ++++---- zboxcore/mocks/HttpClient.go | 11 ++++---- zboxcore/mocks/LiveUploadReader.go | 11 ++++---- zboxcore/mocks/M3u8Writer.go | 11 ++++---- zboxcore/mocks/RefEntity.go | 11 ++++---- zboxcore/mocks/SCRestAPIHandler.go | 11 ++++---- zboxcore/mocks/SignFunc.go | 11 ++++---- zboxcore/mocks/StatusCallback.go | 11 ++++---- zboxcore/mocks/Suite.go | 11 ++++---- zcnbridge/mocks/EthereumClient.go | 11 ++++---- zcnbridge/mocks/JobResult.go | 11 ++++---- zcnbridge/mocks/KeyStore.go | 11 ++++---- zcnbridge/mocks/Transaction.go | 11 ++++---- .../mocks/TransactionCallbackAwaitable.go | 11 ++++---- zcnbridge/mocks/TransactionProvider.go | 11 ++++---- zcnbridge/transaction/mocks/Transaction.go | 11 ++++---- .../mocks/TransactionCallbackAwaitable.go | 11 ++++---- .../transaction/mocks/TransactionProvider.go | 11 ++++---- zcncore/mocks/AuthCallback.go | 11 ++++---- zcncore/mocks/FeeOption.go | 11 ++++---- zcncore/mocks/GetBalanceCallback.go | 11 ++++---- zcncore/mocks/GetInfoCallback.go | 11 ++++---- zcncore/mocks/GetNonceCallback.go | 11 ++++---- zcncore/mocks/MSVoteCallback.go | 11 ++++---- zcncore/mocks/QueryResultHandle.go | 11 ++++---- zcncore/mocks/TransactionCallback.go | 11 ++++---- zcncore/mocks/TransactionCommon.go | 25 ++++++++++++++----- zcncore/mocks/TransactionScheme.go | 25 ++++++++++++++----- zcncore/mocks/WalletCallback.go | 11 ++++---- zcncore/transaction.go | 11 ++++++++ zcncore/transactionauth.go | 11 ++++++++ 35 files changed, 211 insertions(+), 192 deletions(-) diff --git a/core/transaction/entity.go b/core/transaction/entity.go index 70782c439..4764c1794 100644 --- a/core/transaction/entity.go +++ b/core/transaction/entity.go @@ -138,6 +138,7 @@ const ( STORAGESC_WRITE_POOL_LOCK = "write_pool_lock" STORAGESC_WRITE_POOL_UNLOCK = "write_pool_unlock" STORAGESC_UPDATE_SETTINGS = "update_settings" + ADD_HARDFORK = "add_hardfork" STORAGESC_COLLECT_REWARD = "collect_reward" STORAGESC_KILL_BLOBBER = "kill_blobber" STORAGESC_KILL_VALIDATOR = "kill_validator" diff --git a/zboxcore/mocks/AllocationChange.go b/zboxcore/mocks/AllocationChange.go index d5205842a..ade138830 100644 --- a/zboxcore/mocks/AllocationChange.go +++ b/zboxcore/mocks/AllocationChange.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -56,13 +56,12 @@ func (_m *AllocationChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[s return r0 } -type mockConstructorTestingTNewAllocationChange interface { +// NewAllocationChange creates a new instance of AllocationChange. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAllocationChange(t interface { mock.TestingT Cleanup(func()) -} - -// NewAllocationChange creates a new instance of AllocationChange. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewAllocationChange(t mockConstructorTestingTNewAllocationChange) *AllocationChange { +}) *AllocationChange { mock := &AllocationChange{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/EncryptionScheme.go b/zboxcore/mocks/EncryptionScheme.go index bbd4f0286..870ec26cf 100644 --- a/zboxcore/mocks/EncryptionScheme.go +++ b/zboxcore/mocks/EncryptionScheme.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -303,13 +303,12 @@ func (_m *EncryptionScheme) ReEncrypt(encMsg *encryption.EncryptedMessage, reGen return r0, r1 } -type mockConstructorTestingTNewEncryptionScheme interface { +// NewEncryptionScheme creates a new instance of EncryptionScheme. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEncryptionScheme(t interface { mock.TestingT Cleanup(func()) -} - -// NewEncryptionScheme creates a new instance of EncryptionScheme. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewEncryptionScheme(t mockConstructorTestingTNewEncryptionScheme) *EncryptionScheme { +}) *EncryptionScheme { mock := &EncryptionScheme{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/FileNameBuilder.go b/zboxcore/mocks/FileNameBuilder.go index b8df10942..9e3e39350 100644 --- a/zboxcore/mocks/FileNameBuilder.go +++ b/zboxcore/mocks/FileNameBuilder.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -79,13 +79,12 @@ func (_m *FileNameBuilder) OutFile() string { return r0 } -type mockConstructorTestingTNewFileNameBuilder interface { +// NewFileNameBuilder creates a new instance of FileNameBuilder. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFileNameBuilder(t interface { mock.TestingT Cleanup(func()) -} - -// NewFileNameBuilder creates a new instance of FileNameBuilder. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewFileNameBuilder(t mockConstructorTestingTNewFileNameBuilder) *FileNameBuilder { +}) *FileNameBuilder { mock := &FileNameBuilder{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/Hasher.go b/zboxcore/mocks/Hasher.go index 1e20707a3..677eb141d 100644 --- a/zboxcore/mocks/Hasher.go +++ b/zboxcore/mocks/Hasher.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -137,13 +137,12 @@ func (_m *Hasher) WriteToValidationMT(buf []byte) error { return r0 } -type mockConstructorTestingTNewHasher interface { +// NewHasher creates a new instance of Hasher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewHasher(t interface { mock.TestingT Cleanup(func()) -} - -// NewHasher creates a new instance of Hasher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewHasher(t mockConstructorTestingTNewHasher) *Hasher { +}) *Hasher { mock := &Hasher{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/HttpClient.go b/zboxcore/mocks/HttpClient.go index e7a62e27c..1d0389214 100644 --- a/zboxcore/mocks/HttpClient.go +++ b/zboxcore/mocks/HttpClient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -39,13 +39,12 @@ func (_m *HttpClient) Do(req *http.Request) (*http.Response, error) { return r0, r1 } -type mockConstructorTestingTNewHttpClient interface { +// NewHttpClient creates a new instance of HttpClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewHttpClient(t interface { mock.TestingT Cleanup(func()) -} - -// NewHttpClient creates a new instance of HttpClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewHttpClient(t mockConstructorTestingTNewHttpClient) *HttpClient { +}) *HttpClient { mock := &HttpClient{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/LiveUploadReader.go b/zboxcore/mocks/LiveUploadReader.go index 3c470b5bf..d31affade 100644 --- a/zboxcore/mocks/LiveUploadReader.go +++ b/zboxcore/mocks/LiveUploadReader.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -75,13 +75,12 @@ func (_m *LiveUploadReader) Size() int64 { return r0 } -type mockConstructorTestingTNewLiveUploadReader interface { +// NewLiveUploadReader creates a new instance of LiveUploadReader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewLiveUploadReader(t interface { mock.TestingT Cleanup(func()) -} - -// NewLiveUploadReader creates a new instance of LiveUploadReader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewLiveUploadReader(t mockConstructorTestingTNewLiveUploadReader) *LiveUploadReader { +}) *LiveUploadReader { mock := &LiveUploadReader{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/M3u8Writer.go b/zboxcore/mocks/M3u8Writer.go index 858e00d7f..61f66252b 100644 --- a/zboxcore/mocks/M3u8Writer.go +++ b/zboxcore/mocks/M3u8Writer.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -85,13 +85,12 @@ func (_m *M3u8Writer) Write(p []byte) (int, error) { return r0, r1 } -type mockConstructorTestingTNewM3u8Writer interface { +// NewM3u8Writer creates a new instance of M3u8Writer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewM3u8Writer(t interface { mock.TestingT Cleanup(func()) -} - -// NewM3u8Writer creates a new instance of M3u8Writer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewM3u8Writer(t mockConstructorTestingTNewM3u8Writer) *M3u8Writer { +}) *M3u8Writer { mock := &M3u8Writer{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/RefEntity.go b/zboxcore/mocks/RefEntity.go index cd62950fc..92157ac57 100644 --- a/zboxcore/mocks/RefEntity.go +++ b/zboxcore/mocks/RefEntity.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -195,13 +195,12 @@ func (_m *RefEntity) GetUpdatedAt() common.Timestamp { return r0 } -type mockConstructorTestingTNewRefEntity interface { +// NewRefEntity creates a new instance of RefEntity. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewRefEntity(t interface { mock.TestingT Cleanup(func()) -} - -// NewRefEntity creates a new instance of RefEntity. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewRefEntity(t mockConstructorTestingTNewRefEntity) *RefEntity { +}) *RefEntity { mock := &RefEntity{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/SCRestAPIHandler.go b/zboxcore/mocks/SCRestAPIHandler.go index ebccc15fc..3b2e150c3 100644 --- a/zboxcore/mocks/SCRestAPIHandler.go +++ b/zboxcore/mocks/SCRestAPIHandler.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,13 +14,12 @@ func (_m *SCRestAPIHandler) Execute(response map[string][]byte, numSharders int, _m.Called(response, numSharders, err) } -type mockConstructorTestingTNewSCRestAPIHandler interface { +// NewSCRestAPIHandler creates a new instance of SCRestAPIHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewSCRestAPIHandler(t interface { mock.TestingT Cleanup(func()) -} - -// NewSCRestAPIHandler creates a new instance of SCRestAPIHandler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewSCRestAPIHandler(t mockConstructorTestingTNewSCRestAPIHandler) *SCRestAPIHandler { +}) *SCRestAPIHandler { mock := &SCRestAPIHandler{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/SignFunc.go b/zboxcore/mocks/SignFunc.go index 8b657fd3d..38deaf474 100644 --- a/zboxcore/mocks/SignFunc.go +++ b/zboxcore/mocks/SignFunc.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -33,13 +33,12 @@ func (_m *SignFunc) Execute(hash string) (string, error) { return r0, r1 } -type mockConstructorTestingTNewSignFunc interface { +// NewSignFunc creates a new instance of SignFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewSignFunc(t interface { mock.TestingT Cleanup(func()) -} - -// NewSignFunc creates a new instance of SignFunc. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewSignFunc(t mockConstructorTestingTNewSignFunc) *SignFunc { +}) *SignFunc { mock := &SignFunc{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/StatusCallback.go b/zboxcore/mocks/StatusCallback.go index eb2584ff9..bfc896957 100644 --- a/zboxcore/mocks/StatusCallback.go +++ b/zboxcore/mocks/StatusCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -34,13 +34,12 @@ func (_m *StatusCallback) Started(allocationId string, filePath string, op int, _m.Called(allocationId, filePath, op, totalBytes) } -type mockConstructorTestingTNewStatusCallback interface { +// NewStatusCallback creates a new instance of StatusCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewStatusCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewStatusCallback creates a new instance of StatusCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewStatusCallback(t mockConstructorTestingTNewStatusCallback) *StatusCallback { +}) *StatusCallback { mock := &StatusCallback{} mock.Mock.Test(t) diff --git a/zboxcore/mocks/Suite.go b/zboxcore/mocks/Suite.go index 96874b0cb..12381555b 100644 --- a/zboxcore/mocks/Suite.go +++ b/zboxcore/mocks/Suite.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -105,13 +105,12 @@ func (_m *Suite) String() string { return r0 } -type mockConstructorTestingTNewSuite interface { +// NewSuite creates a new instance of Suite. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewSuite(t interface { mock.TestingT Cleanup(func()) -} - -// NewSuite creates a new instance of Suite. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewSuite(t mockConstructorTestingTNewSuite) *Suite { +}) *Suite { mock := &Suite{} mock.Mock.Test(t) diff --git a/zcnbridge/mocks/EthereumClient.go b/zcnbridge/mocks/EthereumClient.go index 1169cfa92..3b7fa8cd6 100644 --- a/zcnbridge/mocks/EthereumClient.go +++ b/zcnbridge/mocks/EthereumClient.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -316,13 +316,12 @@ func (_m *EthereumClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error return r0, r1 } -type mockConstructorTestingTNewEthereumClient interface { +// NewEthereumClient creates a new instance of EthereumClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewEthereumClient(t interface { mock.TestingT Cleanup(func()) -} - -// NewEthereumClient creates a new instance of EthereumClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewEthereumClient(t mockConstructorTestingTNewEthereumClient) *EthereumClient { +}) *EthereumClient { mock := &EthereumClient{} mock.Mock.Test(t) diff --git a/zcnbridge/mocks/JobResult.go b/zcnbridge/mocks/JobResult.go index 78487e1e3..12baeda31 100644 --- a/zcnbridge/mocks/JobResult.go +++ b/zcnbridge/mocks/JobResult.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -58,13 +58,12 @@ func (_m *JobResult) SetAuthorizerID(ID string) { _m.Called(ID) } -type mockConstructorTestingTNewJobResult interface { +// NewJobResult creates a new instance of JobResult. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewJobResult(t interface { mock.TestingT Cleanup(func()) -} - -// NewJobResult creates a new instance of JobResult. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewJobResult(t mockConstructorTestingTNewJobResult) *JobResult { +}) *JobResult { mock := &JobResult{} mock.Mock.Test(t) diff --git a/zcnbridge/mocks/KeyStore.go b/zcnbridge/mocks/KeyStore.go index be25a121c..5ef69ac41 100644 --- a/zcnbridge/mocks/KeyStore.go +++ b/zcnbridge/mocks/KeyStore.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -96,13 +96,12 @@ func (_m *KeyStore) TimedUnlock(_a0 accounts.Account, _a1 string, _a2 time.Durat return r0 } -type mockConstructorTestingTNewKeyStore interface { +// NewKeyStore creates a new instance of KeyStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewKeyStore(t interface { mock.TestingT Cleanup(func()) -} - -// NewKeyStore creates a new instance of KeyStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewKeyStore(t mockConstructorTestingTNewKeyStore) *KeyStore { +}) *KeyStore { mock := &KeyStore{} mock.Mock.Test(t) diff --git a/zcnbridge/mocks/Transaction.go b/zcnbridge/mocks/Transaction.go index b9a833470..3f90f1c66 100644 --- a/zcnbridge/mocks/Transaction.go +++ b/zcnbridge/mocks/Transaction.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -119,13 +119,12 @@ func (_m *Transaction) Verify(ctx context.Context) error { return r0 } -type mockConstructorTestingTNewTransaction interface { +// NewTransaction creates a new instance of Transaction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransaction(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransaction creates a new instance of Transaction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransaction(t mockConstructorTestingTNewTransaction) *Transaction { +}) *Transaction { mock := &Transaction{} mock.Mock.Test(t) diff --git a/zcnbridge/mocks/TransactionCallbackAwaitable.go b/zcnbridge/mocks/TransactionCallbackAwaitable.go index 299573252..8482d18c2 100644 --- a/zcnbridge/mocks/TransactionCallbackAwaitable.go +++ b/zcnbridge/mocks/TransactionCallbackAwaitable.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -58,13 +58,12 @@ func (_m *TransactionCallbackAwaitable) WaitVerifyCall(ctx context.Context) erro return r0 } -type mockConstructorTestingTNewTransactionCallbackAwaitable interface { +// NewTransactionCallbackAwaitable creates a new instance of TransactionCallbackAwaitable. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionCallbackAwaitable(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionCallbackAwaitable creates a new instance of TransactionCallbackAwaitable. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionCallbackAwaitable(t mockConstructorTestingTNewTransactionCallbackAwaitable) *TransactionCallbackAwaitable { +}) *TransactionCallbackAwaitable { mock := &TransactionCallbackAwaitable{} mock.Mock.Test(t) diff --git a/zcnbridge/mocks/TransactionProvider.go b/zcnbridge/mocks/TransactionProvider.go index 25617eb25..339ca06f8 100644 --- a/zcnbridge/mocks/TransactionProvider.go +++ b/zcnbridge/mocks/TransactionProvider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -38,13 +38,12 @@ func (_m *TransactionProvider) NewTransactionEntity(txnFee uint64) (transaction. return r0, r1 } -type mockConstructorTestingTNewTransactionProvider interface { +// NewTransactionProvider creates a new instance of TransactionProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionProvider(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionProvider creates a new instance of TransactionProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionProvider(t mockConstructorTestingTNewTransactionProvider) *TransactionProvider { +}) *TransactionProvider { mock := &TransactionProvider{} mock.Mock.Test(t) diff --git a/zcnbridge/transaction/mocks/Transaction.go b/zcnbridge/transaction/mocks/Transaction.go index b9a833470..3f90f1c66 100644 --- a/zcnbridge/transaction/mocks/Transaction.go +++ b/zcnbridge/transaction/mocks/Transaction.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -119,13 +119,12 @@ func (_m *Transaction) Verify(ctx context.Context) error { return r0 } -type mockConstructorTestingTNewTransaction interface { +// NewTransaction creates a new instance of Transaction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransaction(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransaction creates a new instance of Transaction. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransaction(t mockConstructorTestingTNewTransaction) *Transaction { +}) *Transaction { mock := &Transaction{} mock.Mock.Test(t) diff --git a/zcnbridge/transaction/mocks/TransactionCallbackAwaitable.go b/zcnbridge/transaction/mocks/TransactionCallbackAwaitable.go index 299573252..8482d18c2 100644 --- a/zcnbridge/transaction/mocks/TransactionCallbackAwaitable.go +++ b/zcnbridge/transaction/mocks/TransactionCallbackAwaitable.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -58,13 +58,12 @@ func (_m *TransactionCallbackAwaitable) WaitVerifyCall(ctx context.Context) erro return r0 } -type mockConstructorTestingTNewTransactionCallbackAwaitable interface { +// NewTransactionCallbackAwaitable creates a new instance of TransactionCallbackAwaitable. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionCallbackAwaitable(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionCallbackAwaitable creates a new instance of TransactionCallbackAwaitable. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionCallbackAwaitable(t mockConstructorTestingTNewTransactionCallbackAwaitable) *TransactionCallbackAwaitable { +}) *TransactionCallbackAwaitable { mock := &TransactionCallbackAwaitable{} mock.Mock.Test(t) diff --git a/zcnbridge/transaction/mocks/TransactionProvider.go b/zcnbridge/transaction/mocks/TransactionProvider.go index 25617eb25..339ca06f8 100644 --- a/zcnbridge/transaction/mocks/TransactionProvider.go +++ b/zcnbridge/transaction/mocks/TransactionProvider.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -38,13 +38,12 @@ func (_m *TransactionProvider) NewTransactionEntity(txnFee uint64) (transaction. return r0, r1 } -type mockConstructorTestingTNewTransactionProvider interface { +// NewTransactionProvider creates a new instance of TransactionProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionProvider(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionProvider creates a new instance of TransactionProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionProvider(t mockConstructorTestingTNewTransactionProvider) *TransactionProvider { +}) *TransactionProvider { mock := &TransactionProvider{} mock.Mock.Test(t) diff --git a/zcncore/mocks/AuthCallback.go b/zcncore/mocks/AuthCallback.go index 479e6475e..19f6f8010 100644 --- a/zcncore/mocks/AuthCallback.go +++ b/zcncore/mocks/AuthCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,13 +14,12 @@ func (_m *AuthCallback) OnSetupComplete(status int, err string) { _m.Called(status, err) } -type mockConstructorTestingTNewAuthCallback interface { +// NewAuthCallback creates a new instance of AuthCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAuthCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewAuthCallback creates a new instance of AuthCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewAuthCallback(t mockConstructorTestingTNewAuthCallback) *AuthCallback { +}) *AuthCallback { mock := &AuthCallback{} mock.Mock.Test(t) diff --git a/zcncore/mocks/FeeOption.go b/zcncore/mocks/FeeOption.go index 38116bbf2..97a1a33bc 100644 --- a/zcncore/mocks/FeeOption.go +++ b/zcncore/mocks/FeeOption.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -17,13 +17,12 @@ func (_m *FeeOption) Execute(_a0 *zcncore.TxnFeeOption) { _m.Called(_a0) } -type mockConstructorTestingTNewFeeOption interface { +// NewFeeOption creates a new instance of FeeOption. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFeeOption(t interface { mock.TestingT Cleanup(func()) -} - -// NewFeeOption creates a new instance of FeeOption. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewFeeOption(t mockConstructorTestingTNewFeeOption) *FeeOption { +}) *FeeOption { mock := &FeeOption{} mock.Mock.Test(t) diff --git a/zcncore/mocks/GetBalanceCallback.go b/zcncore/mocks/GetBalanceCallback.go index 2ca67ea8d..83a2d75a9 100644 --- a/zcncore/mocks/GetBalanceCallback.go +++ b/zcncore/mocks/GetBalanceCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,13 +14,12 @@ func (_m *GetBalanceCallback) OnBalanceAvailable(status int, value int64, info s _m.Called(status, value, info) } -type mockConstructorTestingTNewGetBalanceCallback interface { +// NewGetBalanceCallback creates a new instance of GetBalanceCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewGetBalanceCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewGetBalanceCallback creates a new instance of GetBalanceCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewGetBalanceCallback(t mockConstructorTestingTNewGetBalanceCallback) *GetBalanceCallback { +}) *GetBalanceCallback { mock := &GetBalanceCallback{} mock.Mock.Test(t) diff --git a/zcncore/mocks/GetInfoCallback.go b/zcncore/mocks/GetInfoCallback.go index 9e28c1422..b2d070f28 100644 --- a/zcncore/mocks/GetInfoCallback.go +++ b/zcncore/mocks/GetInfoCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,13 +14,12 @@ func (_m *GetInfoCallback) OnInfoAvailable(op int, status int, info string, err _m.Called(op, status, info, err) } -type mockConstructorTestingTNewGetInfoCallback interface { +// NewGetInfoCallback creates a new instance of GetInfoCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewGetInfoCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewGetInfoCallback creates a new instance of GetInfoCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewGetInfoCallback(t mockConstructorTestingTNewGetInfoCallback) *GetInfoCallback { +}) *GetInfoCallback { mock := &GetInfoCallback{} mock.Mock.Test(t) diff --git a/zcncore/mocks/GetNonceCallback.go b/zcncore/mocks/GetNonceCallback.go index 6c0256589..0bb9a08f7 100644 --- a/zcncore/mocks/GetNonceCallback.go +++ b/zcncore/mocks/GetNonceCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,13 +14,12 @@ func (_m *GetNonceCallback) OnNonceAvailable(status int, nonce int64, info strin _m.Called(status, nonce, info) } -type mockConstructorTestingTNewGetNonceCallback interface { +// NewGetNonceCallback creates a new instance of GetNonceCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewGetNonceCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewGetNonceCallback creates a new instance of GetNonceCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewGetNonceCallback(t mockConstructorTestingTNewGetNonceCallback) *GetNonceCallback { +}) *GetNonceCallback { mock := &GetNonceCallback{} mock.Mock.Test(t) diff --git a/zcncore/mocks/MSVoteCallback.go b/zcncore/mocks/MSVoteCallback.go index db825fbfc..bd9e9f724 100644 --- a/zcncore/mocks/MSVoteCallback.go +++ b/zcncore/mocks/MSVoteCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,13 +14,12 @@ func (_m *MSVoteCallback) OnVoteComplete(status int, proposal string, err string _m.Called(status, proposal, err) } -type mockConstructorTestingTNewMSVoteCallback interface { +// NewMSVoteCallback creates a new instance of MSVoteCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMSVoteCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewMSVoteCallback creates a new instance of MSVoteCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewMSVoteCallback(t mockConstructorTestingTNewMSVoteCallback) *MSVoteCallback { +}) *MSVoteCallback { mock := &MSVoteCallback{} mock.Mock.Test(t) diff --git a/zcncore/mocks/QueryResultHandle.go b/zcncore/mocks/QueryResultHandle.go index bd43feefb..ca41aada6 100644 --- a/zcncore/mocks/QueryResultHandle.go +++ b/zcncore/mocks/QueryResultHandle.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -26,13 +26,12 @@ func (_m *QueryResultHandle) Execute(result zcncore.QueryResult) bool { return r0 } -type mockConstructorTestingTNewQueryResultHandle interface { +// NewQueryResultHandle creates a new instance of QueryResultHandle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewQueryResultHandle(t interface { mock.TestingT Cleanup(func()) -} - -// NewQueryResultHandle creates a new instance of QueryResultHandle. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewQueryResultHandle(t mockConstructorTestingTNewQueryResultHandle) *QueryResultHandle { +}) *QueryResultHandle { mock := &QueryResultHandle{} mock.Mock.Test(t) diff --git a/zcncore/mocks/TransactionCallback.go b/zcncore/mocks/TransactionCallback.go index fa43d131e..03d32662a 100644 --- a/zcncore/mocks/TransactionCallback.go +++ b/zcncore/mocks/TransactionCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -27,13 +27,12 @@ func (_m *TransactionCallback) OnVerifyComplete(t *zcncore.Transaction, status i _m.Called(t, status) } -type mockConstructorTestingTNewTransactionCallback interface { +// NewTransactionCallback creates a new instance of TransactionCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionCallback creates a new instance of TransactionCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionCallback(t mockConstructorTestingTNewTransactionCallback) *TransactionCallback { +}) *TransactionCallback { mock := &TransactionCallback{} mock.Mock.Test(t) diff --git a/zcncore/mocks/TransactionCommon.go b/zcncore/mocks/TransactionCommon.go index fd487b7b8..942487cfa 100644 --- a/zcncore/mocks/TransactionCommon.go +++ b/zcncore/mocks/TransactionCommon.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,6 +14,20 @@ type TransactionCommon struct { mock.Mock } +// AddHardfork provides a mock function with given fields: ip +func (_m *TransactionCommon) AddHardfork(ip *zcncore.InputMap) error { + ret := _m.Called(ip) + + var r0 error + if rf, ok := ret.Get(0).(func(*zcncore.InputMap) error); ok { + r0 = rf(ip) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // CancelAllocation provides a mock function with given fields: allocID func (_m *TransactionCommon) CancelAllocation(allocID string) error { ret := _m.Called(allocID) @@ -551,13 +565,12 @@ func (_m *TransactionCommon) ZCNSCUpdateGlobalConfig(_a0 *zcncore.InputMap) erro return r0 } -type mockConstructorTestingTNewTransactionCommon interface { +// NewTransactionCommon creates a new instance of TransactionCommon. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionCommon(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionCommon creates a new instance of TransactionCommon. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionCommon(t mockConstructorTestingTNewTransactionCommon) *TransactionCommon { +}) *TransactionCommon { mock := &TransactionCommon{} mock.Mock.Test(t) diff --git a/zcncore/mocks/TransactionScheme.go b/zcncore/mocks/TransactionScheme.go index 3cd02f514..4acee6e6b 100644 --- a/zcncore/mocks/TransactionScheme.go +++ b/zcncore/mocks/TransactionScheme.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,6 +14,20 @@ type TransactionScheme struct { mock.Mock } +// AddHardfork provides a mock function with given fields: ip +func (_m *TransactionScheme) AddHardfork(ip *zcncore.InputMap) error { + ret := _m.Called(ip) + + var r0 error + if rf, ok := ret.Get(0).(func(*zcncore.InputMap) error); ok { + r0 = rf(ip) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // CancelAllocation provides a mock function with given fields: allocID func (_m *TransactionScheme) CancelAllocation(allocID string) error { ret := _m.Called(allocID) @@ -791,13 +805,12 @@ func (_m *TransactionScheme) ZCNSCUpdateGlobalConfig(_a0 *zcncore.InputMap) erro return r0 } -type mockConstructorTestingTNewTransactionScheme interface { +// NewTransactionScheme creates a new instance of TransactionScheme. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionScheme(t interface { mock.TestingT Cleanup(func()) -} - -// NewTransactionScheme creates a new instance of TransactionScheme. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewTransactionScheme(t mockConstructorTestingTNewTransactionScheme) *TransactionScheme { +}) *TransactionScheme { mock := &TransactionScheme{} mock.Mock.Test(t) diff --git a/zcncore/mocks/WalletCallback.go b/zcncore/mocks/WalletCallback.go index 267600db9..a86b8a3f6 100644 --- a/zcncore/mocks/WalletCallback.go +++ b/zcncore/mocks/WalletCallback.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.22.1. DO NOT EDIT. +// Code generated by mockery v2.36.0. DO NOT EDIT. package mocks @@ -14,13 +14,12 @@ func (_m *WalletCallback) OnWalletCreateComplete(status int, wallet string, err _m.Called(status, wallet, err) } -type mockConstructorTestingTNewWalletCallback interface { +// NewWalletCallback creates a new instance of WalletCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewWalletCallback(t interface { mock.TestingT Cleanup(func()) -} - -// NewWalletCallback creates a new instance of WalletCallback. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewWalletCallback(t mockConstructorTestingTNewWalletCallback) *WalletCallback { +}) *WalletCallback { mock := &WalletCallback{} mock.Mock.Test(t) diff --git a/zcncore/transaction.go b/zcncore/transaction.go index 043ffcbdd..fa123786c 100644 --- a/zcncore/transaction.go +++ b/zcncore/transaction.go @@ -146,6 +146,7 @@ type TransactionCommon interface { MinerScUpdateConfig(*InputMap) error MinerScUpdateGlobals(*InputMap) error StorageScUpdateConfig(*InputMap) error + AddHardfork(ip *InputMap) (err error) FaucetUpdateConfig(*InputMap) error ZCNSCUpdateGlobalConfig(*InputMap) error @@ -749,6 +750,16 @@ func (t *Transaction) StorageScUpdateConfig(ip *InputMap) (err error) { go func() { t.setNonceAndSubmit() }() return } +func (t *Transaction) AddHardfork(ip *InputMap) (err error) { + err = t.createSmartContractTxn(MinerSmartContractAddress, + transaction.ADD_HARDFORK, ip, 0) + if err != nil { + logging.Error(err) + return + } + go func() { t.setNonceAndSubmit() }() + return +} func (t *Transaction) ZCNSCUpdateGlobalConfig(ip *InputMap) (err error) { err = t.createSmartContractTxn(ZCNSCSmartContractAddress, transaction.ZCNSC_UPDATE_GLOBAL_CONFIG, ip, 0) diff --git a/zcncore/transactionauth.go b/zcncore/transactionauth.go index 44ed45916..d321732b4 100644 --- a/zcncore/transactionauth.go +++ b/zcncore/transactionauth.go @@ -467,6 +467,17 @@ func (ta *TransactionWithAuth) StorageScUpdateConfig(ip *InputMap) (err error) { return } +func (t *TransactionWithAuth) AddHardfork(ip *InputMap) (err error) { + err = t.t.createSmartContractTxn(MinerSmartContractAddress, + transaction.ADD_HARDFORK, ip, 0) + if err != nil { + logging.Error(err) + return + } + go func() { t.submitTxn() }() + return +} + func (ta *TransactionWithAuth) ZCNSCUpdateGlobalConfig(ip *InputMap) (err error) { err = ta.t.createSmartContractTxn(ZCNSCSmartContractAddress, transaction.ZCNSC_UPDATE_GLOBAL_CONFIG, ip, 0) if err != nil { From 4eb00b55fcdf7773c8a3a9b539febfd24ae954ca Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Mon, 22 Jan 2024 02:13:57 +0530 Subject: [PATCH 09/12] fix older upload (#1362) --- zboxcore/sdk/chunked_upload_blobber.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zboxcore/sdk/chunked_upload_blobber.go b/zboxcore/sdk/chunked_upload_blobber.go index 4e3662a89..8b42c85f9 100644 --- a/zboxcore/sdk/chunked_upload_blobber.go +++ b/zboxcore/sdk/chunked_upload_blobber.go @@ -400,6 +400,6 @@ func (sb *ChunkedUploadBlobber) processWriteMarker( } size += change.GetSize() } - + rootRef.CalculateHash() return rootRef, lR.LatestWM, size, fileIDMeta, nil } From b885fa2cff83ca50c4db28b1a1aed9e280bcef6d Mon Sep 17 00:00:00 2001 From: Gaurang Patel Date: Mon, 22 Jan 2024 02:34:59 +0530 Subject: [PATCH 10/12] implemented zcnsccollectreward func (#1367) * implemented zcnsccollectreward func #1366 * Update entity.go --- core/transaction/entity.go | 1 + zcncore/mocks/TransactionCommon.go | 14 ++++++++++++++ zcncore/transaction.go | 17 +++++++++++++++++ zcncore/transactionauth.go | 15 +++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/core/transaction/entity.go b/core/transaction/entity.go index 4764c1794..912965259 100644 --- a/core/transaction/entity.go +++ b/core/transaction/entity.go @@ -167,6 +167,7 @@ const ( ZCNSC_ADD_AUTHORIZER = "add-authorizer" ZCNSC_AUTHORIZER_HEALTH_CHECK = "authorizer-health-check" ZCNSC_DELETE_AUTHORIZER = "delete-authorizer" + ZCNSC_COLLECT_REWARD = "collect-rewards" ESTIMATE_TRANSACTION_COST = `/v1/estimate_txn_fee` FEES_TABLE = `/v1/fees_table` diff --git a/zcncore/mocks/TransactionCommon.go b/zcncore/mocks/TransactionCommon.go index 942487cfa..47fc0fb80 100644 --- a/zcncore/mocks/TransactionCommon.go +++ b/zcncore/mocks/TransactionCommon.go @@ -578,3 +578,17 @@ func NewTransactionCommon(t interface { return mock } + +// ZCNSCCollectReward provides a mock function with given fields: providerID, providerType +func (_m *TransactionCommon) ZCNSCCollectReward(providerID string, providerType zcncore.Provider) error { + ret := _m.Called(providerID, providerType) + + var r0 error + if rf, ok := ret.Get(0).(func(string, zcncore.Provider) error); ok { + r0 = rf(providerID, providerType) + } else { + r0 = ret.Error(0) + } + + return r0 +} \ No newline at end of file diff --git a/zcncore/transaction.go b/zcncore/transaction.go index fa123786c..715cf5f6f 100644 --- a/zcncore/transaction.go +++ b/zcncore/transaction.go @@ -168,6 +168,8 @@ type TransactionCommon interface { // ZCNSCDeleteAuthorizer deletes authorizer ZCNSCDeleteAuthorizer(*DeleteAuthorizerPayload) error + + ZCNSCCollectReward(providerID string, providerType Provider) error } // PriceRange represents a price range allowed by user to filter blobbers. @@ -1488,3 +1490,18 @@ func (t *Transaction) ZCNSCDeleteAuthorizer(ip *DeleteAuthorizerPayload) (err er go t.setNonceAndSubmit() return } + +func (t *Transaction) ZCNSCCollectReward(providerId string, providerType Provider) error { + pr := &scCollectReward{ + ProviderId: providerId, + ProviderType: int(providerType), + } + err := t.createSmartContractTxn(ZCNSCSmartContractAddress, + transaction.ZCNSC_COLLECT_REWARD, pr, 0) + if err != nil { + logging.Error(err) + return err + } + go func() { t.setNonceAndSubmit() }() + return err +} diff --git a/zcncore/transactionauth.go b/zcncore/transactionauth.go index d321732b4..1561678a2 100644 --- a/zcncore/transactionauth.go +++ b/zcncore/transactionauth.go @@ -583,3 +583,18 @@ func (ta *TransactionWithAuth) ZCNSCDeleteAuthorizer(ip *DeleteAuthorizerPayload go ta.submitTxn() return } + +func (ta *TransactionWithAuth) ZCNSCCollectReward(providerId string, providerType Provider) error { + pr := &scCollectReward{ + ProviderId: providerId, + ProviderType: int(providerType), + } + err := ta.t.createSmartContractTxn(ZCNSCSmartContractAddress, + transaction.ZCNSC_COLLECT_REWARD, pr, 0) + if err != nil { + logging.Error(err) + return err + } + go func() { ta.t.setNonceAndSubmit() }() + return err +} From 0fbb7023d93c1601970433089ceea0b0731ba280 Mon Sep 17 00:00:00 2001 From: Manohar Reddy Date: Sun, 21 Jan 2024 22:08:54 +0100 Subject: [PATCH 11/12] fix shutdown blobber (#1374) --- zboxcore/sdk/sdk.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zboxcore/sdk/sdk.go b/zboxcore/sdk/sdk.go index 414a95e5e..e2d7970c5 100644 --- a/zboxcore/sdk/sdk.go +++ b/zboxcore/sdk/sdk.go @@ -1292,12 +1292,15 @@ func KillProvider(providerId string, providerType ProviderType) (string, int64, return hash, n, err } -func ShutdownProvider(providerType ProviderType) (string, int64, error) { +func ShutdownProvider(providerType ProviderType, providerID string) (string, int64, error) { if !sdkInitialized { return "", 0, sdkNotInitialized } - var input = map[string]interface{}{} + var input = map[string]interface{}{ + "provider_id": providerID, + } + var sn = transaction.SmartContractTxnData{ InputArgs: input, } From 6da82b9ad4fd14a132d51e76cb920b7fb1559402 Mon Sep 17 00:00:00 2001 From: Hitenjain14 <57557631+Hitenjain14@users.noreply.github.com> Date: Mon, 22 Jan 2024 03:29:13 +0530 Subject: [PATCH 12/12] fix repair for empty file (#1375) --- zboxcore/sdk/downloadworker.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 06ac4acb4..02177b10f 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -373,10 +373,13 @@ func (req *DownloadRequest) processDownload(ctx context.Context) { } fRef := req.fRef if fRef != nil && fRef.ActualFileHash == emptyFileDataHash { - if req.statusCallback != nil { - req.statusCallback.Completed( - req.allocationID, remotePathCB, fRef.Name, "", len(emptyFileDataHash), op) + logger.Logger.Info("File is empty") + _, err := req.fileHandler.Write([]byte(emptyFileDataHash)) + if err != nil { + req.errorCB(errors.Wrap(err, "Write file failed"), remotePathCB) + return } + req.fileHandler.Sync() //nolint return } size, chunksPerShard, blocksPerShard := req.size, req.chunksPerShard, req.blocksPerShard