Merge pull request #1517 from 0chain/feat/wasm-worker

Web workers in wasm

Makefile

@@ -38,13 +38,13 @@ wasm-build: getrev
 	CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -ldflags="-s -w" -buildvcs=false -o ./zcn.wasm  ./wasmsdk
 
 wasm-test: wasm-build
-	env -i $(shell go env) PATH="$(shell go env GOROOT)/misc/wasm:$(PATH)" CGO_ENABLED=0 GOOS=js GOARCH=wasm go test -v github.com/0chain/gosdk/wasmsdk/jsbridge/...
-
+	env -i $(shell go env) PATH="$(shell go env GOROOT)/misc/wasm:$(PATH)" CGO_ENABLED=0 GOOS=js GOARCH=wasm go test -v ./wasmsdk/jsbridge/input_test.go ./wasmsdk/jsbridge/input.go ./wasmsdk/jsbridge/object.go ./wasmsdk/jsbridge/error.go ./wasmsdk/jsbridge/func.go ./wasmsdk/jsbridge/func_test.go ./wasmsdk/jsbridge/sync.go ./wasmsdk/jsbridge/async.go ./wasmsdk/jsbridge/output.go ./wasmsdk/jsbridge/vars.go
+ 
 gosdk-mocks:
 	./generate_mocks.sh
 
 gosdk-test:
-	go test -v -tags bn256 -p 1 ./...
+	go test -tags bn256 -p 1 ./...
 
 install-gosdk: | gosdk-build wasm-build
 

core/version/version.go

@@ -2,5 +2,5 @@
 //====== THIS IS AUTOGENERATED FILE. DO NOT MODIFY ========
 
 package version
-const VERSIONSTR = "v1.14.0-RC7"
+const VERSIONSTR = "v1.15.1-17-g1364fe53"
 

go.mod

@@ -44,6 +44,7 @@ require (
 )
 
 require (
+	github.com/hack-pad/go-webworkers v0.1.0
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
 	github.com/minio/sha256-simd v1.0.1
 	github.com/ybbus/jsonrpc/v3 v3.1.5
@@ -130,6 +131,7 @@ require (
 
 require (
 	github.com/btcsuite/btcd/btcutil v1.1.3
+	github.com/hack-pad/safejs v0.1.1
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/remeh/sizedwaitgroup v1.0.0
@@ -141,4 +143,4 @@ require (
 	google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44 // indirect
 )
 
-//replace github.com/ethereum/go-ethereum => github.com/certifaction/go-ethereum v1.10.3-wasm
+//replace github.com/ethereum/go-ethereum => github.com/certifaction/go-ethereum v1.10.3-wasm

go.sum

@@ -285,6 +285,10 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb h1:GlQyMv2C48qmfPItvAXFoyN341Swxp9JNVeUZxnmbJw=
 github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
+github.com/hack-pad/go-webworkers v0.1.0 h1:QHBJpkXJgW0QRi2iiUGcxwGnmy7lQJL0F8UfsgMXKhA=
+github.com/hack-pad/go-webworkers v0.1.0/go.mod h1:/rmjjgnlw0CursmeqRtP0NGIqo8CR+0o6AtzFydUHJ4=
+github.com/hack-pad/safejs v0.1.1 h1:d5qPO0iQ7h2oVtpzGnLExE+Wn9AtytxIfltcS2b9KD8=
+github.com/hack-pad/safejs v0.1.1/go.mod h1:HdS+bKF1NrE72VoXZeWzxFOVQVUSqZJAG0xNCnb+Tio=
 github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
 github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=

wasmsdk/allocation.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 
 	"github.com/0chain/gosdk/core/transaction"
+	"github.com/0chain/gosdk/wasmsdk/jsbridge"
 	"github.com/0chain/gosdk/zboxcore/sdk"
 )
 
@@ -195,12 +196,24 @@ func updateAllocationWithRepair(allocationID string,
 	statusBar := &StatusBar{wg: wg, isRepair: true}
 	wg.Add(1)
 
-	hash, err := allocationObj.UpdateWithRepair(size, extend, uint64(lock), addBlobberId, addBlobberAuthTicket, removeBlobberId, false, &sdk.FileOptionsParameters{}, statusBar)
-	if err == nil {
-		clearAllocation(allocationID)
+	alloc, hash, isRepairRequired, err := allocationObj.UpdateWithStatus(size, extend, uint64(lock), addBlobberId, addBlobberAuthTicket, removeBlobberId, false, &sdk.FileOptionsParameters{}, statusBar)
+	if err != nil {
+		return hash, err
+	}
+	clearAllocation(allocationID)
+
+	if isRepairRequired {
+		addWebWorkers(alloc)
+		if removeBlobberId != "" {
+			jsbridge.RemoveWorker(removeBlobberId)
+		}
+		err := alloc.RepairAlloc(statusBar)
+		if err != nil {
+			return "", err
+		}
 		wg.Wait()
 		if statusBar.err != nil {
-			return hash, statusBar.err
+			return "", statusBar.err
 		}
 	}
 

wasmsdk/blobber.go

@@ -149,11 +149,12 @@ func Delete(allocationID, remotePath string) (*FileCommandResponse, error) {
 		return nil, err
 	}
 
-	err = allocationObj.DeleteFile(remotePath)
-	if err != nil {
-		return nil, err
-	}
-
+	err = allocationObj.DoMultiOperation([]sdk.OperationRequest{
+		{
+			OperationType: constants.FileOperationDelete,
+			RemotePath:    remotePath,
+		},
+	})
 	sdkLogger.Info(remotePath + " deleted")
 
 	resp := &FileCommandResponse{
@@ -404,6 +405,7 @@ func multiDownload(allocationID, jsonMultiDownloadOptions, authTicket, callbackF
 		}
 		var mf sys.File
 		if option.DownloadToDisk {
+			terminateWorkersWithAllocation(alloc)
 			mf, err = jsbridge.NewFileWriter(fileName)
 			if err != nil {
 				PrintError(err.Error())
@@ -477,6 +479,7 @@ type BulkUploadOption struct {
 	FileSize          int64  `json:"fileSize,omitempty"`
 	ReadChunkFuncName string `json:"readChunkFuncName,omitempty"`
 	CallbackFuncName  string `json:"callbackFuncName,omitempty"`
+	Md5HashFuncName   string `json:"md5HashFuncName,omitempty"`
 	MimeType          string `json:"mimeType,omitempty"`
 	MemoryStorer      bool   `json:"memoryStorer,omitempty"`
 }
@@ -626,6 +629,7 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) {
 		result.Success = false
 		return result, errors.New("Error fetching the allocation")
 	}
+	addWebWorkers(allocationObj)
 	operationRequests := make([]sdk.OperationRequest, n)
 	for idx, option := range options {
 		wg := &sync.WaitGroup{}
@@ -692,6 +696,10 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) {
 				list: make(map[string]*sdk.UploadProgress),
 			}))
 		}
+		if option.Md5HashFuncName != "" {
+			fileHasher := newFileHasher(option.Md5HashFuncName)
+			options = append(options, sdk.WithFileHasher(fileHasher))
+		}
 		operationRequests[idx] = sdk.OperationRequest{
 			FileMeta:       fileMeta,
 			FileReader:     fileReader,
@@ -991,3 +999,53 @@ func skipStatusCheck(allocationID string, checkStatus bool) error {
 	alloc.SetCheckStatus(checkStatus)
 	return nil
 }
+
+func terminateWorkers(allocationID string) {
+	alloc, err := getAllocation(allocationID)
+	if err != nil {
+		return
+	}
+	for _, blobber := range alloc.Blobbers {
+		jsbridge.RemoveWorker(blobber.ID)
+	}
+}
+
+func terminateWorkersWithAllocation(alloc *sdk.Allocation) {
+	for _, blobber := range alloc.Blobbers {
+		jsbridge.RemoveWorker(blobber.ID)
+	}
+}
+
+func createWorkers(allocationID string) {
+	alloc, err := getAllocation(allocationID)
+	if err != nil {
+		return
+	}
+	addWebWorkers(alloc)
+}
+
+func startListener() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	selfWorker, err := jsbridge.NewSelfWorker()
+	if err != nil {
+		return err
+	}
+
+	listener, err := selfWorker.Listen(ctx)
+	if err != nil {
+		return err
+	}
+	sdk.InitHasherMap()
+	for event := range listener {
+		data, err := event.Data()
+		if err != nil {
+			PrintError("Error in getting data from event", err)
+			return err
+		}
+		sdk.ProcessEventData(data)
+	}
+
+	return nil
+}

wasmsdk/cache.go

@@ -6,6 +6,8 @@ package main
 import (
 	"time"
 
+	"github.com/0chain/gosdk/wasmsdk/jsbridge"
+	"github.com/0chain/gosdk/zboxcore/client"
 	"github.com/0chain/gosdk/zboxcore/sdk"
 	lru "github.com/hashicorp/golang-lru/v2"
 )
@@ -36,7 +38,10 @@ func getAllocation(allocationId string) (*sdk.Allocation, error) {
 	sdk.SetShouldVerifyHash(false)
 	it = &cachedAllocation{
 		Allocation: a,
-		Expiration: time.Now().Add(5 * time.Minute),
+		Expiration: time.Now().Add(120 * time.Minute),
+	}
+	if !ok {
+		addWebWorkers(a)
 	}
 
 	cachedAllocations.Add(allocationId, it)
@@ -63,3 +68,10 @@ func reloadAllocation(allocationID string) (*sdk.Allocation, error) {
 
 	return it.Allocation, nil
 }
+
+func addWebWorkers(alloc *sdk.Allocation) {
+	c := client.GetClient()
+	for _, blober := range alloc.Blobbers {
+		jsbridge.NewWasmWebWorker(blober.ID, blober.Baseurl, c.ClientID, c.Keys[0].PublicKey, c.Keys[0].PrivateKey, c.Mnemonic) //nolint:errcheck
+	}
+}

wasmsdk/common.go

@@ -4,6 +4,10 @@
 package main
 
 import (
+	"errors"
+	"syscall/js"
+
+	"github.com/0chain/gosdk/wasmsdk/jsbridge"
 	"github.com/0chain/gosdk/zboxcore/sdk"
 )
 
@@ -42,3 +46,46 @@ func getFileMeta(allocationObj *sdk.Allocation, remotePath string, commit bool)
 
 	return fileMeta, isFile, nil
 }
+
+type hasher struct {
+	md5HashFuncName string
+}
+
+func newFileHasher(md5HashFuncName string) sdk.Hasher {
+	return &hasher{
+		md5HashFuncName: md5HashFuncName,
+	}
+}
+
+func (h *hasher) GetFileHash() (string, error) {
+	md5Callback := js.Global().Get(h.md5HashFuncName)
+	result, err := jsbridge.Await(md5Callback.Invoke())
+	if len(err) > 0 && !err[0].IsNull() {
+		return "", errors.New("file_hash: " + err[0].String())
+	}
+	return result[0].String(), nil
+}
+
+func (h *hasher) WriteToFile(_ []byte) error {
+	return nil
+}
+
+func (h *hasher) GetFixedMerkleRoot() (string, error) {
+	return "", nil
+}
+
+func (h *hasher) WriteToFixedMT(_ []byte) error {
+	return nil
+}
+
+func (h *hasher) GetValidationRoot() (string, error) {
+	return "", nil
+}
+
+func (h *hasher) WriteToValidationMT(_ []byte) error {
+	return nil
+}
+
+func (h *hasher) Finalize() error {
+	return nil
+}

wasmsdk/demo/index.html

@@ -2,7 +2,7 @@
 
 <head>
   <!-- for zcn.wasm-->
-  <script src="https://cdn.jsdelivr.net/gh/herumi/bls-wasm@v1.0.0/browser/bls.js"></script>
+  <script src="https://cdn.jsdelivr.net/gh/herumi/bls-wasm@v1.1.1/browser/bls.js"></script>
   <script src="https://cdn.jsdelivr.net/gh/golang/[email protected]/misc/wasm/wasm_exec.js"></script>
   <script src="zcn.js"></script>
 
@@ -211,7 +211,7 @@ <h2>please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates
 
     let network = query.get('network')
     if (!network || network == 'undefined') {
-      network = "mainnet.zus.network"
+      network = "mob.zus.network"
     }
 
     const blockWorker = 'https://' + network + '/dns';
@@ -457,12 +457,12 @@ <h2>please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates
             allocationId: allocationId,
             remotePath: `/${file.name}`,
             file: file,
-            thumbnailBytes: await readBytes(file),//only for demo, don't upload original file as thumbnail in production
+            // thumbnailBytes: await readBytes(file),//only for demo, don't upload original file as thumbnail in production
             encrypt: false,
             webstreaming: false,
             isUpdate: false,
             isRepair: false,
-            numBlocks: 100,
+            numBlocks: 120,
             callback: function (totalBytes, completedBytes, error) {
               console.log(file.name + " " + completedBytes + "/" + totalBytes + " err:" + error)
             }

wasmsdk/demo/md5worker.js

@@ -0,0 +1,19 @@
+importScripts('https://rawgit.com/satazor/SparkMD5/master/spark-md5.min.js');
+
+self.addEventListener('message', (e) => {
+    const file = e.data;
+    const chunkSize = 128 * 1024;// 128KB
+    //create fileReaderSync
+    const totalParts = Math.ceil(file.size / chunkSize);
+    const fileReaderSync = new FileReaderSync();
+    const spark = new self.SparkMD5.ArrayBuffer();
+    for (let i = 0; i < totalParts; i++) {
+        const start = i * chunkSize;
+        const end = Math.min(file.size, start + chunkSize);
+        const buffer = fileReaderSync.readAsArrayBuffer(file.slice(start, end));
+
+        spark.append(buffer);
+    }
+    const hash = spark.end();
+    self.postMessage(hash);
+});