Fix memory usage

core/encryption/hash.go

@@ -4,7 +4,7 @@ import (
 	"crypto/sha1"
 	"encoding/hex"
 
-	"github.com/zeebo/blake3"
+	"github.com/minio/sha256-simd"
 	"golang.org/x/crypto/sha3"
 )
 
@@ -36,7 +36,7 @@ func RawHash(data interface{}) []byte {
 	return hash.Sum(buf)
 }
 
-func BlakeHash(data interface{}) []byte {
+func ShaHash(data interface{}) []byte {
 	var databuf []byte
 	switch dataImpl := data.(type) {
 	case []byte:
@@ -48,7 +48,7 @@ func BlakeHash(data interface{}) []byte {
 	default:
 		panic("unknown type")
 	}
-	hash := blake3.New()
+	hash := sha256.New()
 	_, _ = hash.Write(databuf)
 	return hash.Sum(nil)
 }

core/util/fixed_merkle_tree.go

@@ -10,7 +10,7 @@ import (
 	goError "errors"
 
 	"github.com/0chain/errors"
-	"github.com/zeebo/blake3"
+	"github.com/minio/sha256-simd"
 )
 
 const (
@@ -20,6 +20,16 @@ const (
 	FixedMTDepth        = 11
 )
 
+var (
+	leafPool = sync.Pool{
+		New: func() interface{} {
+			return &leaf{
+				h: sha256.New(),
+			}
+		},
+	}
+)
+
 type leaf struct {
 	h hash.Hash
 }
@@ -37,9 +47,14 @@ func (l *leaf) Write(b []byte) (int, error) {
 }
 
 func getNewLeaf() *leaf {
-	return &leaf{
-		h: blake3.New(),
+	l, ok := leafPool.Get().(*leaf)
+	if !ok {
+		return &leaf{
+			h: sha256.New(),
+		}
 	}
+	l.h.Reset()
+	return l
 }
 
 // FixedMerkleTree A trusted mekerle tree for outsourcing attack protection. see section 1.8 on whitepager
@@ -165,6 +180,7 @@ func (fmt *FixedMerkleTree) CalculateMerkleRoot() {
 	nodes := make([][]byte, len(fmt.Leaves))
 	for i := 0; i < len(nodes); i++ {
 		nodes[i] = fmt.Leaves[i].GetHashBytes()
+		leafPool.Put(fmt.Leaves[i])
 	}
 
 	for i := 0; i < FixedMTDepth; i++ {

core/util/merkle_tree_interface.go

@@ -4,7 +4,7 @@ import (
 	"encoding/hex"
 
 	"github.com/0chain/gosdk/core/encryption"
-	"github.com/zeebo/blake3"
+	"github.com/minio/sha256-simd"
 )
 
 /*MerkleTreeI - a merkle tree interface required for constructing and providing verification */
@@ -41,7 +41,7 @@ func MHashBytes(h1, h2 []byte) []byte {
 	buf := make([]byte, len(h1)+len(h2))
 	copy(buf, h1)
 	copy(buf[len(h1):], h2)
-	hash := blake3.New()
+	hash := sha256.New()
 	_, _ = hash.Write(buf)
 	return hash.Sum(nil)
 }

core/util/validation_tree.go

@@ -9,7 +9,7 @@ import (
 	"math"
 	"sync"
 
-	"github.com/zeebo/blake3"
+	"github.com/minio/sha256-simd"
 )
 
 const (
@@ -99,7 +99,7 @@ func (v *ValidationTree) calculateRoot() {
 	depth := v.CalculateDepth()
 	nodes := make([][]byte, totalLeaves)
 	copy(nodes, v.leaves)
-	h := blake3.New()
+	h := sha256.New()
 
 	for i := 0; i < depth; i++ {
 		if len(nodes) == 1 {
@@ -154,7 +154,7 @@ func NewValidationTree(dataSize int64) *ValidationTree {
 
 	return &ValidationTree{
 		dataSize: dataSize,
-		h:        blake3.New(),
+		h:        sha256.New(),
 		leaves:   make([][]byte, totalLeaves),
 	}
 }
@@ -223,7 +223,7 @@ If client had required data from 2-9 then blobber would have to provide:
 func (m *MerklePathForMultiLeafVerification) VerifyMultipleBlocks(data []byte) error {
 
 	hashes := make([][]byte, 0)
-	h := blake3.New()
+	h := sha256.New()
 	// Calculate hashes from the data responded from the blobber.
 	for i := 0; i < len(data); i += MaxMerkleLeavesSize {
 		endIndex := i + MaxMerkleLeavesSize
@@ -270,7 +270,7 @@ func (m *MerklePathForMultiLeafVerification) VerifyMultipleBlocks(data []byte) e
 
 func (m *MerklePathForMultiLeafVerification) calculateIntermediateHashes(hashes [][]byte) [][]byte {
 	newHashes := make([][]byte, 0)
-	h := blake3.New()
+	h := sha256.New()
 	if len(hashes)%2 == 0 {
 		for i := 0; i < len(hashes); i += 2 {
 			h.Reset()

core/util/validation_tree_test.go

@@ -8,8 +8,8 @@ import (
 	"math"
 	"testing"
 
+	"github.com/minio/sha256-simd"
 	"github.com/stretchr/testify/require"
-	"github.com/zeebo/blake3"
 )
 
 const (
@@ -144,7 +144,7 @@ func calculateValidationMerkleRoot(data []byte) []byte {
 		if j > len(data) {
 			j = len(data)
 		}
-		h := blake3.New()
+		h := sha256.New()
 		_, _ = h.Write(data[i:j])
 		hashes = append(hashes, h.Sum(nil))
 	}
@@ -156,19 +156,19 @@ func calculateValidationMerkleRoot(data []byte) []byte {
 		newHashes := make([][]byte, 0)
 		if len(hashes)%2 == 0 {
 			for i := 0; i < len(hashes); i += 2 {
-				h := blake3.New()
+				h := sha256.New()
 				_, _ = h.Write(hashes[i])
 				_, _ = h.Write(hashes[i+1])
 				newHashes = append(newHashes, h.Sum(nil))
 			}
 		} else {
 			for i := 0; i < len(hashes)-1; i += 2 {
-				h := blake3.New()
+				h := sha256.New()
 				_, _ = h.Write(hashes[i])
 				_, _ = h.Write(hashes[i+1])
 				newHashes = append(newHashes, h.Sum(nil))
 			}
-			h := blake3.New()
+			h := sha256.New()
 			_, _ = h.Write(hashes[len(hashes)-1])
 			newHashes = append(newHashes, h.Sum(nil))
 		}
@@ -191,7 +191,7 @@ func calculateValidationRootAndNodes(b []byte, startInd, endInd int) (
 
 	hashes := make([][]byte, 0)
 	nodesData := make([]byte, 0)
-	h := blake3.New()
+	h := sha256.New()
 	for i := 0; i < len(b); i += MaxMerkleLeavesSize {
 		j := i + MaxMerkleLeavesSize
 		if j > len(b) {
@@ -212,7 +212,7 @@ func calculateValidationRootAndNodes(b []byte, startInd, endInd int) (
 		newHashes := make([][]byte, 0)
 		if len(hashes)%2 == 0 {
 			for i := 0; i < len(hashes); i += 2 {
-				h := blake3.New()
+				h := sha256.New()
 				_, _ = h.Write(hashes[i])
 				_, _ = h.Write(hashes[i+1])
 				nodesData = append(nodesData, hashes[i]...)
@@ -221,14 +221,14 @@ func calculateValidationRootAndNodes(b []byte, startInd, endInd int) (
 			}
 		} else {
 			for i := 0; i < len(hashes)-1; i += 2 {
-				h := blake3.New()
+				h := sha256.New()
 				_, _ = h.Write(hashes[i])
 				_, _ = h.Write(hashes[i+1])
 				nodesData = append(nodesData, hashes[i]...)
 				nodesData = append(nodesData, hashes[i+1]...)
 				newHashes = append(newHashes, h.Sum(nil))
 			}
-			h := blake3.New()
+			h := sha256.New()
 			_, _ = h.Write(hashes[len(hashes)-1])
 			nodesData = append(nodesData, hashes[len(hashes)-1]...)
 			newHashes = append(newHashes, h.Sum(nil))

go.mod

@@ -45,7 +45,6 @@ require (
 require (
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
 	github.com/minio/sha256-simd v1.0.1
-	github.com/zeebo/blake3 v0.2.3
 )
 
 require (

go.sum

@@ -339,7 +339,6 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
 github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
 github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
 github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
@@ -540,12 +539,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
-github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
-github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
-github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
-github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
-github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
 go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs=
 go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw=
 go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ=

wasmsdk/blobber.go

@@ -624,7 +624,7 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) {
 			numBlocks = 100
 		}
 		if allocationObj.DataShards > 7 {
-			numBlocks = 50
+			numBlocks = 75
 		}
 		options := []sdk.ChunkedUploadOption{
 			sdk.WithThumbnail(option.ThumbnailBytes.Buffer),

zboxcore/sdk/allocation.go

@@ -237,8 +237,8 @@ func GetWritePriceRange() (PriceRange, error) {
 
 func SetWasm() {
 	IsWasm = true
-	BatchSize = 3
-	MultiOpBatchSize = 5
+	BatchSize = 5
+	MultiOpBatchSize = 7
 }
 
 func getPriceRange(name string) (PriceRange, error) {

zboxcore/sdk/chunked_upload_hasher.go

@@ -1,12 +1,11 @@
 package sdk
 
 import (
+	"crypto/md5"
 	"encoding/hex"
 	"hash"
 	"sync"
 
-	"github.com/minio/sha256-simd"
-
 	"github.com/0chain/errors"
 	"github.com/0chain/gosdk/constants"
 	"github.com/0chain/gosdk/core/util"
@@ -39,7 +38,7 @@ type hasher struct {
 // CreateHasher creat Hasher instance
 func CreateHasher(dataSize int64) Hasher {
 	return &hasher{
-		File:         sha256.New(),
+		File:         md5.New(),
 		FixedMT:      util.NewFixedMerkleTree(),
 		ValidationMT: util.NewValidationTree(dataSize),
 	}

zboxcore/sdk/chunked_upload_option.go

@@ -1,14 +1,14 @@
 package sdk
 
 import (
+	"crypto/md5"
 	"encoding/hex"
 	"math"
 	"os"
 	"time"
 
 	"github.com/0chain/gosdk/zboxcore/zboxutil"
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/sha256-simd"
 )
 
 // ChunkedUploadOption set stream option
@@ -26,7 +26,7 @@ func WithThumbnail(buf []byte) ChunkedUploadOption {
 			su.thumbnailBytes = buf
 			su.fileMeta.ActualThumbnailSize = int64(len(buf))
 
-			thumbnailHasher := sha256.New()
+			thumbnailHasher := md5.New()
 			thumbnailHasher.Write(buf)
 
 			su.fileMeta.ActualThumbnailHash = hex.EncodeToString(thumbnailHasher.Sum(nil))

zboxcore/sdk/downloadworker.go

@@ -3,6 +3,7 @@ package sdk
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -17,8 +18,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/minio/sha256-simd"
-
 	"github.com/0chain/errors"
 	"github.com/0chain/gosdk/core/common"
 	"github.com/0chain/gosdk/core/sys"
@@ -428,7 +427,7 @@ func (req *DownloadRequest) processDownload(ctx context.Context) {
 	)
 
 	if !req.shouldVerify && (startBlock == 0 && endBlock == chunksPerShard) {
-		actualFileHasher = sha256.New()
+		actualFileHasher = md5.New()
 		hashDataChan = make(chan []byte, n)
 		hashWg = &sync.WaitGroup{}
 		hashWg.Add(1)

zboxcore/zboxutil/transport.go

@@ -21,6 +21,5 @@ var DefaultTransport = &http.Transport{
 	TLSHandshakeTimeout:   10 * time.Second,
 	ExpectContinueTimeout: 1 * time.Second,
 	MaxIdleConnsPerHost:   100,
-	ReadBufferSize:        34 * 1024 * 1024,
-	WriteBufferSize:       34 * 1024 * 1024,
+	WriteBufferSize:       256 * 1024,
 }