init buffer in first read

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.2
 	github.com/hashicorp/golang-lru/v2 v2.0.1
 	github.com/herumi/bls-go-binary v1.31.0
-	github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9
+	github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c
 	github.com/influxdata/influxdb v1.8.3
 	github.com/klauspost/reedsolomon v1.11.8
 	github.com/lithammer/shortuuid/v3 v3.0.7
@@ -47,6 +47,7 @@ require (
 	github.com/hack-pad/go-webworkers v0.1.0
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
 	github.com/minio/sha256-simd v1.0.1
+	github.com/valyala/bytebufferpool v1.0.0
 	github.com/ybbus/jsonrpc/v3 v3.1.5
 )
 
@@ -116,7 +117,6 @@ require (
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.51.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.dedis.ch/fixbuf v1.0.3 // indirect
@@ -143,4 +143,4 @@ require (
 	google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44 // indirect
 )
 
-//replace github.com/ethereum/go-ethereum => github.com/certifaction/go-ethereum v1.10.3-wasm
+//replace github.com/ethereum/go-ethereum => github.com/certifaction/go-ethereum v1.10.3-wasm

go.sum

@@ -308,8 +308,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/herumi/bls-go-binary v1.31.0 h1:L1goQ2tMtGgpXCg5AwHAdJQpLs/pfnWWEc3Wog6OhmI=
 github.com/herumi/bls-go-binary v1.31.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y=
-github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 h1:Z6Mu2JCsW2hbqx91L0HNPRPQ10RyAFvPocQHlrRo1Jk=
-github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk=
+github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c h1:lDSIbcLu5TdT+uwb4wPzZgo1pQvKjP/tArL5QKjDJdI=
+github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk=
 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
 github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
 github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8=

wasmsdk/jsbridge/file_reader.go

@@ -5,7 +5,6 @@ package jsbridge
 
 import (
 	"errors"
-	"fmt"
 	"io"
 	"syscall/js"
 
@@ -67,7 +66,6 @@ func (r *FileReader) Read(p []byte) (int, error) {
 		buff := &bytebufferpool.ByteBuffer{
 			B: r.buf,
 		}
-		fmt.Println("put reader buffer: ", cap(buff.B))
 		common.MemPool.Put(buff)
 		return n, io.EOF
 	}
@@ -82,7 +80,6 @@ func (r *FileReader) initBuffer() error {
 	}
 	buff := common.MemPool.Get()
 	if cap(buff.B) < int(bufSize) {
-		fmt.Println("creating reader buffer: ", cap(buff.B), bufSize)
 		buff.B = make([]byte, bufSize)
 	}
 	r.buf = buff.B[:bufSize]

zboxcore/sdk/chunked_upload_blobber.go

@@ -131,7 +131,7 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest(
 				buff := &bytebufferpool.ByteBuffer{
 					B: dataBuffers[ind].Bytes(),
 				}
-				uploadPool.Put(buff)
+				formDataPool.Put(buff)
 
 				if err != nil {
 					return err

zboxcore/sdk/chunked_upload_chunk_reader.go

@@ -1,7 +1,6 @@
 package sdk
 
 import (
-	"fmt"
 	"io"
 	"math"
 	"strconv"
@@ -10,12 +9,16 @@ import (
 	"github.com/0chain/errors"
 	"github.com/0chain/gosdk/constants"
 	"github.com/0chain/gosdk/zboxcore/encryption"
+	"github.com/0chain/gosdk/zboxcore/logger"
 	"github.com/0chain/gosdk/zboxcore/zboxutil"
 	"github.com/klauspost/reedsolomon"
 	"github.com/valyala/bytebufferpool"
 )
 
-var uploadPool bytebufferpool.Pool
+var (
+	uploadPool   bytebufferpool.Pool
+	formDataPool bytebufferpool.Pool
+)
 
 type ChunkedUploadChunkReader interface {
 	// Next read, encode and encrypt next chunk
@@ -59,7 +62,8 @@ type chunkedUploadChunkReader struct {
 	fileShardsDataBuffer *bytebufferpool.ByteBuffer
 
 	//offset
-	offset int64
+	offset      int64
+	chunkNumber int64
 
 	// nextChunkIndex next index for reading
 	nextChunkIndex int
@@ -114,6 +118,7 @@ func createChunkReader(fileReader io.Reader, size, chunkSize int64, dataShards,
 		hasher:          hasher,
 		hasherDataChan:  make(chan []byte, 3*chunkNumber),
 		hasherWG:        sync.WaitGroup{},
+		chunkNumber:     int64(chunkNumber),
 	}
 
 	if r.encryptOnUpload {
@@ -126,22 +131,6 @@ func createChunkReader(fileReader io.Reader, size, chunkSize int64, dataShards,
 
 	r.chunkDataSizePerRead = r.chunkDataSize * int64(dataShards)
 	r.totalChunkDataSizePerRead = r.chunkDataSize * int64(dataShards+parityShards)
-	totalDataSize := r.totalChunkDataSizePerRead * int64(chunkNumber)
-	readSize := r.chunkDataSizePerRead * int64(chunkNumber)
-	if size > 0 && readSize > size {
-		chunkNum := (size + r.chunkDataSizePerRead - 1) / r.chunkDataSizePerRead
-		totalDataSize = r.totalChunkDataSizePerRead * chunkNum
-	}
-	buf := uploadPool.Get()
-	if cap(buf.B) < int(totalDataSize) {
-		buf.B = make([]byte, 0, totalDataSize)
-		// logger.Logger.Debug("creating buffer with size: ", " totalDataSize: ", totalDataSize)
-		fmt.Println("creating buffer with size: ", totalDataSize, cap(buf.B))
-	} else {
-		// logger.Logger.Debug("reusing buffer with size: ", cap(buf.B), " totalDataSize: ", totalDataSize, " len: ", len(buf.B))
-		fmt.Println("reusing buffer with size: ", cap(buf.B), totalDataSize)
-	}
-	r.fileShardsDataBuffer = buf
 	if CurrentMode == UploadModeHigh {
 		r.hasherWG.Add(1)
 		go r.hashData()
@@ -178,6 +167,23 @@ func (r *chunkedUploadChunkReader) Next() (*ChunkData, error) {
 		return nil, errors.Throw(constants.ErrInvalidParameter, "r")
 	}
 
+	if r.fileShardsDataBuffer == nil {
+		totalDataSize := r.totalChunkDataSizePerRead * r.chunkNumber
+		readSize := r.chunkDataSizePerRead * r.chunkNumber
+		if r.size > 0 && readSize > r.size {
+			chunkNum := (r.size + r.chunkDataSizePerRead - 1) / r.chunkDataSizePerRead
+			totalDataSize = r.totalChunkDataSizePerRead * chunkNum
+		}
+		buf := uploadPool.Get()
+		if cap(buf.B) < int(totalDataSize) {
+			logger.Logger.Debug("creating buffer with size: ", " totalDataSize: ", totalDataSize)
+			buf.B = make([]byte, 0, totalDataSize)
+		} else {
+			logger.Logger.Debug("reusing buffer with size: ", cap(buf.B), " totalDataSize: ", totalDataSize, " len: ", len(buf.B))
+		}
+		r.fileShardsDataBuffer = buf
+	}
+
 	chunk := &ChunkData{
 		Index:   r.nextChunkIndex,
 		IsFinal: false,
@@ -309,9 +315,7 @@ func (r *chunkedUploadChunkReader) Close() {
 	r.closeOnce.Do(func() {
 		close(r.hasherDataChan)
 		r.hasherWG.Wait()
-		fmt.Println("put buffer in pool: ", cap(r.fileShardsDataBuffer.B))
 		uploadPool.Put(r.fileShardsDataBuffer)
-		r.fileShardsDataBuffer = nil
 	})
 
 }

zboxcore/sdk/chunked_upload_form_builder.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/hex"
 	"encoding/json"
-	"fmt"
 	"io"
 	"mime/multipart"
 	"sync"
@@ -99,10 +98,9 @@ func (b *chunkedUploadFormBuilder) Build(
 		if endRange > len(fileChunksData) {
 			endRange = len(fileChunksData)
 		}
-		buff := uploadPool.Get()
+		buff := formDataPool.Get()
 		bufSize := (CHUNK_SIZE * (endRange - startRange)) + 1024
 		if cap(buff.B) < bufSize {
-			fmt.Println("creating form buffer: ", cap(buff.B), bufSize)
 			buff.B = make([]byte, 0, bufSize)
 		}
 		bodyBuf := buff.B

zboxcore/sdk/chunked_upload_process_js.go

@@ -672,8 +672,7 @@ func sendUploadRequest(dataBuffers []*bytes.Buffer, contentSlice []string, blobb
 				buff := &bytebufferpool.ByteBuffer{
 					B: dataBuffers[ind].Bytes(),
 				}
-				fmt.Println("put form buffer: ", cap(buff.B))
-				uploadPool.Put(buff)
+				formDataPool.Put(buff)
 
 				if err != nil {
 					return err