Update gosdk for enterprise blobbers

cmd/disk-cache.go

@@ -255,9 +255,9 @@
 	// fetch diskCache if object is currently cached or nearest available cache drive
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
+		log.Println("errorGettingLoc: ", err)
 		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
-
 	cacheReader, numCacheHits, cacheErr := dcache.Get(ctx, bucket, object, rs, h, opts)
 	if cacheErr == nil {
 		cacheObjSize = cacheReader.ObjInfo.Size
@@ -941,6 +941,7 @@
 	}
 	cReader, _, bErr := dcache.Get(ctx, oi.Bucket, oi.Name, nil, http.Header{}, ObjectOptions{})
 	if bErr != nil {
+		log.Println("errorGettingReader: ", bErr)
 		return
 	}
 	defer cReader.Close()
@@ -1014,6 +1015,11 @@
 	if err != nil {
 		return nil, err
 	}
+	uploadGoPool, err := ants.NewMultiPool(10, 40, ants.RoundRobin)
+	if err != nil {
+		return nil, err
+	}
+
 	c := &cacheObjects{
 		cache:                   cache,
 		exclude:                 config.Exclude,

cmd/gateway/zcn/dStorage.go

@@ -25,7 +25,7 @@ import (
 var tempdir string
 
 const (
-	pageLimit = 100
+	pageLimit = 500
 	numBlocks = 100
 	dirType   = "d"
 	fileType  = "f"
@@ -376,7 +376,7 @@ func getFileReader(ctx context.Context,
 
 }
 
-func putFile(ctx context.Context, alloc *sdk.Allocation, remotePath, contentType string, r io.Reader, size int64, isUpdate bool, userDefined map[string]string) (err error) {
+func putFile(ctx context.Context, alloc *sdk.Allocation, remotePath, contentType string, r io.Reader, size int64, _ bool, userDefined map[string]string) (err error) {
 	fileName := filepath.Base(remotePath)
 	var customMeta string
 	if len(userDefined) > 0 {
@@ -409,9 +409,7 @@ func putFile(ctx context.Context, alloc *sdk.Allocation, remotePath, contentType
 		},
 		StreamUpload: isStreamUpload,
 	}
-	if isUpdate {
-		opRequest.OperationType = constants.FileOperationUpdate
-	}
+
 	if isStreamUpload {
 		err = alloc.DoMultiOperation([]sdk.OperationRequest{opRequest})
 		if err != nil && !isSameRootError(err) {

cmd/gateway/zcn/gateway-zcn.go

@@ -19,7 +19,6 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/mimedb"
 	"github.com/mitchellh/go-homedir"
-	"golang.org/x/sync/semaphore"
 
 	"github.com/0chain/gosdk/zboxcore/sdk"
 	"github.com/minio/cli"
@@ -122,7 +121,6 @@ func (z *ZCN) Name() string {
 }
 
 var (
-	contentMap  map[string]*semaphore.Weighted
 	contentLock sync.Mutex
 )
 
@@ -156,7 +154,6 @@ func (z *ZCN) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, erro
 	if err != nil {
 		return nil, err
 	}
-	contentMap = make(map[string]*semaphore.Weighted)
 	ctx, cancel := context.WithCancel(context.Background())
 	zob.ctxCancel = cancel
 	IntiBatchUploadWorkers(ctx, allocation, serverConfig.BatchWaitTime, serverConfig.MaxBatchSize, serverConfig.BatchWorkers)
@@ -446,6 +443,10 @@ func (zob *zcnObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 
 // ListObjects Lists files of directories as objects
 func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
+	now := time.Now()
+	defer func() {
+		log.Println("ListObjectsTook: ", time.Since(now).Milliseconds())
+	}()
 	// objFileType For root path list objects should only provide file and not dirs.
 	// Dirs under root path are presented as buckets as well
 	var remotePath, objFileType string
@@ -493,15 +494,15 @@ func (zob *zcnObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 			},
 			nil
 	}
-
-	if len(prefix) > 0 && prefix[len(prefix)-1] != '/' {
-		return minio.ListObjectsInfo{
-				IsTruncated: false,
-				Objects:     []minio.ObjectInfo{},
-				Prefixes:    []string{prefix + "/"},
-			},
-			nil
-	}
+	// warp does not send paths with trailing slash
+	// if len(prefix) > 0 && prefix[len(prefix)-1] != '/' {
+	// 	return minio.ListObjectsInfo{
+	// 			IsTruncated: false,
+	// 			Objects:     []minio.ObjectInfo{},
+	// 			Prefixes:    []string{prefix + "/"},
+	// 		},
+	// 		nil
+	// }
 
 	var objects []minio.ObjectInfo
 	if prefix != "" {
@@ -600,27 +601,27 @@ func (zob *zcnObjects) PutObject(ctx context.Context, bucket, object string, r *
 		remotePath = filepath.Join(rootPath, bucket, object)
 	}
 
-	var ref *sdk.ORef
-	var isUpdate bool
-	err = lockPath(ctx, remotePath)
-	if err != nil {
-		return
-	}
-	ref, err = getSingleRegularRef(zob.alloc, remotePath)
-	if err != nil {
-		if !isPathNoExistError(err) {
-			unlockPath(remotePath)
-			return
-		}
-	}
-
-	if ref != nil {
-		logger.Info("updateFile: ", remotePath)
-		isUpdate = true
-		unlockPath(remotePath)
-	} else {
-		defer unlockPath(remotePath)
-	}
+	// var ref *sdk.ORef
+	// var isUpdate bool
+	// err = lockPath(ctx, remotePath)
+	// if err != nil {
+	// 	return
+	// }
+	// ref, err = getSingleRegularRef(zob.alloc, remotePath)
+	// if err != nil {
+	// 	if !isPathNoExistError(err) {
+	// 		unlockPath(remotePath)
+	// 		return
+	// 	}
+	// }
+
+	// if ref != nil {
+	// 	logger.Info("updateFile: ", remotePath)
+	// 	isUpdate = true
+	// 	unlockPath(remotePath)
+	// } else {
+	// 	defer unlockPath(remotePath)
+	// }
 
 	contentType := opts.UserDefined["content-type"]
 	if contentType == "" {
@@ -653,7 +654,7 @@ func (zob *zcnObjects) PutObject(ctx context.Context, bucket, object string, r *
 		}
 	}
 
-	err = putFile(ctx, zob.alloc, remotePath, contentType, r, r.Size(), isUpdate, opts.UserDefined)
+	err = putFile(ctx, zob.alloc, remotePath, contentType, r, r.Size(), false, opts.UserDefined)
 	if err != nil {
 		return
 	}
@@ -834,24 +835,6 @@ func (zob *zcnObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _
 	return
 }
 
-func lockPath(ctx context.Context, path string) error {
-	contentLock.Lock()
-	defer contentLock.Unlock()
-	if _, ok := contentMap[path]; !ok {
-		contentMap[path] = semaphore.NewWeighted(1)
-	}
-	return contentMap[path].Acquire(ctx, 1)
-}
-
-func unlockPath(path string) {
-	contentLock.Lock()
-	defer contentLock.Unlock()
-	if sem, ok := contentMap[path]; ok {
-		sem.Release(1)
-		delete(contentMap, path)
-	}
-}
-
 /*
 //Unfortunately share file is done by minio client which does't need to communicate with server. It generates share url with access key id and
 //secret key

cmd/gateway/zcn/multipart.go

@@ -108,16 +108,16 @@ func (zob *zcnObjects) newMultiPartUpload(localStorageDir, bucket, object, conte
 	} else {
 		remotePath = filepath.Join(rootPath, bucket, object)
 	}
-	ref, err := getSingleRegularRef(zob.alloc, remotePath)
-	if err != nil {
-		if !isPathNoExistError(err) {
-			return "", err
-		}
-	}
-
-	if ref != nil {
-		isUpdate = true
-	}
+	// ref, err := getSingleRegularRef(zob.alloc, remotePath)
+	// if err != nil {
+	// 	if !isPathNoExistError(err) {
+	// 		return "", err
+	// 	}
+	// }
+
+	// if ref != nil {
+	// 	isUpdate = true
+	// }
 	uploadID := uuid.New().String()
 	mapLock.Lock()
 	memFile := &memFile{

go.mod

@@ -7,7 +7,7 @@ toolchain go1.21.6
 require (
 	cloud.google.com/go/storage v1.27.0
 	github.com/0chain/errors v1.0.3
-	github.com/0chain/gosdk v1.17.6
+	github.com/0chain/gosdk v1.17.12-0.20241023084317-d4d99f55ac84
 	github.com/Azure/azure-pipeline-go v0.2.2
 	github.com/Azure/azure-storage-blob-go v0.10.0
 	github.com/Shopify/sarama v1.28.0
@@ -92,7 +92,6 @@ require (
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.17.0
 	golang.org/x/oauth2 v0.5.0
-	golang.org/x/sync v0.5.0
 	golang.org/x/sys v0.15.0
 	golang.org/x/time v0.3.0
 	google.golang.org/api v0.110.0
@@ -169,7 +168,7 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/herumi/bls-go-binary v1.31.0 // indirect
-	github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 // indirect
+	github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c // indirect
 	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
@@ -251,6 +250,7 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect

go.sum

@@ -63,8 +63,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs
 github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
 github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
 github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
-github.com/0chain/gosdk v1.17.6 h1:WKlyz8DiOrJ+nE18KO3KLdeEU+HBE8ZN8z7cHIvOE2c=
-github.com/0chain/gosdk v1.17.6/go.mod h1:y7Ucdmv40VltqulZnncMNjNQ4piX5Dta5ujNmPmXnxg=
+github.com/0chain/gosdk v1.17.12-0.20241023084317-d4d99f55ac84 h1:oAQZf4XFL8dw9pLieL+5SDaKLUcLg7veQ6kmWf2bGwE=
+github.com/0chain/gosdk v1.17.12-0.20241023084317-d4d99f55ac84/go.mod h1:+oA+SR2eQLJSo4NfNew93HeE39CiRFqznWE6t1grjDM=
 github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY=
@@ -897,8 +897,8 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/herumi/bls-go-binary v1.31.0 h1:L1goQ2tMtGgpXCg5AwHAdJQpLs/pfnWWEc3Wog6OhmI=
 github.com/herumi/bls-go-binary v1.31.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y=
-github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 h1:Z6Mu2JCsW2hbqx91L0HNPRPQ10RyAFvPocQHlrRo1Jk=
-github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk=
+github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c h1:lDSIbcLu5TdT+uwb4wPzZgo1pQvKjP/tArL5QKjDJdI=
+github.com/hitenjain14/fasthttp v0.0.0-20240916135632-f9303a91736c/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk=
 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
 github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
 github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8=
@@ -1973,8 +1973,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=