Sprint july 2 (#490)

* Move min lock demand (#458) * min_lock_demand * min_lock_demand * remove dead code * go mod * go mod * updated gosdk (#466) * Rename to not_available (#457) * remove unstake * not_available * go mod * go mod * repoint to not_available gosdk --------- Co-authored-by: Kishan-Dhakan <[email protected]> Co-authored-by: Kishan Dhakan <[email protected]> * point gosdk to staging * fix RepairCompleted (#470) * multi upload in cli (#468) * multi upload in cli * workdir * make remotepath optional * multi download * update go version * update gosdkk * update gosdk (#472) * Update gosdk (#450) * use gosdk of fix thumb hash * fix gosdk version * update gosdk * update gosdk * update gosdk * updated gosdk * updated gosdk --------- Co-authored-by: dabasov <[email protected]> * add flag blobber_id to download file from single blobber (#437) * add flag blobber_id to download file from single blobber * Updated gosdk * update gosdk * Delete a.txt * chore(gomod): upgraded bls-go-binary with bls v1.35 (#396) * added go setup inside runner for build-zbox.yaml (#446) * added go setup inside runner for build-zbox.yaml * fixed build-zboxcli.yml * go mod tidy * change exit status to 0 --------- Co-authored-by: Jayash <[email protected]> Co-authored-by: Lz <[email protected]> Co-authored-by: shahnawaz-creator <[email protected]> Co-authored-by: Jayash Satolia <[email protected]> Co-authored-by: Yury <[email protected]> * update gosdk to latest commit (#476) * Update gosdk for repair allocation (#461) * update gosdk for repair alloc * update gosdk * update gosdk * update gosdk --------- Co-authored-by: Kishan-Dhakan <[email protected]> Co-authored-by: Kishan Dhakan <[email protected]> * point to feat gosdk (#478) * Update gosdk - fix file callback (#477) * update gosdk * update gosdk * update gosdk * update gosdk * add preferred_blobbers flag to cli (#482) Co-authored-by: Yury <[email protected]> * Fix exclude path (#479) * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Updated gosdk * Show actual num blocks, show size for directories (#456) * show size for dir, add actual numblocks * update gosdk * update gosdk * update gosdk * update gosdk * update gosdk * update gosdk * update gosdk * update gosdk * updated gosdk * updated gosdk * Fix (#487) * Fix (#488) * docs: command line cli for update commmand (#481) * docs: command line cli for update commmand * update description * fix multi-download in cli (#484) * fix multidownload: add wait group * cleanup variable names * use multidownload gosdk commit * upload as encrypted * fix actual_size error * Update the logic for update blobber and validator calls as per new models (#480) * updating the logic for valiator and blobber udpate calls. * fixing npe in tests * updating the logic to populate terms and stakepoolsettings * update the gosdk version * update the gosdk version * Refactoring to take the provided id as input for validator and blobber udpate setting calls. * Updating gosdk version * update gosdk (#489) * update gosdk to staging --------- Co-authored-by: Piers Shepperson <[email protected]> Co-authored-by: Yury <[email protected]> Co-authored-by: boddumanohar <[email protected]> Co-authored-by: Dinmukhammed <[email protected]> Co-authored-by: Hitenjain14 <[email protected]> Co-authored-by: Jayash <[email protected]> Co-authored-by: Lz <[email protected]> Co-authored-by: shahnawaz-creator <[email protected]> Co-authored-by: Jayash Satolia <[email protected]> Co-authored-by: stewartie4 <[email protected]> Co-authored-by: Sunil Kumar <[email protected]>
0chain · Jul 16, 2023 · 7f9d0a5 · 7f9d0a5
1 parent 7846e3d
commit 7f9d0a5
Show file tree

Hide file tree

Showing 13 changed files with 275 additions and 68 deletions.
diff --git a/cmd/common.go b/cmd/common.go
@@ -53,6 +53,7 @@ func (s *StatusBar) Error(allocationID string, filePath string, op int, err erro
 func (s *StatusBar) RepairCompleted(filesRepaired int) {
 	defer s.wg.Done()
 	allocUnderRepair = false
+	s.success = true
 	fmt.Println("Repair file completed, Total files repaired: ", filesRepaired)
 }
 

diff --git a/cmd/download.go b/cmd/download.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"os"
 	"strings"
 	"sync"
@@ -19,7 +20,7 @@ var downloadCmd = &cobra.Command{
 	Args:  cobra.MinimumNArgs(0),
 	Run: func(cmd *cobra.Command, args []string) {
 		fflags := cmd.Flags() // fflags is a *flag.FlagSet
-		if !(fflags.Changed("remotepath") || fflags.Changed("authticket")) {
+		if !(fflags.Changed("remotepath") || fflags.Changed("authticket") || fflags.Changed("multidownloadjson")) {
 			PrintError("Error: remotepath / authticket flag is missing")
 			os.Exit(1)
 		}
@@ -83,6 +84,11 @@ var downloadCmd = &cobra.Command{
 		var errE error
 		var allocationObj *sdk.Allocation
 
+		var multidownloadJSON string
+		if fflags.Changed("multidownloadjson") {
+			multidownloadJSON = cmd.Flag("multidownloadjson").Value.String()
+		}
+
 		if len(authTicket) > 0 {
 			at, err := sdk.InitAuthTicket(authTicket).Unmarshall()
 
@@ -144,15 +150,36 @@ var downloadCmd = &cobra.Command{
 				PrintError("Error fetching the allocation", err)
 				os.Exit(1)
 			}
+
+			var blobberID string
+			if fflags.Changed("blobber_id") {
+				blobberID = cmd.Flag("blobber_id").Value.String()
+			}
+
 			if thumbnail {
 				errE = allocationObj.DownloadThumbnail(localPath, remotePath, verifyDownload, statusBar, true)
+			} else if blobberID != "" {
+				errE = allocationObj.DownloadFromBlobber(blobberID, localPath, remotePath, statusBar)
 			} else {
 				if startBlock != 0 || endBlock != 0 {
 					errE = allocationObj.DownloadFileByBlock(localPath, remotePath, startBlock, endBlock, numBlocks, verifyDownload, statusBar, true)
 				} else {
 					errE = allocationObj.DownloadFile(localPath, remotePath, verifyDownload, statusBar, true)
 				}
 			}
+		} else if len(multidownloadJSON) > 0 {
+			if fflags.Changed("allocation") == false { // check if the flag "path" is set
+				PrintError("Error: allocation flag is missing") // If not, we'll let the user know
+				os.Exit(1)                                      // and return
+			}
+			allocationID := cmd.Flag("allocation").Value.String()
+			allocationObj, err = sdk.GetAllocation(allocationID)
+			if err != nil {
+				PrintError("Error: getting allocation", err)
+				os.Exit(1)
+			}
+
+			errE = MultiDownload(allocationObj, multidownloadJSON, statusBar)
 		}
 
 		if errE == nil {
@@ -162,19 +189,67 @@ var downloadCmd = &cobra.Command{
 			os.Exit(1)
 		}
 		if !statusBar.success {
-			os.Exit(1)
+			// status bar always returns failure when downloading from sigle blobber. Hence returning the zero exit status
+			if fflags.Changed("blobber_id") {
+				os.Exit(0)
+			} else {
+				os.Exit(1)
+			}
 		}
 
 	},
 }
 
+type MultiDownloadOption struct {
+	RemotePath       string `json:"remotePath"`
+	LocalPath        string `json:"localPath"`
+	DownloadOp       int    `json:"downloadOp"`
+	RemoteFileName   string `json:"remoteFileName,omitempty"`   //Required only for file download with auth ticket
+	RemoteLookupHash string `json:"remoteLookupHash,omitempty"` //Required only for file download with auth ticket
+}
+
+func MultiDownload(a *sdk.Allocation, jsonMultiDownloadOptions string, statusBar *StatusBar) error {
+	var options []MultiDownloadOption
+	file, err := os.Open(jsonMultiDownloadOptions)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	decoder := json.NewDecoder(file)
+
+	err = decoder.Decode(&options)
+	if err != nil {
+		return err
+	}
+
+	lastOp := len(options) - 1
+	for i := 0; i <= len(options)-1; i++ {
+		if i > 0 {
+			statusBar.wg.Add(1)
+		}
+		if options[i].DownloadOp == 1 {
+			err = a.DownloadFile(options[i].LocalPath, options[i].RemotePath, true, statusBar, i == lastOp)
+		} else {
+			err = a.DownloadThumbnail(options[i].LocalPath, options[i].RemotePath, false, statusBar, i == lastOp)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
 func init() {
 	rootCmd.AddCommand(downloadCmd)
 	downloadCmd.PersistentFlags().String("allocation", "", "Allocation ID")
 	downloadCmd.PersistentFlags().String("remotepath", "", "Remote path to download")
 	downloadCmd.PersistentFlags().String("localpath", "", "Local path of file to download")
+	downloadCmd.PersistentFlags().String("blobber_id", "", "to download the data shard present in that blobber")
 	downloadCmd.PersistentFlags().String("authticket", "", "Auth ticket fot the file to download if you dont own it")
 	downloadCmd.PersistentFlags().String("lookuphash", "", "The remote lookuphash of the object retrieved from the list")
+	downloadCmd.PersistentFlags().String("multidownloadjson", "", "A JSON file containing multi download options")
 	downloadCmd.Flags().BoolP("thumbnail", "t", false, "(default false) pass this option to download only the thumbnail")
 
 	downloadCmd.Flags().Int64P("startblock", "s", 1,

diff --git a/cmd/list.go b/cmd/list.go
@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"strconv"
 	"strings"
@@ -148,13 +147,13 @@ func printListDirResult(outJson bool, ref *sdk.ListResult) {
 		return
 	}
 
-	header := []string{"Type", "Name", "Path", "Size", "Actual Size", "Num Blocks", "Lookup Hash", "Is Encrypted"}
+	header := []string{"Type", "Name", "Path", "Size", "Num Blocks", "Actual Size", "Actual Num Blocks", "Lookup Hash", "Is Encrypted"}
 	data := make([][]string, len(ref.Children))
 	for idx, child := range ref.Children {
 		size := strconv.FormatInt(child.Size, 10)
-		if child.Type == fileref.DIRECTORY {
-			size = ""
-		}
+		numBlocks := strconv.FormatInt(child.NumBlocks, 10)
+		actualSize := strconv.FormatInt(child.ActualSize, 10)
+		actualNumBlocks := strconv.FormatInt(child.ActualNumBlocks, 10)
 		isEncrypted := ""
 		if child.Type == fileref.FILE {
 			if len(child.EncryptionKey) > 0 {
@@ -168,8 +167,9 @@ func printListDirResult(outJson bool, ref *sdk.ListResult) {
 			child.Name,
 			child.Path,
 			size,
-			fmt.Sprint(child.ActualSize),
-			strconv.FormatInt(child.NumBlocks, 10),
+			numBlocks,
+			actualSize,
+			actualNumBlocks,
 			child.LookupHash,
 			isEncrypted,
 		}

diff --git a/cmd/newallocation.go b/cmd/newallocation.go
@@ -11,8 +11,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/0chain/gosdk/zboxcore/blockchain"
-
 	"github.com/spf13/pflag"
 
 	"github.com/0chain/gosdk/zboxcore/sdk"
@@ -25,6 +23,7 @@ var (
 	datashards, parityshards *int
 	size                     *int64
 	allocationFileName       *string
+	preferred_blobbers       []string
 )
 
 func getPriceRange(val string) (pr sdk.PriceRange, err error) {
@@ -110,6 +109,14 @@ var newallocationCmd = &cobra.Command{
 			writePrice = sdk.PriceRange{Min: 0, Max: maxPrice}
 		)
 
+		if flags.Changed("preferred_blobbers") {
+			b, err := flags.GetString("preferred_blobbers")
+			if err != nil {
+				log.Fatal("invalid read_price value: ", err)
+			}
+			preferred_blobbers = strings.Split(b, ",")
+		}
+
 		if flags.Changed("read_price") {
 			rps, err := flags.GetString("read_price")
 			if err != nil {
@@ -223,8 +230,25 @@ var newallocationCmd = &cobra.Command{
 
 		var allocationID string
 		if len(owner) == 0 {
-			allocationID, _, _, err = sdk.CreateAllocation(*datashards, *parityshards,
-				*size, expireAt, readPrice, writePrice, lock, thirdPartyExtendable, &fileOptionParams)
+			options := sdk.CreateAllocationOptions{
+				DataShards:   *datashards,
+				ParityShards: *parityshards,
+				Size:         *size,
+				Expiry:       expireAt,
+				ReadPrice: sdk.PriceRange{
+					Min: uint64(readPrice.Min),
+					Max: uint64(readPrice.Max),
+				},
+				WritePrice: sdk.PriceRange{
+					Min: uint64(writePrice.Min),
+					Max: uint64(writePrice.Max),
+				},
+				Lock:                 uint64(lock),
+				BlobberIds:           preferred_blobbers,
+				FileOptionsParams:    &fileOptionParams,
+				ThirdPartyExtendable: thirdPartyExtendable,
+			}
+			allocationID, _, _, err = sdk.CreateAllocationWith(options)
 			if err != nil {
 				log.Fatal("Error creating allocation: ", err)
 			}
@@ -240,7 +264,7 @@ var newallocationCmd = &cobra.Command{
 			}
 
 			allocationID, _, _, err = sdk.CreateAllocationForOwner(owner, ownerPublicKey, *datashards, *parityshards,
-				*size, expireAt, readPrice, writePrice, lock, blockchain.GetPreferredBlobbers(), thirdPartyExtendable, &fileOptionParams)
+				*size, expireAt, readPrice, writePrice, lock, preferred_blobbers, thirdPartyExtendable, &fileOptionParams)
 			if err != nil {
 				log.Fatal("Error creating allocation: ", err)
 			}
@@ -281,7 +305,7 @@ func init() {
 	rootCmd.AddCommand(newallocationCmd)
 	datashards = newallocationCmd.PersistentFlags().Int("data", 2, "the number of blobbers to be used as data shards")
 	parityshards = newallocationCmd.PersistentFlags().Int("parity", 2, "the number of blobber to be used as parity shards")
-	size = newallocationCmd.PersistentFlags().Int64("size", 2147483648, "the size of the allocation")
+	size = newallocationCmd.PersistentFlags().Int64("size", 2*GB, "the size of the allocation")
 	allocationFileName = newallocationCmd.PersistentFlags().String("allocationFileName", "allocation.txt", "name of the file in configDir to store the generated allocationID")
 	newallocationCmd.PersistentFlags().
 		Float64("lock", 0.0,
@@ -310,6 +334,7 @@ func init() {
 		"public key of owner, user when creating an allocation for somone else")
 
 	newallocationCmd.Flags().String("name", "", "allocation name")
+	newallocationCmd.Flags().String("preferred_blobbers", "", "coma seperated list of preferred blobbers")
 
 	newallocationCmd.Flags().Bool("third_party_extendable", false, "(default false) specify if the allocation can be extended by users other than the owner")
 	newallocationCmd.Flags().Bool("forbid_upload", false, "(default false) specify if users cannot upload to this allocation")
@@ -332,5 +357,4 @@ func storeAllocation(allocationID string) {
 	defer file.Close()
 	//Only one allocation ID per file.
 	fmt.Fprint(file, allocationID)
-
 }