Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Filter for video files #663

Merged
merged 19 commits into from
Feb 10, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/build-zbox.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
make install

- name: 'Upload Artifact'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: zbox-linux
path: zbox
Expand Down Expand Up @@ -93,7 +93,7 @@ jobs:
zip zbox-windows.zip zbox.exe libgcc_s_seh-1.dll libstdc++-6.dll libwinpthread-1.dll

- name: 'Upload Artifact'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: zbox-windows.zip
path: zbox-windows.zip
Expand Down Expand Up @@ -121,7 +121,7 @@ jobs:
run: make install

- name: 'Upload Artifact'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: zbox-macos
path: zbox
Expand Down
6 changes: 6 additions & 0 deletions cmd/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"fmt"
"io"
"log"
"os"
"sync"
Expand All @@ -16,6 +17,10 @@ const (

func (s *StatusBar) Started(allocationId, filePath string, op int, totalBytes int) {
s.b = pb.StartNew(totalBytes)
if s.f != nil {
s.b.Output = s.f
s.b.NotPrint = true
}
s.b.Set(0)
}
func (s *StatusBar) InProgress(allocationId, filePath string, op int, completedBytes int, data []byte) {
Expand Down Expand Up @@ -60,6 +65,7 @@ func (s *StatusBar) RepairCompleted(filesRepaired int) {
type StatusBar struct {
b *pb.ProgressBar
wg *sync.WaitGroup
f io.Writer
success bool
}

Expand Down
9 changes: 9 additions & 0 deletions cmd/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,15 @@ var downloadCmd = &cobra.Command{
sdk.SetNumBlockDownloads(numBlocks)
wg := &sync.WaitGroup{}
statusBar := &StatusBar{wg: wg}
if logFilePath != "" {
f, err := os.Create(logFilePath)
if err != nil {
PrintError("Error creating log file", err)
os.Exit(1)
}
defer f.Close()
statusBar.f = f
}
wg.Add(1)
var errE error
var allocationObj *sdk.Allocation
Expand Down
7 changes: 5 additions & 2 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ var bSilent bool
var allocUnderRepair bool

var walletJSON string
var logFilePath string

var rootCmd = &cobra.Command{
Use: "zbox",
Expand All @@ -58,6 +59,7 @@ func init() {
rootCmd.PersistentFlags().StringVar(&cDir, "configDir", "", "configuration directory (default is $HOME/.zcn)")
rootCmd.PersistentFlags().BoolVar(&bSilent, "silent", false, "(default false) Do not show interactive sdk logs (shown by default)")
rootCmd.PersistentFlags().Float64Var(&txFee, "fee", 0, "transaction fee for the given transaction (if unset, it will be set to blockchain min fee)")
rootCmd.PersistentFlags().StringVar(&logFilePath, "log", "", "log file path where progress will be logged")
}

func Execute() {
Expand Down Expand Up @@ -89,8 +91,9 @@ func initConfig() {
logger.SyncLoggers([]*logger.Logger{zcncore.GetLogger(), sdk.GetLogger()})

// set the log file
zcncore.SetLogFile("cmdlog.log", !bSilent)
sdk.SetLogFile("cmdlog.log", !bSilent)
logPath := "cmdlog.log"
zcncore.SetLogFile(logPath, !bSilent)
sdk.SetLogFile(logPath, !bSilent)

err = client.Init(context.Background(), cfg)
if err != nil {
Expand Down
34 changes: 32 additions & 2 deletions cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,30 @@ func filterEmptyFiles(localPath string, lDiff []sdk.FileDiff) (filterDiff []sdk.
return
}

func filterVideoFiles(localPath string, lDiff []sdk.FileDiff) (filterDiff []sdk.FileDiff) {
localPath = strings.TrimRight(localPath, "/")
for _, f := range lDiff {
path := localPath + f.Path
parentPath := filepath.Dir(path)
videoPath := filepath.Dir(parentPath)
//get extension of directory name
parentPathExt := filepath.Ext(parentPath)
if parentPathExt == "" && filepath.Base(parentPath) != "preview" {
filterDiff = append(filterDiff, f)
continue
}
if filepath.Base(path) == "thumbnail_generated.jpg" || filepath.Base(path) == "0kb" || (f.Type == "d" && filepath.Base(path) == "preview") {
continue
}

ext := filepath.Ext(videoPath)
if ext != ".mp4" && ext != ".mkv" && ext != ".avi" && ext != ".mov" && ext != ".flv" && ext != ".wmv" && ext != ".webm" {
filterDiff = append(filterDiff, f)
}
}
return
}

func startMultiUploadUpdate(allocationObj *sdk.Allocation, argsSlice []chunkedUploadArgs) error {
totalOperations := len(argsSlice)
if totalOperations == 0 {
Expand Down Expand Up @@ -220,6 +244,7 @@ var syncCmd = &cobra.Command{
}

lDiff = filterEmptyFiles(localpath, lDiff)
lDiff = filterVideoFiles(localpath, lDiff)

if len(lDiff) > 0 {
printTable(lDiff)
Expand Down Expand Up @@ -282,7 +307,11 @@ var syncCmd = &cobra.Command{
fileMetas[f.Path] = fileMeta
// TODO: User confirm??
fmt.Printf("Deleting remote %s...\n", f.Path)
err = allocationObj.DeleteFile(f.Path)
opReq := sdk.OperationRequest{
RemotePath: f.Path,
OperationType: constants.FileOperationDelete,
}
err = allocationObj.DoMultiOperation([]sdk.OperationRequest{opReq})
if err != nil {
PrintError("Error deleting remote file,", err.Error())
}
Expand Down Expand Up @@ -374,6 +403,7 @@ var getDiffCmd = &cobra.Command{
PrintError("Error getting diff.", err)
os.Exit(1)
}
lDiff = filterVideoFiles(localpath, lDiff)

util.PrintJSON(lDiff)
},
Expand All @@ -394,7 +424,7 @@ func init() {
If file exists, this will be used for comparison with remote.
After sync complete, remote snapshot will be updated to the same file for next use.`)
syncCmd.PersistentFlags().StringArray("excludepath", []string{}, "Remote folder paths exclude to sync")
syncCmd.Flags().BoolP("verifydownload", "v", true, "pass this option to verify downloaded blocks")
syncCmd.Flags().BoolP("verifydownload", "v", false, "pass this option to verify downloaded blocks")

syncCmd.MarkFlagRequired("allocation")
syncCmd.MarkFlagRequired("localpath")
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ toolchain go1.22.5

require (
github.com/0chain/errors v1.0.3
github.com/0chain/gosdk v1.19.0-RC2.0.20250117164514-4933fa5602a5
github.com/0chain/gosdk v1.19.2
github.com/icza/bitio v1.1.0
github.com/olekukonko/tablewriter v0.0.5
github.com/spf13/cobra v1.6.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ github.com/0chain/common v1.18.3 h1:42dYOv2KyMTSanuS67iDtfv+ErbSRqR8NJ3MG72MwaI=
github.com/0chain/common v1.18.3/go.mod h1:Lapu2Tj7z5Sm4r+X141e7vsz4NDODTEypeElYAP3iSw=
github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
github.com/0chain/gosdk v1.19.0-RC2.0.20250117164514-4933fa5602a5 h1:A8Ig1uLfMIaGtP4/m2X7/KZC9+thbsOJNBaMY/ZsycY=
github.com/0chain/gosdk v1.19.0-RC2.0.20250117164514-4933fa5602a5/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
github.com/0chain/gosdk v1.19.2 h1:OpGwtUAObAqU4HatQ8VM/Z+m07LG65NhF1dO/1q9RTs=
github.com/0chain/gosdk v1.19.2/go.mod h1:8unFy9Dx2YyPKMYPDGR3MFhUEymbAfQcRDm9bobVLGw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
Expand Down
Loading