fix: fix sync paste dir out bug, add time to task log for debug, and syncbuffertofile chunk size up to 8MB

Author: lovehunter9

pkg/drives/sync.go

@@ -337,11 +337,12 @@ func (rs *SyncResourceService) PasteDirFrom(task *pool.Task, fs afero.Fs, srcFil
 		fsrc := filepath.Join(src, item.Name)
 		fdst := filepath.Join(fdstBase, item.Name)
 
+		klog.Infof("~~~Debug log: srcUri=%s, fsrc=%s", srcUri, fsrc)
 		fsrcFileParam := &models.FileParam{
 			Owner:    srcFileParam.Owner,
 			FileType: srcFileParam.FileType,
 			Extend:   srcFileParam.Extend,
-			Path:     strings.TrimPrefix(fsrc, strings.TrimPrefix(srcUri, "/data")),
+			Path:     strings.TrimPrefix("/sync"+fsrc, strings.TrimPrefix(srcUri, "/data")),
 		}
 		klog.Infof("~~~Debug log: dstUri=%s, fdst=%s", dstUri, fdst)
 		fdstFileParam := &models.FileParam{
@@ -1127,9 +1128,10 @@ func SyncFileToBuffer(task *pool.Task, src, bufferFilePath string, srcFileParam
 	//}
 
 	repoID := srcFileParam.Extend
-	prefix, filename := filepath.Split(srcFileParam.Path)
+	prefix, filename := filepath.Split(strings.Trim(srcFileParam.Path, "/"))
 
-	dlUrl := "http://127.0.0.1:80/seahub/lib/" + repoID + "/file/" + common.EscapeURLWithSpace(prefix+filename) + "/" + "?dl=1"
+	dlUrl := "http://127.0.0.1:80/seahub/lib/" + repoID + "/file/" + prefix + "/" + filename + "/" + "?dl=1"
+	klog.Infof("~~~Debug log: dlURL=%s", dlUrl)
 
 	request, err := http.NewRequestWithContext(task.Ctx, "GET", dlUrl, nil)
 	if err != nil {
@@ -1209,7 +1211,7 @@ func SyncFileToBuffer(task *pool.Task, src, bufferFilePath string, srcFileParam
 			}
 
 			if lastProgress != progress {
-				task.Log = append(task.Log, fmt.Sprintf("downloaded from seafile %d/%d with progress %d", totalRead, diskSize, progress))
+				task.Log = append(task.Log, "["+time.Now().Format("2006-01-02 15:04:05")+"]"+fmt.Sprintf("downloaded from seafile %d/%d with progress %d", totalRead, diskSize, progress))
 				lastProgress = progress
 			}
 			task.Progress = mappedProgress
@@ -1321,7 +1323,7 @@ func SyncBufferToFile(task *pool.Task, bufferFilePath, dst string, dstFileParam
 	}
 	fileSize := fileInfo.Size()
 
-	chunkSize := int64(5 * 1024 * 1024) // 5 MB
+	chunkSize := int64(8 * 1024 * 1024) // 8MB
 	totalChunks := (fileSize + chunkSize - 1) / chunkSize
 	identifier := generateUniqueIdentifier(common.EscapeAndJoin(filename, "/"))
 

pkg/fileutils/rsync.go

@@ -406,7 +406,7 @@ func ExecuteRsync(task *pool.Task, src, dst string, progressLeft, progressRight
 	if firstErr != nil {
 		if task.Status != "failed" {
 			errMsg := parseRsyncError(firstErr, task)
-			task.Log = append(task.Log, errMsg)
+			task.Log = append(task.Log, "["+time.Now().Format("2006-01-02 15:04:05")+"]"+errMsg)
 			task.FailedReason = errMsg
 			pool.FailTask(task.ID)
 		}

pkg/pool/worker_pool.go

@@ -179,7 +179,7 @@ func (t *Task) UpdateProgress() {
 					return
 				}
 				if log != "" {
-					t.Log = append(t.Log, log)
+					t.Log = append(t.Log, "["+time.Now().Format("2006-01-02 15:04:05")+"]"+log)
 					klog.Infof("[%s] Log collected: %s (total: %d)", t.ID, log, len(t.Log))
 				}
 			case <-t.Ctx.Done():
@@ -235,7 +235,7 @@ func (t *Task) UpdateProgress() {
 				break
 			}
 			klog.Errorf("[%s Error] %v", t.ID, err)
-			t.Log = append(t.Log, fmt.Sprintf("ERROR: %v", err))
+			t.Log = append(t.Log, "["+time.Now().Format("2006-01-02 15:04:05")+"]"+fmt.Sprintf("ERROR: %v", err))
 			t.FailedReason = err.Error()
 
 		case progress, ok := <-t.ProgressChan:
@@ -296,12 +296,12 @@ func (t *Task) UpdateProgress() {
 
 func (t *Task) Logging(entry string) {
 	klog.Infof("TASK LOGGING [%s] %s", t.ID, entry)
-	t.Log = append(t.Log, entry)
+	t.Log = append(t.Log, "["+time.Now().Format("2006-01-02 15:04:05")+"]"+entry)
 }
 
 func (t *Task) LoggingError(entry string) {
 	klog.Infof("TASK LOGGING ERROR [%s] %s", t.ID, entry)
-	t.Log = append(t.Log, "[ERROR] "+entry)
+	t.Log = append(t.Log, "["+time.Now().Format("2006-01-02 15:04:05")+"]"+"[ERROR] "+entry)
 	t.FailedReason = entry
 }