Merge pull request #209 from tus/dw/memory-usage

Decrease memory usage when resuming an upload

Author: donnywals

Sources/TUSKit/Files.swift

@@ -157,6 +157,38 @@ final class Files {
         }
     }
 
+    @available(iOS 13.4, macOS 10.15.4, *)
+    func streamingData(_ dataGenerator: () -> Data?, id: UUID, preferredFileExtension: String? = nil) throws -> URL {
+        try queue.sync {
+            try makeDirectoryIfNeeded()
+            
+            let fileName: String
+            if let fileExtension = preferredFileExtension {
+                fileName = id.uuidString + fileExtension
+            } else {
+                fileName = id.uuidString
+            }
+            
+            let targetLocation = storageDirectory.appendingPathComponent(fileName)
+            if !FileManager.default.fileExists(atPath: targetLocation.path) {
+                FileManager.default.createFile(atPath: targetLocation.path, contents: nil)
+            }
+            
+            let destinationHandle = try FileHandle(forWritingTo: targetLocation)
+            try destinationHandle.truncate(atOffset: 0)
+            defer {
+                try? destinationHandle.close()
+            }
+            
+            while let data = dataGenerator() {
+                guard !data.isEmpty else { throw FilesError.dataIsEmpty }
+                try destinationHandle.write(contentsOf: data)
+            }
+            
+            return targetLocation
+        }
+    }
+    
     /// Removes metadata and its related file from disk
     /// - Parameter metaData: The metadata description
     /// - Throws: Any error from FileManager when removing a file.

Sources/TUSKit/TUSAPI.swift

@@ -15,6 +15,23 @@ public enum TUSAPIError: Error {
     case couldNotRetrieveOffset
     case couldNotRetrieveLocation
     case failedRequest(HTTPURLResponse)
+    
+    public var localizedDescription: String {
+        switch self {
+        case .underlyingError(let error):
+            return "Underlying error: " + error.localizedDescription
+        case .couldNotFetchStatus:
+            return "Could not fetch status from server."
+        case .couldNotFetchServerInfo:
+            return "Could not fetch server info."
+        case .couldNotRetrieveOffset:
+            return "Could not retrieve offset from response."
+        case .couldNotRetrieveLocation:
+            return "Could not retrieve location from response."
+        case .failedRequest(let response):
+            return "Failed request with status code \(response.statusCode)."
+        }
+    }
 }
 
 /// The status of an upload.

Sources/TUSKit/TUSClient.swift

@@ -402,12 +402,21 @@ public final class TUSClient {
   @discardableResult
   public func resume(id: UUID) throws -> Bool {
       do {
-          var uploadMetadata: UploadMetadata?
+          var metaData: UploadMetadata?
           queue.sync {
-              uploadMetadata = uploads[id]
+              metaData = uploads[id]
           }
-          guard uploadMetadata != nil else { return false }
-          guard let metaData = try files.findMetadata(id: id) else {
+          
+          if metaData == nil {
+              guard let storedMetadata = try files.findMetadata(id: id) else {
+                  return false
+              }
+              
+              metaData = storedMetadata
+          }
+          
+          guard let metaData else {
+              // should never happen...
               return false
           }
 

Sources/TUSKit/TUSClientError.swift

@@ -19,8 +19,53 @@ public enum TUSClientError: Error {
     case couldnotRemoveFinishedUploads(underlyingError: Error)
     case receivedUnexpectedOffset
     case missingRemoteDestination
-    case emptyUploadRange
     case rangeLargerThanFile
     case taskCancelled
     case customURLSessionWithBackgroundConfigurationNotSupported
+    case emptyUploadRange
+    
+    public var localizedDescription: String {
+        switch self {
+        case .couldNotCopyFile(let underlyingError):
+            return "Could not copy file: \(underlyingError.localizedDescription)"
+        case .couldNotStoreFile(let underlyingError):
+            return "Could not store file: \(underlyingError.localizedDescription)"
+        case .fileSizeUnknown:
+            return "The file size is unknown."
+        case .couldNotLoadData(let underlyingError):
+            return "Could not load data: \(underlyingError.localizedDescription)"
+        case .couldNotStoreFileMetadata(let underlyingError):
+            return "Could not store file metadata: \(underlyingError.localizedDescription)"
+        case .couldNotCreateFileOnServer:
+            return "Could not create file on server."
+        case .couldNotUploadFile(let underlyingError):
+            return "Could not upload file: \(underlyingError.localizedDescription)"
+        case .couldNotGetFileStatus:
+            return "Could not get file status."
+        case .fileSizeMismatchWithServer:
+            return "File size mismatch with server."
+        case .couldNotDeleteFile(let underlyingError):
+            return "Could not delete file: \(underlyingError.localizedDescription)"
+        case .uploadIsAlreadyFinished:
+            return "The upload is already finished."
+        case .couldNotRetryUpload:
+            return "Could not retry upload."
+        case .couldNotResumeUpload:
+            return "Could not resume upload."
+        case .couldnotRemoveFinishedUploads(let underlyingError):
+            return "Could not remove finished uploads: \(underlyingError.localizedDescription)"
+        case .receivedUnexpectedOffset:
+            return "Received unexpected offset."
+        case .missingRemoteDestination:
+            return "Missing remote destination for upload."
+        case .emptyUploadRange:
+            return "The upload range is empty."
+        case .rangeLargerThanFile:
+            return "The upload range is larger than the file size."
+        case .taskCancelled:
+            return "The task was cancelled."
+        case .customURLSessionWithBackgroundConfigurationNotSupported:
+            return "Custom URLSession with background configuration is not supported."
+        }
+    }
 }

Sources/TUSKit/Tasks/UploadDataTask.swift

@@ -182,9 +182,25 @@ final class UploadDataTask: NSObject, IdentifiableTask {
 
         // Can't use switch with #available :'(
         let data: Data
-        if let range = self.range, #available(iOS 13.0, macOS 10.15, *) { // Has range, for newer versions
-            try fileHandle.seek(toOffset: UInt64(range.startIndex))
-            data = fileHandle.readData(ofLength: range.count)
+        if let range = self.range, #available(iOS 13.4, macOS 10.15.4, *) { // Has range, for newer versions
+            var offset = range.startIndex
+            
+            return try files.streamingData({
+                autoreleasepool {
+                    do {
+                        let chunkSize = min(1024 * 1024 * 500, range.endIndex - offset)
+                        try fileHandle.seek(toOffset: UInt64(offset))
+                        guard offset < range.endIndex else { return nil }
+                        
+                        let data = fileHandle.readData(ofLength: chunkSize)
+                        print("read data of size \(data.count) at offset \(offset)")
+                        offset += chunkSize
+                        return data
+                    } catch {
+                        return nil
+                    }
+                }
+            }, id: metaData.id, preferredFileExtension: "uploadData")
         } else if let range = self.range { // Has range, for older versions
             fileHandle.seek(toFileOffset: UInt64(range.startIndex))
             data = fileHandle.readData(ofLength: range.count)

TUSKitExample/TUSKitExample/Helpers/TUSWrapper.swift

@@ -40,12 +40,20 @@ class TUSWrapper: ObservableObject {
 
     @MainActor
     func resumeUpload(id: UUID) {
-        _ = try? client.resume(id: id)
-        
-        if case let .paused(bytesUploaded, totalBytes) = uploads[id] {
-            withAnimation {
-                uploads[id] = .uploading(bytesUploaded: bytesUploaded, totalBytes: totalBytes)
+        do {
+            guard try client.resume(id: id) == true else {
+                print("Upload not resumed; metadata not found")
+                return
             }
+            
+            if case let .paused(bytesUploaded, totalBytes) = uploads[id] {
+                withAnimation {
+                    uploads[id] = .uploading(bytesUploaded: bytesUploaded, totalBytes: totalBytes)
+                }
+            }
+        } catch {
+            print("Could not resume upload with id \(id)")
+            print(error)
         }
     }
 
@@ -99,6 +107,10 @@ extension TUSWrapper: TUSClientDelegate {
 
     func uploadFailed(id: UUID, error: Error, context: [String : String]?, client: TUSClient) {
         Task { @MainActor in
+            // Pausing an upload means we cancel it, so we don't want to show it as failed.
+            if let tusError = error as? TUSClientError, case .taskCancelled = tusError {
+                return
+            }
 
             withAnimation {
                 uploads[id] = .failed(error: error)

TUSKitExample/TUSKitExample/Screens/Upload/UploadsListView.swift

@@ -117,8 +117,6 @@ extension UploadsListView {
 
 
     // MARK: - Records List View
-    
-    
     @ViewBuilder
     private func uploadRecordsListView(items: [UUID: UploadStatus]) -> some View {
         ScrollView {
@@ -135,6 +133,10 @@ extension UploadsListView {
                                 UploadedRowView(key: idx.key, url: url)
                             case .failed(let error):
                                 FailedRowView(key: idx.key, error: error)
+                                    .onAppear {
+                                        print(error)
+                                        print(error.localizedDescription)
+                                    }
                         }
                     }
                     Divider()