Merge pull request #268 from reef-technologies/DownloadRefactor3

Download refactor

Author: ppolewicz

.github/workflows/ci.yml

@@ -44,8 +44,30 @@ jobs:
         run: python -m pip install --upgrade nox pip setuptools
       - name: Build the distribution
         run: nox -vs build
-  test:
+  cleanup_buckets:
     needs: lint
+    env:
+      B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
+      B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+      - name: Install dependencies
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        run: python -m pip install --upgrade nox pip setuptools
+      - name: Find and remove old buckets
+        if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}  # TODO: skip this whole job instead
+        run: nox -vs cleanup_old_buckets
+  test:
+    needs: cleanup_buckets
     env:
       B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
       B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
@@ -72,7 +94,7 @@ jobs:
         run: nox -vs unit
       - name: Run integration tests
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
-        run: nox -vs integration
+        run: nox -vs integration -- --dont-cleanup-old-buckets
   doc:
     needs: build
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+* `download_file_*` methods refactored to allow for inspecting DownloadVersion before downloading the whole file
+
+### Fixed
+* Fix EncryptionSetting.from_response_headers
+* Fix FileVersion.size and FileVersion.mod_time_millis type ambiguity
+* Old buckets (from past tests) are cleaned up before running integration tests in a single thread
+
 ## [1.9.0] - 2021-06-07
 
 ### Added

b2sdk/_v2/__init__.py

@@ -25,9 +25,10 @@
 from b2sdk.encryption.setting import EncryptionSetting
 from b2sdk.encryption.setting import EncryptionSettingFactory
 from b2sdk.encryption.setting import EncryptionKey
-from b2sdk.encryption.setting import SSE_NONE, SSE_B2_AES, SSE_C_KEY_ID_FILE_INFO_KEY_NAME
+from b2sdk.encryption.setting import SSE_NONE, SSE_B2_AES
 from b2sdk.encryption.types import EncryptionAlgorithm
 from b2sdk.encryption.types import EncryptionMode
+from b2sdk.http_constants import SSE_C_KEY_ID_FILE_INFO_KEY_NAME
 
 # account info
 
@@ -50,23 +51,28 @@
     b2_url_encode,
     b2_url_decode,
     choose_part_ranges,
+    current_time_millis,
     fix_windows_path_limit,
     format_and_scale_fraction,
     format_and_scale_number,
     hex_sha1_of_stream,
     hex_sha1_of_bytes,
+    hex_sha1_of_file,
     TempDir,
 )
 
 from b2sdk.utils import trace_call
 
 # data classes
 
+from b2sdk.file_version import DownloadVersion
+from b2sdk.file_version import DownloadVersionFactory
 from b2sdk.file_version import FileIdAndName
 from b2sdk.file_version import FileVersion
 from b2sdk.file_version import FileVersionFactory
 from b2sdk.large_file.part import Part
 from b2sdk.large_file.unfinished_large_file import UnfinishedLargeFile
+from b2sdk.utils.range_ import Range
 
 # file lock
 
@@ -116,11 +122,7 @@
 
 # source / destination
 
-from b2sdk.download_dest import AbstractDownloadDestination
-from b2sdk.download_dest import DownloadDestBytes
-from b2sdk.download_dest import DownloadDestLocalFile
-from b2sdk.download_dest import DownloadDestProgressWrapper
-from b2sdk.download_dest import PreSeekedDownloadDest
+from b2sdk.transfer.inbound.downloaded_file import DownloadedFile
 
 from b2sdk.transfer.outbound.outbound_source import OutboundTransferSource
 from b2sdk.transfer.outbound.copy_source import CopySource
@@ -136,7 +138,6 @@
 # trasfer
 
 from b2sdk.transfer.inbound.downloader.abstract import AbstractDownloader
-from b2sdk.transfer.inbound.file_metadata import FileMetadata
 from b2sdk.transfer.outbound.large_file_upload_state import LargeFileUploadState
 from b2sdk.transfer.inbound.downloader.parallel import AbstractDownloaderThread
 from b2sdk.transfer.inbound.downloader.parallel import FirstPartDownloaderThread
@@ -145,7 +146,6 @@
 from b2sdk.transfer.inbound.downloader.parallel import PartToDownload
 from b2sdk.transfer.inbound.downloader.parallel import WriterThread
 from b2sdk.transfer.outbound.progress_reporter import PartProgressReporter
-from b2sdk.transfer.inbound.downloader.range import Range
 from b2sdk.transfer.inbound.downloader.simple import SimpleDownloader
 
 # sync
@@ -208,4 +208,5 @@
 from b2sdk.cache import AuthInfoCache
 from b2sdk.cache import DummyCache
 from b2sdk.cache import InMemoryCache
+from b2sdk.http_constants import SRC_LAST_MODIFIED_MILLIS
 from b2sdk.session import B2Session

b2sdk/api.py

@@ -8,22 +8,24 @@
 #
 ######################################################################
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
 
 from .bucket import Bucket, BucketFactory
 from .encryption.setting import EncryptionSetting
 from .exception import NonExistentBucket, RestrictedBucket
 from .file_lock import FileRetentionSetting, LegalHold
-from .file_version import FileIdAndName, FileVersionFactory
+from .file_version import DownloadVersionFactory, FileIdAndName, FileVersionFactory
 from .large_file.services import LargeFileServices
 from .raw_api import API_VERSION
+from .progress import AbstractProgressListener
 from .session import B2Session
 from .transfer import (
     CopyManager,
     DownloadManager,
     Emerger,
     UploadManager,
 )
+from .transfer.inbound.downloaded_file import DownloadedFile
 from .utils import B2TraceMeta, b2_url_encode, limit_trace_arguments
 
 
@@ -86,6 +88,7 @@ class handles several things that simplify the task of uploading
     BUCKET_CLASS = staticmethod(Bucket)
     SESSION_CLASS = staticmethod(B2Session)
     FILE_VERSION_FACTORY_CLASS = staticmethod(FileVersionFactory)
+    DOWNLOAD_VERSION_FACTORY_CLASS = staticmethod(DownloadVersionFactory)
 
     def __init__(
         self,
@@ -123,6 +126,7 @@ def __init__(
         """
         self.session = self.SESSION_CLASS(account_info=account_info, cache=cache, raw_api=raw_api)
         self.file_version_factory = self.FILE_VERSION_FACTORY_CLASS(self)
+        self.download_version_factory = self.DOWNLOAD_VERSION_FACTORY_CLASS(self)
         self.services = Services(
             self,
             max_upload_workers=max_upload_workers,
@@ -220,39 +224,23 @@ def create_bucket(
 
     def download_file_by_id(
         self,
-        file_id,
-        download_dest,
-        progress_listener=None,
-        range_=None,
+        file_id: str,
+        progress_listener: Optional[AbstractProgressListener] = None,
+        range_: Optional[Tuple[int, int]] = None,
         encryption: Optional[EncryptionSetting] = None,
-    ):
+    ) -> DownloadedFile:
         """
         Download a file with the given ID.
 
         :param str file_id: a file ID
-        :param download_dest: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.DownloadDestLocalFile`,\
-        :class:`~b2sdk.v1.DownloadDestBytes`,\
-        :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\
-        :class:`~b2sdk.v1.PreSeekedDownloadDest`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination`
-        :param progress_listener: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.PartProgressReporter`,\
-        :class:`~b2sdk.v1.TqdmProgressListener`,\
-        :class:`~b2sdk.v1.SimpleProgressListener`,\
-        :class:`~b2sdk.v1.DoNothingProgressListener`,\
-        :class:`~b2sdk.v1.ProgressListenerForTest`,\
-        :class:`~b2sdk.v1.SyncFileReporter`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractProgressListener`
-        :param list range_: a list of two integers, the first one is a start\
+        :param progress_listener: a progress listener object to use, or ``None`` to not track progress
+        :param range_: a list of two integers, the first one is a start\
         position, and the second one is the end position in the file
-        :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown)
-        :return: context manager that returns an object that supports iter_content()
+        :param encryption: encryption settings (``None`` if unknown)
         """
         url = self.session.get_download_url_by_id(file_id)
         return self.services.download_manager.download_file_from_url(
             url,
-            download_dest,
             progress_listener,
             range_,
             encryption,

b2sdk/bucket.py

@@ -9,7 +9,7 @@
 ######################################################################
 
 import logging
-from typing import Optional
+from typing import Optional, Tuple
 
 from .encryption.setting import EncryptionSetting, EncryptionSettingFactory
 from .encryption.types import EncryptionMode
@@ -21,10 +21,11 @@
     UNKNOWN_BUCKET_RETENTION,
     LegalHold,
 )
-from .file_version import FileVersion
-from .progress import DoNothingProgressListener
+from .file_version import DownloadVersion, FileVersion
+from .progress import AbstractProgressListener, DoNothingProgressListener
 from .transfer.emerge.executor import AUTO_CONTENT_TYPE
 from .transfer.emerge.write_intent import WriteIntent
+from .transfer.inbound.downloaded_file import DownloadedFile
 from .transfer.outbound.copy_source import CopySource
 from .transfer.outbound.upload_source import UploadSourceBytes, UploadSourceLocalFile
 from .utils import B2TraceMeta, disable_trace, limit_trace_arguments
@@ -157,67 +158,51 @@ def cancel_large_file(self, file_id):
 
     def download_file_by_id(
         self,
-        file_id,
-        download_dest,
-        progress_listener=None,
-        range_=None,
+        file_id: str,
+        progress_listener: Optional[AbstractProgressListener] = None,
+        range_: Optional[Tuple[int, int]] = None,
         encryption: Optional[EncryptionSetting] = None,
-    ):
+    ) -> DownloadedFile:
         """
         Download a file by ID.
 
         .. note::
           download_file_by_id actually belongs in :py:class:`b2sdk.v1.B2Api`, not in :py:class:`b2sdk.v1.Bucket`; we just provide a convenient redirect here
 
-        :param str file_id: a file ID
-        :param download_dest: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.DownloadDestLocalFile`,\
-        :class:`~b2sdk.v1.DownloadDestBytes`,\
-        :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\
-        :class:`~b2sdk.v1.PreSeekedDownloadDest`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination`
-        :param b2sdk.v1.AbstractProgressListener, None progress_listener: a progress listener object to use, or ``None`` to not report progress
-        :param tuple[int, int] range_: two integer values, start and end offsets
-        :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown)
+        :param file_id: a file ID
+        :param progress_listener: a progress listener object to use, or ``None`` to not track progress
+        :param range_: two integer values, start and end offsets
+        :param encryption: encryption settings (``None`` if unknown)
         """
         return self.api.download_file_by_id(
             file_id,
-            download_dest,
             progress_listener,
             range_=range_,
             encryption=encryption,
         )
 
     def download_file_by_name(
         self,
-        file_name,
-        download_dest,
-        progress_listener=None,
-        range_=None,
+        file_name: str,
+        progress_listener: Optional[AbstractProgressListener] = None,
+        range_: Optional[Tuple[int, int]] = None,
         encryption: Optional[EncryptionSetting] = None,
-    ):
+    ) -> DownloadedFile:
         """
         Download a file by name.
 
         .. seealso::
 
             :ref:`Synchronizer <sync>`, a *high-performance* utility that synchronizes a local folder with a Bucket.
 
-        :param str file_name: a file name
-        :param download_dest: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.DownloadDestLocalFile`,\
-        :class:`~b2sdk.v1.DownloadDestBytes`,\
-        :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\
-        :class:`~b2sdk.v1.PreSeekedDownloadDest`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination`
-        :param b2sdk.v1.AbstractProgressListener, None progress_listener: a progress listener object to use, or ``None`` to not track progress
-        :param tuple[int, int] range_: two integer values, start and end offsets
-        :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown)
+        :param file_name: a file name
+        :param progress_listener: a progress listener object to use, or ``None`` to not track progress
+        :param range_: two integer values, start and end offsets
+        :param encryption: encryption settings (``None`` if unknown)
         """
         url = self.api.session.get_download_url_by_name(self.name, file_name)
         return self.api.services.download_manager.download_file_from_url(
             url,
-            download_dest,
             progress_listener,
             range_,
             encryption=encryption,
@@ -232,15 +217,14 @@ def get_file_info_by_id(self, file_id: str) -> FileVersion:
         """
         return self.api.file_version_factory.from_api_response(self.api.get_file_info(file_id))
 
-    def get_file_info_by_name(self, file_name: str) -> FileVersion:
+    def get_file_info_by_name(self, file_name: str) -> DownloadVersion:
         """
-        Gets a file version's by its name.
+        Gets a file's DownloadVersion by name.
 
         :param str file_name: the name of the file who's info will be retrieved.
-        :rtype: generator[b2sdk.v1.FileVersionInfo]
         """
         try:
-            return self.api.file_version_factory.from_response_headers(
+            return self.api.download_version_factory.from_response_headers(
                 self.api.session.get_file_info_by_name(self.name, file_name)
             )
         except FileOrBucketNotFound: