Merge pull request #68 from deepghs/dev/download

dev(narugo): add soft check for download single file

Author: narugo1992

Makefile

@@ -38,7 +38,8 @@ unittest:
 		$(shell for type in ${COV_TYPES}; do echo "--cov-report=$$type"; done) \
 		--cov="${RANGE_SRC_DIR}" \
 		$(if ${MIN_COVERAGE},--cov-fail-under=${MIN_COVERAGE},) \
-		$(if ${WORKERS},-n ${WORKERS},)
+		$(if ${WORKERS},-n ${WORKERS},) \
+		--reruns 8 --reruns-delay 2 --only-rerun '(OSError|Timeout|HTTPError.*502|HTTPError.*504|check your connection)'
 
 docs:
 	$(MAKE) -C "${DOC_DIR}" build

hfutils/entry/download.py

@@ -142,6 +142,7 @@ def download(
                 file_in_repo=file_in_repo,
                 repo_type=repo_type,
                 revision=revision,
+                soft_mode_when_check=soft_mode_when_check,
             )
 
         elif archive_in_repo:

hfutils/operate/download.py

@@ -79,7 +79,7 @@ def _raw_download_file(td: str, local_file: str, repo_id: str, file_in_repo: str
 
 def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
                           repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
-                          hf_token: Optional[str] = None):
+                          soft_mode_when_check: bool = False, hf_token: Optional[str] = None):
     """
     Download a file from a Hugging Face repository and save it to a local file.
 
@@ -93,19 +93,32 @@ def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
     :type repo_type: RepoTypeTyping
     :param revision: The revision of the repository (e.g., branch, tag, commit hash).
     :type revision: str
+    :param soft_mode_when_check: Just check the size of the expected file when enabled. Default is False.
+    :type soft_mode_when_check: bool
     :param hf_token: Huggingface token for API client, use ``HF_TOKEN`` variable if not assigned.
     :type hf_token: str, optional
     """
     with TemporaryDirectory() as td:
-        _raw_download_file(
-            td=td,
-            local_file=local_file,
-            repo_id=repo_id,
-            file_in_repo=file_in_repo,
-            repo_type=repo_type,
-            revision=revision,
-            hf_token=hf_token,
-        )
+        if os.path.exists(local_file) and is_local_file_ready(
+                repo_id=repo_id,
+                repo_type=repo_type,
+                local_file=local_file,
+                file_in_repo=file_in_repo,
+                revision=revision,
+                hf_token=hf_token,
+                soft_mode=soft_mode_when_check,
+        ):
+            logging.info(f'Local file {local_file!r} is ready, download skipped.')
+        else:
+            _raw_download_file(
+                td=td,
+                local_file=local_file,
+                repo_id=repo_id,
+                file_in_repo=file_in_repo,
+                repo_type=repo_type,
+                revision=revision,
+                hf_token=hf_token,
+            )
 
 
 def download_archive_as_directory(local_directory: str, repo_id: str, file_in_repo: str,

test/operate/test_download.py

@@ -12,14 +12,46 @@
 class TestOperateDownload:
     def test_download_file_to_file(self):
         target_file = get_testfile('mashu.png')
-        with isolated_directory():
+
+        call_times = 0
+
+        def _my_download(*args, **kwargs):
+            nonlocal call_times
+            call_times += 1
+            return _raw_download_file(*args, **kwargs)
+
+        with patch('hfutils.operate.download._raw_download_file', _my_download), \
+                isolated_directory():
+            download_file_to_file(
+                'mashu_download.png',
+                repo_id='deepghs/game_character_skins',
+                file_in_repo='fgo/1/常夏的泳装Ver_02.png',
+            )
+            file_compare(target_file, 'mashu_download.png')
+
+        assert call_times == 1
+
+    def test_download_file_to_file_skip(self):
+        target_file = get_testfile('mashu.png')
+
+        call_times = 0
+
+        def _my_download(*args, **kwargs):
+            nonlocal call_times
+            call_times += 1
+            return _raw_download_file(*args, **kwargs)
+
+        with patch('hfutils.operate.download._raw_download_file', _my_download), \
+                isolated_directory({'mashu_download.png': target_file}):
             download_file_to_file(
                 'mashu_download.png',
                 repo_id='deepghs/game_character_skins',
                 file_in_repo='fgo/1/常夏的泳装Ver_02.png',
             )
             file_compare(target_file, 'mashu_download.png')
 
+        assert call_times == 0
+
     def test_download_archive_as_directory(self):
         target_dir = get_testfile('surtr_ds')
         with isolated_directory():