Skip to content

Commit 24fb01b

Browse files
authored
Don't compute checksums if not necessary in datasets-cli test (huggingface#5603)
don't compute checksums if not necessary
1 parent 6569014 commit 24fb01b

File tree

1 file changed

+3
-1
lines changed

1 file changed

+3
-1
lines changed

src/datasets/commands/test.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,9 @@ def get_builders() -> Generator[DatasetBuilder, None, None]:
140140

141141
for j, builder in enumerate(get_builders()):
142142
print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})")
143-
builder._record_infos = True
143+
builder._record_infos = os.path.exists(
144+
os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME)
145+
) # record checksums only if we need to update a (deprecated) dataset_infos.json
144146
builder.download_and_prepare(
145147
download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
146148
if not self._force_redownload

0 commit comments

Comments
 (0)