Skip to content

Commit 5fe0aa9

Browse files
committed
Skip reads when completely overwriting boundary chunks
Uses `slice(..., None)` to indicate that a `chunk_selection` ends at the boundary of the current chunk. Also does so for a last chunk that is shorter than the chunk size. `is_total_slice` now understands this convention, and correctly detects boundary chunks as total slices. Closes zarr-developers#757
1 parent fc08f31 commit 5fe0aa9

File tree

2 files changed

+23
-5
lines changed

2 files changed

+23
-5
lines changed

src/zarr/core/indexing.py

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -403,16 +403,24 @@ def __iter__(self) -> Iterator[ChunkDimProjection]:
403403
dim_chunk_sel_start = self.start - dim_offset
404404
dim_out_offset = 0
405405

406+
# Use None to indicate this selection ends at the chunk edge
407+
# This is useful for is_total_slice at boundary chunks,
406408
if self.stop > dim_limit:
407409
# selection ends after current chunk
408-
dim_chunk_sel_stop = dim_chunk_len
410+
dim_chunk_sel_stop = None # dim_chunk_len
409411

410412
else:
411413
# selection ends within current chunk
412-
dim_chunk_sel_stop = self.stop - dim_offset
414+
if dim_chunk_ix == (self.nchunks - 1):
415+
# all of the last chunk is included
416+
dim_chunk_sel_stop = None
417+
else:
418+
dim_chunk_sel_stop = self.stop - dim_offset
413419

414420
dim_chunk_sel = slice(dim_chunk_sel_start, dim_chunk_sel_stop, self.step)
415-
dim_chunk_nitems = ceildiv((dim_chunk_sel_stop - dim_chunk_sel_start), self.step)
421+
dim_chunk_nitems = ceildiv(
422+
((dim_chunk_sel_stop or dim_chunk_len) - dim_chunk_sel_start), self.step
423+
)
416424

417425
# If there are no elements on the selection within this chunk, then skip
418426
if dim_chunk_nitems == 0:
@@ -1376,7 +1384,17 @@ def is_total_slice(item: Selection, shape: ChunkCoords) -> bool:
13761384
isinstance(dim_sel, slice)
13771385
and (
13781386
(dim_sel == slice(None))
1379-
or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None]))
1387+
or (
1388+
dim_sel.stop is not None
1389+
and (dim_sel.stop - dim_sel.start == dim_len)
1390+
and (dim_sel.step in [1, None])
1391+
)
1392+
# starts exactly at a chunk
1393+
or (
1394+
(dim_sel.start % dim_len == 0)
1395+
and dim_sel.stop is None
1396+
and (dim_sel.step in [1, None])
1397+
)
13801398
)
13811399
for dim_sel, dim_len in zip(item, shape, strict=False)
13821400
)

src/zarr/storage/_logging.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def log(self, hint: Any = "") -> Generator[None, None, None]:
8888
op = f"{type(self._store).__name__}.{method}"
8989
if hint:
9090
op = f"{op}({hint})"
91-
self.logger.info("Calling %s", op)
91+
self.logger.info(" Calling %s", op)
9292
start_time = time.time()
9393
try:
9494
self.counter[method] += 1

0 commit comments

Comments
 (0)