Skip to content

Commit 3d4c9e1

Browse files
authored
feat: eliminate redundant host-side solver run for Docker builds (#1081)
## Summary - When building with Docker, `get_package_paths()` triggers a full finalized conda solver run on the host via `api.render(finalize=True)`. This is redundant because Docker's `conda-build` re-solves from scratch anyway. - Adds a `finalize` parameter to `get_package_paths()` and skips the expensive finalized render when a `docker_builder` is present - Non-finalized metas use `bypass_env_check=True`, which avoids costly dependency resolution. As documented in `utils.py:479-480`, this does not change the package build string. - Adds `--no-fast-resolve` CLI flag for debugging/opt-out ## Savings 10-60s per recipe by eliminating a redundant solver invocation. For R packages, a single solve can take 10+ minutes. ## Risks - **Medium**: Non-finalized metas may produce slightly different build strings if `pin_compatible` or `run_exports` affect the hash. This could cause: - False "not in channel" -> rebuilds something that exists (wasteful but not incorrect) - False "in channel" -> skips a needed build (lower risk since `check_recipe_skippable()` already does a fast check) - **Mitigation**: Debug logging when fast path is used; `--no-fast-resolve` opt-out flag - Only changes behavior when `docker_builder is not None` ## Test plan - [x] Run existing test suite: `pytest test/` - [x] Integration test with Docker: `bioconda-utils build recipes/ config.yml --docker` on pyfaidx - [x] Compare build results with and without `--no-fast-resolve` - [x] Verify built packages are identical
1 parent 975acdb commit 3d4c9e1

File tree

3 files changed

+24
-3
lines changed

3 files changed

+24
-3
lines changed

bioconda_utils/build.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ def build_recipes(
366366
live_logs: bool = True,
367367
exclude: List[str] = None,
368368
subdag_depth: int = None,
369+
fast_resolve: bool = True,
369370
):
370371
"""
371372
Build one or many bioconda packages.
@@ -482,7 +483,17 @@ def build_recipes(
482483

483484
logger.info("Determining expected packages for %s", recipe)
484485
try:
485-
pkg_paths = utils.get_package_paths(recipe, check_channels, force=force)
486+
# When building with Docker, skip the expensive finalized render
487+
# on the host since Docker's conda-build will re-solve anyway.
488+
# Non-finalized metas use bypass_env_check which avoids costly
489+
# dependency resolution. The --no-fast-resolve flag can override this.
490+
finalize = (docker_builder is None) if fast_resolve else True
491+
pkg_paths = utils.get_package_paths(
492+
recipe,
493+
check_channels,
494+
force=force,
495+
finalize=finalize,
496+
)
486497
except utils.DivergentBuildsError as exc:
487498
logger.error(
488499
"BUILD ERROR: packages with divergent build strings in repository "

bioconda_utils/cli.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -619,6 +619,12 @@ def do_lint(
619619
action="store_true",
620620
help="Disable live logging during the build process",
621621
)
622+
@arg(
623+
"--no-fast-resolve",
624+
action="store_true",
625+
help="Disable fast resolve: always run the full finalized conda solver on the host, "
626+
"even when building with Docker. Useful for debugging build string mismatches.",
627+
)
622628
@arg("--exclude", nargs="+", help="Packages to exclude during this run")
623629
@arg(
624630
"--subdag-depth",
@@ -652,6 +658,7 @@ def build(
652658
record_build_failures=False,
653659
skiplist_leafs=False,
654660
disable_live_logs=False,
661+
no_fast_resolve=False,
655662
exclude=None,
656663
subdag_depth=None,
657664
):
@@ -726,6 +733,7 @@ def build(
726733
live_logs=(not disable_live_logs),
727734
exclude=exclude,
728735
subdag_depth=subdag_depth,
736+
fast_resolve=not no_fast_resolve,
729737
)
730738
exit(0 if success else 1)
731739

bioconda_utils/utils.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,12 +1184,14 @@ def _filter_existing_packages(metas, check_channels):
11841184
return new_metas, existing_metas, divergent_builds
11851185

11861186

1187-
def get_package_paths(recipe, check_channels, force=False):
1187+
def get_package_paths(recipe, check_channels, force=False, finalize=True):
11881188
if not force:
11891189
if check_recipe_skippable(recipe, check_channels):
11901190
# NB: If we skip early here, we don't detect possible divergent builds.
11911191
return []
1192-
platform, metas = _load_platform_metas(recipe, finalize=True)
1192+
if not finalize:
1193+
logger.debug("Using non-finalized render for %s (fast resolve)", recipe)
1194+
platform, metas = _load_platform_metas(recipe, finalize=finalize)
11931195

11941196
# The recipe likely defined skip: True
11951197
if not metas:

0 commit comments

Comments
 (0)