diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 179fc55..1004cc8 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -22,15 +22,15 @@ jobs: - {os: macOS-latest, r: 'devel' } - {os: macOS-latest, r: 'release' } - {os: macOS-latest, r: 'oldrel' } - - {os: ubuntu-20.04, r: 'devel' } - - {os: ubuntu-20.04, r: 'release' } - - {os: ubuntu-20.04, r: 'oldrel' } - - {os: ubuntu-20.04, r: 'oldrel-1' } - - {os: ubuntu-20.04, r: 'oldrel-2' } - - {os: ubuntu-20.04, r: '3.4' } - - {os: ubuntu-20.04, r: 'release' , language: ko, label: ko } - - {os: ubuntu-20.04, r: 'release' , language: zh_CN, label: zh_CN } - - {os: ubuntu-20.04, r: 'release' , language: zh_TW, label: zh_TW } + - {os: ubuntu-22.04, r: 'devel' } + - {os: ubuntu-22.04, r: 'release' } + - {os: ubuntu-22.04, r: 'oldrel' } + - {os: ubuntu-22.04, r: 'oldrel-1' } + - {os: ubuntu-22.04, r: 'oldrel-2' } + - {os: ubuntu-22.04, r: '3.6' } + - {os: ubuntu-22.04, r: 'release' , language: ko, label: ko } + - {os: ubuntu-22.04, r: 'release' , language: zh_CN, label: zh_CN } + - {os: ubuntu-22.04, r: 'release' , language: zh_TW, label: zh_TW } env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes @@ -47,7 +47,7 @@ jobs: LANGUAGE: ${{ matrix.config.language }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 diff --git a/.github/workflows/covr.yaml b/.github/workflows/covr.yaml index b26bbae..458e1a0 100644 --- a/.github/workflows/covr.yaml +++ b/.github/workflows/covr.yaml @@ -8,7 +8,7 @@ jobs: timeout-minutes: 20 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 name: covr @@ -17,7 +17,7 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - RSPM: https://packagemanager.rstudio.com/cran/__linux__/focal/latest + RSPM: https://packagemanager.rstudio.com/cran/__linux__/jammy/latest R_REMOTES_NO_ERRORS_FROM_WARNINGS: true ## R CMD check _R_CHECK_LENGTH_1_CONDITION_: true diff --git a/.github/workflows/future_tests.yaml b/.github/workflows/future_tests.yaml index 32711c7..d405e28 100644 --- a/.github/workflows/future_tests.yaml +++ b/.github/workflows/future_tests.yaml @@ -8,7 +8,7 @@ jobs: timeout-minutes: 30 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 name: future.plan=${{ matrix.future.plan }} @@ -16,12 +16,14 @@ jobs: fail-fast: false matrix: future: - - { plan: 'future.batchtools::batchtools_local' } + - { plan: 'future.batchtools::batchtools_local' } + - { plan: 'future.batchtools::batchtools_multicore' } + - { plan: 'future.batchtools::batchtools_bash' } - { plan: 'future.batchtools::batchtools_interactive' } env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - RSPM: https://packagemanager.rstudio.com/cran/__linux__/focal/latest + RSPM: https://packagemanager.rstudio.com/cran/__linux__/jammy/latest R_REMOTES_NO_ERRORS_FROM_WARNINGS: true ## R CMD check _R_CHECK_LENGTH_1_CONDITION_: true @@ -32,7 +34,7 @@ jobs: R_FUTURE_RNG_ONMISUSE: error steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: r-lib/actions/setup-pandoc@v2 diff --git a/.make/Makefile b/.make/Makefile index 7cc14cb..697e013 100644 --- a/.make/Makefile +++ b/.make/Makefile @@ -356,12 +356,12 @@ test_files: $(R_OUTDIR)/tests/*.R test: $(R_OUTDIR)/tests/%.R $(CD) $(R_OUTDIR)/tests;\ - $(R_SCRIPT) -e "for (f in list.files(pattern='[.]R$$')) { print(f); source(f, chdir=TRUE, echo=TRUE) }" + $(R_SCRIPT) -e "for (f in list.files(pattern='[.]R$$')) { print(f); source(f, echo=TRUE) }" test_full: $(R_OUTDIR)/tests/%.R $(CD) $(R_OUTDIR)/tests;\ export _R_CHECK_FULL_=TRUE;\ - $(R_SCRIPT) -e "for (f in list.files(pattern='[.]R$$')) { print(f); source(f, chdir=TRUE, echo=TRUE) }" + $(R_SCRIPT) -e "for (f in list.files(pattern='[.]R$$')) { print(f); source(f, echo=TRUE) }" # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/DESCRIPTION b/DESCRIPTION index 939f0ca..3ba96b8 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,11 +1,11 @@ Package: future.batchtools -Version: 0.12.0 +Version: 0.12.1 Depends: R (>= 3.2.0), parallelly, future (>= 1.31.0) Imports: - batchtools (>= 0.9.13), + batchtools (>= 0.9.16), utils Suggests: globals, @@ -16,8 +16,10 @@ Suggests: VignetteBuilder: R.rsp Title: A Future API for Parallel and Distributed Processing using 'batchtools' -Authors@R: c(person("Henrik", "Bengtsson", role=c("aut", "cre", "cph"), - email = "henrikb@braju.com")) +Authors@R: c(person("Henrik", "Bengtsson", + role = c("aut", "cre", "cph"), + email = "henrikb@braju.com", + comment = c(ORCID = "0000-0002-7579-5165"))) Description: Implementation of the Future API on top of the 'batchtools' package. This allows you to process futures, as defined by the 'future' package, in parallel out of the box, not only on your local machine or ad-hoc diff --git a/NEWS.md b/NEWS.md index 3a69839..81a63a5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,11 @@ +# Version 0.12.1 [2023-12-19] + +## Bug Fixes + + * `plan(batchtools_multicore)` did not support `workers` argument + being a function. + + # Version 0.12.0 [2023-02-24] ## New Features @@ -76,8 +84,8 @@ Because of this, we temporarily set `fs.latency = 1.0` (second) timeout for trying to find the log file. This makes a big difference in case the template used a `--output=` location - other than `--output=<%= log.file %>`l in such cases the log file - would not be found, requiring a timeout. + other than `--output=<%= log.file %>`. In such cases, the log file + would not be found, requiring a full timeout to complete. ## Bug Fixes diff --git a/R/BatchtoolsFuture-class.R b/R/BatchtoolsFuture-class.R index 3c48684..71f3663 100644 --- a/R/BatchtoolsFuture-class.R +++ b/R/BatchtoolsFuture-class.R @@ -294,6 +294,7 @@ loggedError <- function(...) UseMethod("loggedError") #' @importFrom batchtools getErrorMessages +#' @rdname loggedOutput #' @export loggedError.BatchtoolsFuture <- function(future, ...) { stat <- status(future) diff --git a/R/batchtools_multicore.R b/R/batchtools_multicore.R index b1c2546..63b3f05 100644 --- a/R/batchtools_multicore.R +++ b/R/batchtools_multicore.R @@ -33,7 +33,11 @@ batchtools_multicore <- function(expr, envir = parent.frame(), registry = list(), ...) { if (substitute) expr <- substitute(expr) - if (is.null(workers)) workers <- availableCores(constraints = "multicore") + if (is.null(workers)) { + workers <- availableCores(constraints = "multicore") + } else if (is.function(workers)) { + workers <- workers() + } stop_if_not(length(workers) == 1L, is.numeric(workers), is.finite(workers), workers >= 1L) diff --git a/R/nbrOfWorkers.R b/R/nbrOfWorkers.R index 2e1750b..ee1ce8b 100644 --- a/R/nbrOfWorkers.R +++ b/R/nbrOfWorkers.R @@ -18,6 +18,7 @@ nbrOfWorkers.batchtools <- function(evaluator) { expr <- formals(evaluator)$workers workers <- eval(expr, enclos = baseenv()) if (!is.null(workers)) { + if (is.function(workers)) workers <- workers() stop_if_not(length(workers) >= 1) if (is.numeric(workers)) return(prod(workers)) if (is.character(workers)) return(length(workers)) @@ -57,6 +58,7 @@ nbrOfWorkers.batchtools_multicore <- function(evaluator) { ## 1. Infer from 'workers' argument expr <- formals(evaluator)$workers workers <- eval(expr, enclos = baseenv()) + if (is.function(workers)) workers <- workers() stop_if_not(length(workers) == 1L, is.numeric(workers), !is.na(workers), is.finite(workers), workers >= 1) workers } diff --git a/R/waitForWorker.R b/R/waitForWorker.R index 96cf620..59285c2 100644 --- a/R/waitForWorker.R +++ b/R/waitForWorker.R @@ -44,7 +44,8 @@ waitForWorker.BatchtoolsFuture <- function(future, await = NULL, timeout = getOption("future.wait.timeout", 30 * 24 * 60 * 60), delta = getOption("future.wait.interval", 0.2), - alpha = getOption("future.wait.alpha", 1.01)) { + alpha = getOption("future.wait.alpha", 1.01), + ...) { debug <- getOption("future.debug", FALSE) stop_if_not(is.null(await) || is.function(await)) diff --git a/README.md b/README.md index d0c7f7b..1ccd5e2 100644 --- a/README.md +++ b/README.md @@ -7,24 +7,26 @@ # future.batchtools: A Future API for Parallel and Distributed Processing using 'batchtools' ## Introduction -The [future] package provides a generic API for using futures in R. -A future is a simple yet powerful mechanism to evaluate an R expression -and retrieve its value at some point in time. Futures can be resolved -in many different ways depending on which strategy is used. -There are various types of synchronous and asynchronous futures to -choose from in the [future] package. - -This package, [future.batchtools], provides a type of futures that -utilizes the [batchtools] package. This means that _any_ type of -backend that the batchtools package supports can be used as a future. -More specifically, future.batchtools will allow you or users of your -package to leverage the compute power of high-performance computing -(HPC) clusters via a simple switch in settings - without having to -change any code at all. - -For instance, if batchtools is properly configures, the below two + +The **[future]** package provides a generic API for using futures in +R. A future is a simple yet powerful mechanism to evaluate an R +expression and retrieve its value at some point in time. Futures can +be resolved in many different ways depending on which strategy is +used. There are various types of synchronous and asynchronous futures +to choose from in the **[future]** package. + +This package, **[future.batchtools]**, provides a type of futures that +utilizes the **[batchtools]** package. This means that _any_ type of +backend that the **batchtools** package supports can be used as a +future. More specifically, **future.batchtools** will allow you or +users of your package to leverage the compute power of +high-performance computing (HPC) clusters via a simple switch in +settings - without having to change any code at all. + +For instance, if **batchtools** is properly configures, the below two expressions for futures `x` and `y` will be processed on two different compute nodes: + ```r > library("future.batchtools") > plan(batchtools_torque) @@ -34,8 +36,9 @@ compute nodes: > x + y [1] 5.85 ``` -This is obviously a toy example to illustrate what futures look like -and how to work with them. + +This is just a toy example to illustrate what futures look like and +how to work with them. A more realistic example comes from the field of cancer research where very large data FASTQ files, which hold a large number of short @@ -46,6 +49,7 @@ we can have each file be processed by a separate compute node and each node we can use 24 parallel processes such that each process aligns a separate chromosome. Here is an outline of how this nested parallelism could be implemented using futures. + ```r library("future") library("listenv") @@ -81,22 +85,29 @@ for (ss in seq_along(fqs)) { ## Resolve the "sample" futures and return as a list bams <- as.list(bams) ``` -Note that a user who do not have access to a cluster could use the same script processing samples sequentially and chromosomes in parallel on a single machine using: + +Note that a user who do not have access to a cluster could use the +same script processing samples sequentially and chromosomes in +parallel on a single machine using: + ```r plan(list(sequential, multisession)) ``` + or samples in parallel and chromosomes sequentially using: + ```r plan(list(multisession, sequential)) ``` For an introduction as well as full details on how to use futures, -please consult the package vignettes of the [future] package. +please consult the package vignettes of the **[future]** package. ## Choosing batchtools backend -The future.batchtools package implements a generic future wrapper + +The **future.batchtools** package implements a generic future wrapper for all batchtools backends. Below are the most common types of batchtools backends. @@ -109,18 +120,22 @@ batchtools backends. | `batchtools_lsf` | Futures are evaluated via a [Load Sharing Facility (LSF)] job scheduler | N/A | `batchtools_openlava` | Futures are evaluated via an [OpenLava] job scheduler | N/A | `batchtools_custom` | Futures are evaluated via a custom batchtools configuration R script or via a set of cluster functions | N/A -| `batchtools_interactive` | sequential evaluation in the calling R environment | `plan(transparent)` | `batchtools_multicore` | parallel evaluation by forking the current R process | `plan(multicore)` | `batchtools_local` | sequential evaluation in a separate R process (on current machine) | `plan(cluster, workers = "localhost")` ### Examples -Below is an examples illustrating how to use `batchtools_torque` to configure the batchtools backend. For further details and examples on how to configure batchtools, see the [batchtools configuration] wiki page. +Below is an examples illustrating how to use `batchtools_torque` to +configure the batchtools backend. For further details and examples on +how to configure batchtools, see the [batchtools configuration] wiki +page. + +To configure **batchtools** for job schedulers we need to setup a +`*.tmpl` template file that is used to generate the script used by the +scheduler. This is what a template file for TORQUE / PBS may look +like: -To configure batchtools for job schedulers we need to setup a `*.tmpl` template -file that is used to generate the script used by the scheduler. -This is what a template file for TORQUE / PBS may look like: ```sh #!/bin/bash @@ -147,45 +162,69 @@ This is what a template file for TORQUE / PBS may look like: ## Launch R and evaluated the batchtools R job Rscript -e 'batchtools::doJobCollection("<%= uri %>")' ``` -If this template is saved to file `batchtools.torque.tmpl` (without period) -in the working directory or as `.batchtools.torque.tmpl` (with period) the -user's home directory, then it will be automatically located by the -batchtools framework and loaded when doing: + +If this template is saved to file `batchtools.torque.tmpl` (without +period) in the working directory or as `.batchtools.torque.tmpl` (with +period) the user's home directory, then it will be automatically +located by the **batchtools** framework and loaded when doing: + ```r > plan(batchtools_torque) ``` -Resource parameters can be specified via argument `resources` which should be a named list and is passed as is to the template file. For example, to request that each job would get alloted 12 cores (one a single machine) and up to 5 GiB of memory, use: + +Resource parameters can be specified via argument `resources` which +should be a named list and is passed as is to the template file. For +example, to request that each job would get alloted 12 cores (one a +single machine) and up to 5 GiB of memory, use: + ```r > plan(batchtools_torque, resources = list(nodes = "1:ppn=12", vmem = "5gb")) ``` -To specify the `resources` argument at the same time as using nested future strategies, one can use `tweak()` to tweak the default arguments. For instance, +To specify the `resources` argument at the same time as using nested +future strategies, one can use `tweak()` to tweak the default +arguments. For instance, + ```r plan(list( tweak(batchtools_torque, resources = list(nodes = "1:ppn=12", vmem = "5gb")), multisession )) ``` -causes the first level of futures to be submitted via the TORQUE job scheduler requesting 12 cores and 5 GiB of memory per job. The second level of futures will be evaluated using multisession using the 12 cores given to each job by the scheduler. -A similar filename format is used for the other types of job schedulers supported. For instance, for Slurm the template file should be named `./batchtools.slurm.tmpl` or `~/.batchtools.slurm.tmpl` in order for +causes the first level of futures to be submitted via the TORQUE job +scheduler requesting 12 cores and 5 GiB of memory per job. The second +level of futures will be evaluated using multisession using the 12 +cores given to each job by the scheduler. + +A similar filename format is used for the other types of job +schedulers supported. For instance, for Slurm the template file +should be named `./batchtools.slurm.tmpl` or +`~/.batchtools.slurm.tmpl` in order for + ```r > plan(batchtools_slurm) ``` -to locate the file automatically. To specify this template file explicitly, use argument `template`, e.g. + +to locate the file automatically. To specify this template file +explicitly, use argument `template`, e.g. + ```r > plan(batchtools_slurm, template = "/path/to/batchtools.slurm.tmpl") ``` -For further details and examples on how to configure batchtools per se, see the [batchtools configuration] wiki page. +For further details and examples on how to configure **batchtools** per +se, see the [batchtools configuration] wiki page. ## Demos -The [future] package provides a demo using futures for calculating a -set of Mandelbrot planes. The demo does not assume anything about -what type of futures are used. -_The user has full control of how futures are evaluated_. -For instance, to use local batchtools futures, run the demo as: + +The **[future]** package provides a demo using futures for calculating +a set of Mandelbrot planes. The demo does not assume anything about +what type of futures are used. _The user has full control of how +futures are evaluated_. For instance, to use local batchtools +futures, run the demo as: + ```r library("future.batchtools") plan(batchtools_local) diff --git a/cran-comments.md b/cran-comments.md index 8a62aa2..54b0503 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,6 +1,6 @@ -# CRAN submission future.batchtools 0.12.0 +# CRAN submission future.batchtools 0.12.1 -on 2023-02-24 +on 2023-12-19 I've verified this submission has no negative impact on any of the 5 reverse package dependencies available on CRAN. @@ -13,13 +13,13 @@ Thank you The package has been verified using `R CMD check --as-cran` on: -| R version | GitHub | R-hub | mac/win-builder | -| --------- | ------ | ------ | --------------- | -| 3.4.x | L | | | -| 4.0.x | L | | | -| 4.1.x | L M W | M | | -| 4.2.x | L M W | L W | M1 W | -| devel | L M W | L | M1 W | +| R version | GitHub | R-hub | mac/win-builder | +| --------- | ------ | ----- | --------------- | +| 3.6.x | L | | | +| 4.1.x | L | | | +| 4.2.x | L M W | | | +| 4.3.x | L M W | . W | M1 W | +| devel | L M W | . | W | _Legend: OS: L = Linux, M = macOS, M1 = macOS M1, W = Windows_ @@ -31,7 +31,6 @@ res <- rhub::check(platforms = c( "debian-clang-devel", "fedora-gcc-devel", "debian-gcc-patched", - "macos-highsierra-release-cran", "windows-x86_64-release" )) print(res) @@ -40,47 +39,39 @@ print(res) gives ``` -── future.batchtools 0.12.0: OK +── future.batchtools 0.12.1: OK - Build ID: future.batchtools_0.12.0.tar.gz-641472edd3b5461d9aecf43df6440b2b + Build ID: future.batchtools_0.12.1.tar.gz-0cf2af4123d14ff3ac742eb611644044 Platform: Debian Linux, R-devel, clang, ISO-8859-15 locale - Submitted: 1h 47m 26.1s ago - Build time: 1h 10m 17.8s + Submitted: 3h 40m 16.2s ago + Build time: 1h 10m 30.4s 0 errors ✔ | 0 warnings ✔ | 0 notes ✔ -── future.batchtools 0.12.0: IN-PROGRESS +── future.batchtools 0.12.1: OK - Build ID: future.batchtools_0.12.0.tar.gz-7a87266bfdb64039bcec5c69000be01c + Build ID: future.batchtools_0.12.1.tar.gz-f68417d7155c4af6a5ac7d6619aff851 Platform: Fedora Linux, R-devel, GCC - Submitted: 1h 47m 26.1s ago + Submitted: 3h 40m 16.2s ago + Build time: 52m 37.8s +0 errors ✔ | 0 warnings ✔ | 0 notes ✔ -── future.batchtools 0.12.0: IN-PROGRESS +── future.batchtools 0.12.1: OK - Build ID: future.batchtools_0.12.0.tar.gz-ee81fb66f73c41d8908a2dff029f1479 + Build ID: future.batchtools_0.12.1.tar.gz-26c0209f468c4dd5af5f3f4f908cb3c7 Platform: Debian Linux, R-patched, GCC - Submitted: 1h 47m 26.1s ago - - -── future.batchtools 0.12.0: WARNING + Submitted: 3h 40m 16.2s ago + Build time: 1h 5m 2.8s - Build ID: future.batchtools_0.12.0.tar.gz-77b23546d6c94278b9afa8b7d1807eff - Platform: macOS 10.13.6 High Sierra, R-release, CRAN's setup - Submitted: 1h 47m 26.1s ago - Build time: 6m 1.1s - -❯ checking whether package ‘future.batchtools’ can be installed ... WARNING - See below... - -0 errors ✔ | 1 warning ✖ | 0 notes ✔ +0 errors ✔ | 0 warnings ✔ | 0 notes ✔ -── future.batchtools 0.12.0: OK +── future.batchtools 0.12.1: OK - Build ID: future.batchtools_0.12.0.tar.gz-680748ca9a384491932b2502119bab87 + Build ID: future.batchtools_0.12.1.tar.gz-64ca57c8a5b543e68b5122ba5eb2a3d1 Platform: Windows Server 2022, R-release, 32/64 bit - Submitted: 1h 47m 26.1s ago - Build time: 5m 43.7s + Submitted: 3h 40m 16.2s ago + Build time: 6m 41s 0 errors ✔ | 0 warnings ✔ | 0 notes ✔ ``` diff --git a/man/loggedOutput.Rd b/man/loggedOutput.Rd index e32bdcc..53688c3 100644 --- a/man/loggedOutput.Rd +++ b/man/loggedOutput.Rd @@ -3,9 +3,12 @@ \name{loggedOutput} \alias{loggedOutput} \alias{loggedError} +\alias{loggedError.BatchtoolsFuture} \title{Logged output of batchtools future} \usage{ loggedOutput(...) + +\method{loggedError}{BatchtoolsFuture}(future, ...) } \arguments{ \item{future}{The future.} diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml index c2e3386..0a0f12b 100644 --- a/pkgdown/_pkgdown.yml +++ b/pkgdown/_pkgdown.yml @@ -6,7 +6,7 @@ home: href: https://github.com/HenrikBengtsson/future.batchtools/milestones - text: The Futureverse Project href: https://www.futureverse.org/ - - text: Futureverse User Forum (new!) + - text: Futureverse User Forum href: https://github.com/HenrikBengtsson/future/discussions navbar: @@ -14,13 +14,11 @@ navbar: right: [futureverse, pkgs, cran, github] components: futureverse: - icon: fas fa-fast-backward + icon: fas fa-home href: https://www.futureverse.org/ pkgs: text: Packages menu: - - text: BiocParallel.FutureParam - href: https://BiocParallel.FutureParam.futureverse.org - text: doFuture href: https://doFuture.futureverse.org - text: furrr @@ -43,6 +41,12 @@ navbar: href: https://parallelly.futureverse.org - text: progressr href: https://progressr.futureverse.org + - text: BiocParallel.FutureParam (experimental) + href: https://BiocParallel.FutureParam.futureverse.org + - text: future.tools (experimental) + href: https://future.tools.futureverse.org + - text: future.mirai (experimental) + href: https://future.mirai.futureverse.org - text: future.mapreduce (experimental) href: https://future.mapreduce.futureverse.org - text: marshal (experimental) diff --git a/pkgdown/_pkgdown.yml.rsp b/pkgdown/_pkgdown.yml.rsp index eaf9d08..6d931ca 100644 --- a/pkgdown/_pkgdown.yml.rsp +++ b/pkgdown/_pkgdown.yml.rsp @@ -1,6 +1,6 @@ <% -pkgs <- c("globals", "listenv", "parallelly", "future", "future.apply", "furrr", "future.tests", "future.callr", "future.batchtools", "doFuture", "progressr", "BiocParallel.FutureParam") -pkgs_extra <- c("future.mapreduce", "marshal") +pkgs <- c("globals", "listenv", "parallelly", "future", "future.apply", "furrr", "future.tests", "future.callr", "future.batchtools", "doFuture", "progressr") +pkgs_extra <- c("BiocParallel.FutureParam", "future.tools", "future.mirai", "future.mapreduce", "marshal") pkgs <- c(sort(pkgs), pkgs_extra) urls <- sprintf("https://%s.futureverse.org", pkgs) names(urls) <- pkgs @@ -19,7 +19,7 @@ home: href: https://github.com/HenrikBengtsson/<%= pkg %>/milestones - text: The Futureverse Project href: https://www.futureverse.org/ - - text: Futureverse User Forum (new!) + - text: Futureverse User Forum href: https://github.com/HenrikBengtsson/future/discussions navbar: @@ -27,7 +27,7 @@ navbar: right: [futureverse, pkgs, cran, github] components: futureverse: - icon: fas fa-fast-backward + icon: fas fa-home href: https://www.futureverse.org/ pkgs: text: Packages diff --git a/revdep/README.md b/revdep/README.md index 5ee6b5d..c14b567 100644 --- a/revdep/README.md +++ b/revdep/README.md @@ -1,51 +1,50 @@ # Platform -|field |value | -|:--------|:----------------------------| -|version |R version 4.2.2 (2022-10-31) | -|os |Ubuntu 22.04.2 LTS | -|system |x86_64, linux-gnu | -|ui |X11 | -|language |en | -|collate |en_US.UTF-8 | -|ctype |en_US.UTF-8 | -|tz |Europe/Oslo | -|date |2023-02-24 | -|pandoc |2.9.2.1 @ /usr/bin/pandoc | +|field |value | +|:--------|:----------------------------------------------------------------------| +|version |R version 4.3.2 (2023-10-31) | +|os |Rocky Linux 8.8 (Green Obsidian) | +|system |x86_64, linux-gnu | +|ui |X11 | +|language |(EN) | +|collate |en_US.UTF-8 | +|ctype |en_US.UTF-8 | +|tz |America/Los_Angeles | +|date |2023-12-19 | +|pandoc |3.1.10 @ /wynton/home/cbi/shared/software/CBI/pandoc-3.1.10/bin/pandoc | # Dependencies -|package |old |new |Δ | -|:-----------------|:------|:------|:--| -|future.batchtools |0.11.0 |0.12.0 |* | -|backports |1.4.1 |1.4.1 | | -|base64url |1.4 |1.4 | | -|batchtools |0.9.16 |0.9.16 | | -|brew |1.0-8 |1.0-8 | | -|checkmate |2.1.0 |2.1.0 | | -|cli |3.6.0 |3.6.0 | | -|codetools |0.2-19 |0.2-19 | | -|crayon |1.5.2 |1.5.2 | | -|data.table |1.14.8 |1.14.8 | | -|digest |0.6.31 |0.6.31 | | -|ellipsis |0.3.2 |0.3.2 | | -|fs |1.6.1 |1.6.1 | | -|future |1.31.0 |1.31.0 | | -|globals |0.16.2 |0.16.2 | | -|glue |1.6.2 |1.6.2 | | -|hms |1.1.2 |1.1.2 | | -|lifecycle |1.0.3 |1.0.3 | | -|listenv |0.9.0 |0.9.0 | | -|parallelly |1.34.0 |1.34.0 | | -|pkgconfig |2.0.3 |2.0.3 | | -|prettyunits |1.1.1 |1.1.1 | | -|progress |1.2.2 |1.2.2 | | -|R6 |2.5.1 |2.5.1 | | -|rappdirs |0.3.3 |0.3.3 | | -|rlang |1.0.6 |1.0.6 | | -|stringi |1.7.12 |1.7.12 | | -|vctrs |0.5.2 |0.5.2 | | -|withr |2.5.0 |2.5.0 | | +|package |old |new |Δ | +|:-----------------|:-------|:-----------|:--| +|future.batchtools |0.12.0 |0.12.0-9003 |* | +|backports |1.4.1 |1.4.1 | | +|base64url |1.4 |1.4 | | +|batchtools |0.9.17 |0.9.17 | | +|brew |1.0-10 |1.0-10 | | +|checkmate |2.3.1 |2.3.1 | | +|cli |3.6.2 |3.6.2 | | +|codetools |0.2-19 |0.2-19 | | +|crayon |1.5.2 |1.5.2 | | +|data.table |1.14.10 |1.14.10 | | +|digest |0.6.33 |0.6.33 | | +|fs |1.6.3 |1.6.3 | | +|future |1.33.0 |1.33.0 | | +|globals |0.16.2 |0.16.2 | | +|glue |1.6.2 |1.6.2 | | +|hms |1.1.3 |1.1.3 | | +|lifecycle |1.0.4 |1.0.4 | | +|listenv |0.9.0 |0.9.0 | | +|parallelly |1.36.0 |1.36.0 | | +|pkgconfig |2.0.3 |2.0.3 | | +|prettyunits |1.2.0 |1.2.0 | | +|progress |1.2.3 |1.2.3 | | +|R6 |2.5.1 |2.5.1 | | +|rappdirs |0.3.3 |0.3.3 | | +|rlang |1.1.2 |1.1.2 | | +|stringi |1.8.3 |1.8.3 | | +|vctrs |0.6.5 |0.6.5 | | +|withr |2.5.2 |2.5.2 | | # Revdeps @@ -53,9 +52,9 @@ |package |version |error |warning |note | |:----------|:-------|:-----|:-------|:----| -|[batchtools](problems.md#batchtools)|0.9.16 | | |2 | +|[batchtools](problems.md#batchtools)|0.9.17 | | |2 | |FAMoS |0.3.0 | | | | -|nncc |1.0.0 | | | | -|[SimDesign](problems.md#simdesign)|2.10.1 | | |2 | -|targets |0.14.2 | | | | +|nncc |1.0.1 | | | | +|SimDesign |2.13 | | | | +|[targets](problems.md#targets)|1.4.0 | | |1 | diff --git a/revdep/problems.md b/revdep/problems.md index 18666e4..8472e7c 100644 --- a/revdep/problems.md +++ b/revdep/problems.md @@ -2,11 +2,11 @@
-* Version: 0.9.16 +* Version: 0.9.17 * GitHub: https://github.com/mllg/batchtools * Source code: https://github.com/cran/batchtools -* Date/Publication: 2023-02-03 13:52:38 UTC -* Number of recursive dependencies: 82 +* Date/Publication: 2023-04-20 14:20:06 UTC +* Number of recursive dependencies: 84 Run `revdep_details(, "batchtools")` for more info @@ -14,27 +14,48 @@ Run `revdep_details(, "batchtools")` for more info ## In both -* checking package dependencies ... NOTE +* checking dependencies in R code ... NOTE ``` - Package suggested but not available for checking: ‘doMPI’ + dev1.wynton.ucsf.edu:pid550439: PSM3 can't open nic unit: 0 (err=23) + dev1.wynton.ucsf.edu:pid550439.R: Unable to create send CQ of size 5080 on mlx5_0: Cannot allocate memory + dev1.wynton.ucsf.edu:pid550439.R: Unable to initialize verbs NIC /sys/class/infiniband/mlx5_0 (unit 0:0) + dev1.wynton.ucsf.edu:pid550439: PSM3 can't open nic unit: 0 (err=23) + dev1.wynton.ucsf.edu:pid550439.R: Unable to create send CQ of size 5080 on mlx5_0: Cannot allocate memory + dev1.wynton.ucsf.edu:pid550439.R: Unable to initialize verbs NIC /sys/class/infiniband/mlx5_0 (unit 0:0) + dev1.wynton.ucsf.edu:pid550439: PSM3 can't open nic unit: 0 (err=23) + dev1.wynton.ucsf.edu:pid550439.R: Unable to create send CQ of size 5080 on mlx5_0: Cannot allocate memory + dev1.wynton.ucsf.edu:pid550439.R: Unable to initialize verbs NIC /sys/class/infiniband/mlx5_0 (unit 0:0) + -------------------------------------------------------------------------- + ... + dev1.wynton.ucsf.edu:pid550439.R: Unable to create send CQ of size 5080 on mlx5_0: Cannot allocate memory + dev1.wynton.ucsf.edu:pid550439.R: Unable to initialize verbs NIC /sys/class/infiniband/mlx5_0 (unit 0:0) + -------------------------------------------------------------------------- + Open MPI failed an OFI Libfabric library call (fi_endpoint). This is highly + unusual; your job may behave unpredictably (and/or abort) after this. + + Local host: dev1 + Location: mtl_ofi_component.c:513 + Error: Invalid argument (22) + -------------------------------------------------------------------------- ``` -* checking Rd cross-references ... NOTE +* checking for detritus in the temp directory ... NOTE ``` - Package unavailable to check Rd xrefs: ‘Rmpi’ + Found the following files/directories: + ‘ompi.dev1.34002’ ``` -# SimDesign +# targets
-* Version: 2.10.1 -* GitHub: https://github.com/philchalmers/SimDesign -* Source code: https://github.com/cran/SimDesign -* Date/Publication: 2023-02-01 15:30:02 UTC -* Number of recursive dependencies: 104 +* Version: 1.4.0 +* GitHub: https://github.com/ropensci/targets +* Source code: https://github.com/cran/targets +* Date/Publication: 2023-12-11 17:00:02 UTC +* Number of recursive dependencies: 166 -Run `revdep_details(, "SimDesign")` for more info +Run `revdep_details(, "targets")` for more info
@@ -42,13 +63,6 @@ Run `revdep_details(, "SimDesign")` for more info * checking package dependencies ... NOTE ``` - Package suggested but not available for checking: ‘doMPI’ - ``` - -* checking installed package size ... NOTE - ``` - installed size is 6.8Mb - sub-directories of 1Mb or more: - doc 6.2Mb + Package suggested but not available for checking: ‘qs’ ``` diff --git a/vignettes/future.batchtools.md.rsp b/vignettes/future.batchtools.md.rsp index 9ca378c..7551ce5 100644 --- a/vignettes/future.batchtools.md.rsp +++ b/vignettes/future.batchtools.md.rsp @@ -20,24 +20,26 @@ options(mc.cores = 2L) # A Future for batchtools ## Introduction -The [future] package provides a generic API for using futures in R. -A future is a simple yet powerful mechanism to evaluate an R expression -and retrieve its value at some point in time. Futures can be resolved -in many different ways depending on which strategy is used. -There are various types of synchronous and asynchronous futures to -choose from in the [future] package. - -This package, [future.batchtools], provides a type of futures that -utilizes the [batchtools] package. This means that _any_ type of -backend that the batchtools package supports can be used as a future. -More specifically, future.batchtools will allow you or users of your -package to leverage the compute power of high-performance computing -(HPC) clusters via a simple switch in settings - without having to -change any code at all. - -For instance, if batchtools is properly configures, the below two + +The **[future]** package provides a generic API for using futures in +R. A future is a simple yet powerful mechanism to evaluate an R +expression and retrieve its value at some point in time. Futures can +be resolved in many different ways depending on which strategy is +used. There are various types of synchronous and asynchronous futures +to choose from in the **[future]** package. + +This package, **[future.batchtools]**, provides a type of futures that +utilizes the **[batchtools]** package. This means that _any_ type of +backend that the **batchtools** package supports can be used as a +future. More specifically, **future.batchtools** will allow you or +users of your package to leverage the compute power of +high-performance computing (HPC) clusters via a simple switch in +settings - without having to change any code at all. + +For instance, if **batchtools** is properly configures, the below two expressions for futures `x` and `y` will be processed on two different compute nodes: + ```r > library("future.batchtools") > plan(batchtools_torque) @@ -47,8 +49,9 @@ compute nodes: > x + y [1] 5.85 ``` -This is obviously a toy example to illustrate what futures look like -and how to work with them. + +This is just a toy example to illustrate what futures look like and +how to work with them. A more realistic example comes from the field of cancer research where very large data FASTQ files, which hold a large number of short @@ -59,6 +62,7 @@ we can have each file be processed by a separate compute node and each node we can use 24 parallel processes such that each process aligns a separate chromosome. Here is an outline of how this nested parallelism could be implemented using futures. + ```r library("future") library("listenv") @@ -94,22 +98,29 @@ for (ss in seq_along(fqs)) { ## Resolve the "sample" futures and return as a list bams <- as.list(bams) ``` -Note that a user who do not have access to a cluster could use the same script processing samples sequentially and chromosomes in parallel on a single machine using: + +Note that a user who do not have access to a cluster could use the +same script processing samples sequentially and chromosomes in +parallel on a single machine using: + ```r plan(list(sequential, multisession)) ``` + or samples in parallel and chromosomes sequentially using: + ```r plan(list(multisession, sequential)) ``` For an introduction as well as full details on how to use futures, -please consult the package vignettes of the [future] package. +please consult the package vignettes of the **[future]** package. ## Choosing batchtools backend -The future.batchtools package implements a generic future wrapper + +The **future.batchtools** package implements a generic future wrapper for all batchtools backends. Below are the most common types of batchtools backends. @@ -123,18 +134,22 @@ batchtools backends. | `batchtools_openlava` | Futures are evaluated via an [OpenLava] job scheduler | N/A <%-- | `batchtools_docker` | Futures are evaluated via a [Docker Swarm] cluster | N/A --%> | `batchtools_custom` | Futures are evaluated via a custom batchtools configuration R script or via a set of cluster functions | N/A -| `batchtools_interactive` | sequential evaluation in the calling R environment | `plan(transparent)` | `batchtools_multicore` | parallel evaluation by forking the current R process | `plan(multicore)` | `batchtools_local` | sequential evaluation in a separate R process (on current machine) | `plan(cluster, workers = "localhost")` ### Examples -Below is an examples illustrating how to use `batchtools_torque` to configure the batchtools backend. For further details and examples on how to configure batchtools, see the [batchtools configuration] wiki page. +Below is an examples illustrating how to use `batchtools_torque` to +configure the batchtools backend. For further details and examples on +how to configure batchtools, see the [batchtools configuration] wiki +page. + +To configure **batchtools** for job schedulers we need to setup a +`*.tmpl` template file that is used to generate the script used by the +scheduler. This is what a template file for TORQUE / PBS may look +like: -To configure batchtools for job schedulers we need to setup a `*.tmpl` template -file that is used to generate the script used by the scheduler. -This is what a template file for TORQUE / PBS may look like: ```sh #!/bin/bash @@ -161,45 +176,69 @@ This is what a template file for TORQUE / PBS may look like: ## Launch R and evaluated the batchtools R job Rscript -e 'batchtools::doJobCollection("<%%= uri %%>")' ``` -If this template is saved to file `batchtools.torque.tmpl` (without period) -in the working directory or as `.batchtools.torque.tmpl` (with period) the -user's home directory, then it will be automatically located by the -batchtools framework and loaded when doing: + +If this template is saved to file `batchtools.torque.tmpl` (without +period) in the working directory or as `.batchtools.torque.tmpl` (with +period) the user's home directory, then it will be automatically +located by the **batchtools** framework and loaded when doing: + ```r > plan(batchtools_torque) ``` -Resource parameters can be specified via argument `resources` which should be a named list and is passed as is to the template file. For example, to request that each job would get alloted 12 cores (one a single machine) and up to 5 GiB of memory, use: + +Resource parameters can be specified via argument `resources` which +should be a named list and is passed as is to the template file. For +example, to request that each job would get alloted 12 cores (one a +single machine) and up to 5 GiB of memory, use: + ```r > plan(batchtools_torque, resources = list(nodes = "1:ppn=12", vmem = "5gb")) ``` -To specify the `resources` argument at the same time as using nested future strategies, one can use `tweak()` to tweak the default arguments. For instance, +To specify the `resources` argument at the same time as using nested +future strategies, one can use `tweak()` to tweak the default +arguments. For instance, + ```r plan(list( tweak(batchtools_torque, resources = list(nodes = "1:ppn=12", vmem = "5gb")), multisession )) ``` -causes the first level of futures to be submitted via the TORQUE job scheduler requesting 12 cores and 5 GiB of memory per job. The second level of futures will be evaluated using multisession using the 12 cores given to each job by the scheduler. -A similar filename format is used for the other types of job schedulers supported. For instance, for Slurm the template file should be named `./batchtools.slurm.tmpl` or `~/.batchtools.slurm.tmpl` in order for +causes the first level of futures to be submitted via the TORQUE job +scheduler requesting 12 cores and 5 GiB of memory per job. The second +level of futures will be evaluated using multisession using the 12 +cores given to each job by the scheduler. + +A similar filename format is used for the other types of job +schedulers supported. For instance, for Slurm the template file +should be named `./batchtools.slurm.tmpl` or +`~/.batchtools.slurm.tmpl` in order for + ```r > plan(batchtools_slurm) ``` -to locate the file automatically. To specify this template file explicitly, use argument `template`, e.g. + +to locate the file automatically. To specify this template file +explicitly, use argument `template`, e.g. + ```r > plan(batchtools_slurm, template = "/path/to/batchtools.slurm.tmpl") ``` -For further details and examples on how to configure batchtools per se, see the [batchtools configuration] wiki page. +For further details and examples on how to configure **batchtools** per +se, see the [batchtools configuration] wiki page. ## Demos -The [future] package provides a demo using futures for calculating a -set of Mandelbrot planes. The demo does not assume anything about -what type of futures are used. -_The user has full control of how futures are evaluated_. -For instance, to use local batchtools futures, run the demo as: + +The **[future]** package provides a demo using futures for calculating +a set of Mandelbrot planes. The demo does not assume anything about +what type of futures are used. _The user has full control of how +futures are evaluated_. For instance, to use local batchtools +futures, run the demo as: + ```r library("future.batchtools") plan(batchtools_local)