From eecd045cda603955a741787d43e90fff82002109 Mon Sep 17 00:00:00 2001 From: hb Date: Wed, 18 Jul 2018 23:33:28 +0200 Subject: [PATCH 01/16] Bump develop version [ci skip] --- DESCRIPTION | 2 +- NEWS | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index c959376..0f6ed4d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.batchtools -Version: 0.7.1 +Version: 0.7.1-9000 Depends: R (>= 3.2.0), future (>= 1.8.1) diff --git a/NEWS b/NEWS index 1421d3f..e8028df 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,11 @@ Package: future.batchtools ========================== +Version: 0.7.1-9000 [2018-07-18] + + o ... + + Version: 0.7.1 [2018-07-18] NEW FEATURES: From 00a1c73be69d6eca4a76f5dbf570dbc0b1aeaf67 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Sun, 12 Aug 2018 19:05:53 +0200 Subject: [PATCH 02/16] Add a simple example(future_custom) [#29] --- DESCRIPTION | 2 +- NEWS | 8 +++++--- R/batchtools_custom.R | 2 ++ incl/batchtools_custom.R | 10 ++++++++++ 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 incl/batchtools_custom.R diff --git a/DESCRIPTION b/DESCRIPTION index 0f6ed4d..b63323d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -25,5 +25,5 @@ License: LGPL (>= 2.1) LazyLoad: TRUE URL: https://github.com/HenrikBengtsson/future.batchtools BugReports: https://github.com/HenrikBengtsson/future.batchtools/issues -RoxygenNote: 6.0.1 +RoxygenNote: 6.1.0 Roxygen: list(markdown = TRUE) diff --git a/NEWS b/NEWS index e8028df..ad07d1d 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,12 @@ Package: future.batchtools ========================== -Version: 0.7.1-9000 [2018-07-18] +Version: 0.7.1-9000 [2018-08-12] + +DOCUMENTATION: + + o Add a simple example(future_custom). - o ... - Version: 0.7.1 [2018-07-18] diff --git a/R/batchtools_custom.R b/R/batchtools_custom.R index 51bba19..8fa5fae 100644 --- a/R/batchtools_custom.R +++ b/R/batchtools_custom.R @@ -12,6 +12,8 @@ #' #' @return An object of class `BatchtoolsFuture`. #' +#' @example incl/batchtools_custom.R +#' #' @export #' @importFrom utils file_test batchtools_custom <- function(expr, envir = parent.frame(), substitute = TRUE, diff --git a/incl/batchtools_custom.R b/incl/batchtools_custom.R new file mode 100644 index 0000000..d2487e7 --- /dev/null +++ b/incl/batchtools_custom.R @@ -0,0 +1,10 @@ +cf <- batchtools::makeClusterFunctionsInteractive(external = TRUE) +plan(batchtools_custom, cluster.functions = cf) + +## Create explicit future +f <- future({ + cat("PID:", Sys.getpid(), "\n") + 42L +}) +v <- value(f) +print(v) From a587089a4725b0149f1faa1f2a29ab24d03ace1b Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Sun, 12 Aug 2018 19:06:11 +0200 Subject: [PATCH 03/16] Rebuild with roxygen2 6.1.0 --- NEWS | 50 +++++++++++++++++----------------- man/BatchtoolsFuture.Rd | 4 +-- man/batchtools_custom.Rd | 16 +++++++++-- man/batchtools_multicore.Rd | 4 +-- man/batchtools_template.Rd | 20 +++++++------- man/delete.BatchtoolsFuture.Rd | 4 +-- 6 files changed, 55 insertions(+), 43 deletions(-) diff --git a/NEWS b/NEWS index ad07d1d..6c7cbec 100644 --- a/NEWS +++ b/NEWS @@ -5,19 +5,19 @@ Version: 0.7.1-9000 [2018-08-12] DOCUMENTATION: - o Add a simple example(future_custom). + * Add a simple example(future_custom). Version: 0.7.1 [2018-07-18] NEW FEATURES: - o The batchtools_* backends support the handling of the standard output as + * The batchtools_* backends support the handling of the standard output as implemented in future (>= 1.9.0). BUG FIXES: - o A bug was introduced in future.batchtools 0.7.0 that could result in "Error + * A bug was introduced in future.batchtools 0.7.0 that could result in "Error in readLog(id, reg = reg) : Log file for job with id 1 not available" when using one of the batchtools backends. It occurred when the value was queried. It was observered using 'batchtools_torque' but not when using @@ -29,7 +29,7 @@ Version: 0.7.0 [2018-05-03] NEW FEATURES: - o Argument 'workers' of future strategies may now also be a function, which + * Argument 'workers' of future strategies may now also be a function, which is called without argument when the future strategy is set up and used as is. For instance, plan(callr, workers = halfCores) where halfCores <- function() { max(1, round(availableCores() / 2)) } will use @@ -38,7 +38,7 @@ NEW FEATURES: CODE REFACTORING: - o Preparing for futures to gather a richer set of results from batchtools + * Preparing for futures to gather a richer set of results from batchtools backends. @@ -46,83 +46,83 @@ Version: 0.6.0 [2017-09-10] NEW FEATURES: - o If the built-in attempts of batchtools for finding a default template file + * If the built-in attempts of batchtools for finding a default template file fails, then system("templates", package = "future.batchtools") is searched for template files as well. Currently, there exists a `torque.tmpl` file. - o A job's name in the scheduler is now set as the future's label (requires + * A job's name in the scheduler is now set as the future's label (requires batchtools 0.9.4 or newer). If no label is specified, the default job name is controlled by batchtools. - o The period between each poll of the scheduler to check whether a future + * The period between each poll of the scheduler to check whether a future (job) is finished or not now increases geometrically as a function of number of polls. This lowers the load on the scheduler for long running jobs. - o The error message for expired batchtools futures now include the last few + * The error message for expired batchtools futures now include the last few lines of the logged output, which sometimes includes clues on why the future expired. For instance, if a TORQUE/PBS job use more than the allocated amount of memory it might be terminated by the scheduler leaving the message "PBS: job killed: vmem 1234000 exceeded limit 1048576" in the output. - o print() for BatchtoolsFuture returns the object invisibly. + * print() for BatchtoolsFuture returns the object invisibly. BUG FIXES: - o Calling future_lapply() with functions containing globals part of non-default + * Calling future_lapply() with functions containing globals part of non-default packages would when using batchtools futures give an error complaining that the global is missing. This was due to updates in future (>= 1.4.0) that broke this package. - o loggedOutput() for BatchtoolsFuture would always return NULL unless an error + * loggedOutput() for BatchtoolsFuture would always return NULL unless an error had occurred. Version: 0.5.0 [2017-06-02] - o First version submitted to CRAN. + * First version submitted to CRAN. SOFTWARE QUALITY: - o Added more tests; test coverage now at 93%. + * Added more tests; test coverage now at 93%. Version: 0.4.0 [2017-05-16] NEW FEATURES: - o Added batchtools_custom() for specifying batchtools futures using any type + * Added batchtools_custom() for specifying batchtools futures using any type of batchtools cluster functions. - o batchtools_template(pathname = NULL, type = ) now relies on the + * batchtools_template(pathname = NULL, type = ) now relies on the batchtools package for locating the template file. - o nbrOfWorkers() for batchtools futures now defaults to +Inf unless the + * nbrOfWorkers() for batchtools futures now defaults to +Inf unless the evaluator's 'workers' or 'cluster.functions' specify something else. - o Renamed argument 'pathname' to 'template' for batchtools_() functions. + * Renamed argument 'pathname' to 'template' for batchtools_() functions. BUG FIXES: - o Under plan(batchjobs_*), when being created futures would produce an error + * Under plan(batchjobs_*), when being created futures would produce an error on "all(is.finite(workers)) is not TRUE" due to an outdated sanity check. SOFTWARE QUALITY: - o TESTS: Added test of future_lapply() for batchtools backends. + * TESTS: Added test of future_lapply() for batchtools backends. - o TESTS: Added optional tests for batchjobs_* HPC schedulers listed in + * TESTS: Added optional tests for batchjobs_* HPC schedulers listed in environment variable 'R_FUTURE_TESTS_STRATEGIES'. CODE REFACTORING: - o CLEANUP: Package no longer depends on R.utils. + * CLEANUP: Package no longer depends on R.utils. Version: 0.3.0 [2017-03-19] NEW FEATURES: - o The number of jobs one can add to the queues of HPC schedulers is in + * The number of jobs one can add to the queues of HPC schedulers is in principle unlimited, which is why the number of available workers for such batchtools_* backends is reported as +Inf. However, as the number of workers is used by future_lapply() to decide how many futures should be used @@ -135,7 +135,7 @@ Version: 0.2.0 [2017-02-23] GLOBALS: - o batchtools (>= 0.9.2) now supports exporting objects with any type of names + * batchtools (>= 0.9.2) now supports exporting objects with any type of names (previously only possible if they mapped to to strictly valid filenames). This allowed me to avoid lots of internal workaround code encoding and decoding globals. @@ -143,6 +143,6 @@ GLOBALS: Version: 0.1.0 [2017-02-11] - o Package created by porting the code of future.BatchJobs. This version passes + * Package created by porting the code of future.BatchJobs. This version passes 'R CMD check --as-cran' with all OK after a minimal amount of adjustments to the ported code. diff --git a/man/BatchtoolsFuture.Rd b/man/BatchtoolsFuture.Rd index 10b8cf6..d6edc1f 100644 --- a/man/BatchtoolsFuture.Rd +++ b/man/BatchtoolsFuture.Rd @@ -4,8 +4,8 @@ \alias{BatchtoolsFuture} \title{A batchtools future is a future whose value will be resolved via batchtools} \usage{ -BatchtoolsFuture(expr = NULL, envir = parent.frame(), substitute = TRUE, - globals = TRUE, packages = NULL, label = NULL, +BatchtoolsFuture(expr = NULL, envir = parent.frame(), + substitute = TRUE, globals = TRUE, packages = NULL, label = NULL, cluster.functions = NULL, resources = list(), workers = NULL, finalize = getOption("future.finalize", TRUE), ...) } diff --git a/man/batchtools_custom.Rd b/man/batchtools_custom.Rd index af0ae35..35f7eeb 100644 --- a/man/batchtools_custom.Rd +++ b/man/batchtools_custom.Rd @@ -5,8 +5,8 @@ \title{Batchtools futures for custom batchtools configuration} \usage{ batchtools_custom(expr, envir = parent.frame(), substitute = TRUE, - globals = TRUE, label = NULL, cluster.functions, resources = list(), - workers = NULL, ...) + globals = TRUE, label = NULL, cluster.functions, + resources = list(), workers = NULL, ...) } \arguments{ \item{expr}{The R expression to be evaluated} @@ -48,3 +48,15 @@ An object of class \code{BatchtoolsFuture}. \description{ Batchtools futures for custom batchtools configuration } +\examples{ +cf <- batchtools::makeClusterFunctionsInteractive(external = TRUE) +plan(batchtools_custom, cluster.functions = cf) + +## Create explicit future +f <- future({ + cat("PID:", Sys.getpid(), "\\n") + 42L +}) +v <- value(f) +print(v) +} diff --git a/man/batchtools_multicore.Rd b/man/batchtools_multicore.Rd index 49fc956..2b2b126 100644 --- a/man/batchtools_multicore.Rd +++ b/man/batchtools_multicore.Rd @@ -5,8 +5,8 @@ \title{batchtools multicore futures} \usage{ batchtools_multicore(expr, envir = parent.frame(), substitute = TRUE, - globals = TRUE, label = NULL, workers = availableCores(constraints = - "multicore"), ...) + globals = TRUE, label = NULL, workers = availableCores(constraints + = "multicore"), ...) } \arguments{ \item{expr}{The R expression to be evaluated} diff --git a/man/batchtools_template.Rd b/man/batchtools_template.Rd index 5139888..c3c0598 100644 --- a/man/batchtools_template.Rd +++ b/man/batchtools_template.Rd @@ -10,24 +10,24 @@ \title{Batchtools futures for LSF, OpenLava, SGE, Slurm, TORQUE etc.} \usage{ batchtools_lsf(expr, envir = parent.frame(), substitute = TRUE, - globals = TRUE, label = NULL, template = NULL, resources = list(), - workers = Inf, ...) + globals = TRUE, label = NULL, template = NULL, + resources = list(), workers = Inf, ...) batchtools_openlava(expr, envir = parent.frame(), substitute = TRUE, - globals = TRUE, label = NULL, template = NULL, resources = list(), - workers = Inf, ...) + globals = TRUE, label = NULL, template = NULL, + resources = list(), workers = Inf, ...) batchtools_sge(expr, envir = parent.frame(), substitute = TRUE, - globals = TRUE, label = NULL, template = NULL, resources = list(), - workers = Inf, ...) + globals = TRUE, label = NULL, template = NULL, + resources = list(), workers = Inf, ...) batchtools_slurm(expr, envir = parent.frame(), substitute = TRUE, - globals = TRUE, label = NULL, template = NULL, resources = list(), - workers = Inf, ...) + globals = TRUE, label = NULL, template = NULL, + resources = list(), workers = Inf, ...) batchtools_torque(expr, envir = parent.frame(), substitute = TRUE, - globals = TRUE, label = NULL, template = NULL, resources = list(), - workers = Inf, ...) + globals = TRUE, label = NULL, template = NULL, + resources = list(), workers = Inf, ...) } \arguments{ \item{expr}{The R expression to be evaluated} diff --git a/man/delete.BatchtoolsFuture.Rd b/man/delete.BatchtoolsFuture.Rd index 3ce0d1a..622e0da 100644 --- a/man/delete.BatchtoolsFuture.Rd +++ b/man/delete.BatchtoolsFuture.Rd @@ -4,8 +4,8 @@ \alias{delete.BatchtoolsFuture} \title{Removes a batchtools future} \usage{ -\method{delete}{BatchtoolsFuture}(future, onRunning = c("warning", "error", - "skip"), onFailure = c("error", "warning", "ignore"), +\method{delete}{BatchtoolsFuture}(future, onRunning = c("warning", + "error", "skip"), onFailure = c("error", "warning", "ignore"), onMissing = c("ignore", "warning", "error"), times = 10L, delta = getOption("future.wait.interval", 1), alpha = getOption("future.wait.alpha", 1.01), ...) From e3105cd961618ff6a7fd246f7efcae3cf6959858 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Sat, 1 Dec 2018 14:16:29 -0800 Subject: [PATCH 04/16] BUG FIX: resolve() on a lazy batchtools future would stall and never return Found thanks to new future.tests package. --- DESCRIPTION | 2 +- NAMESPACE | 2 ++ NEWS | 6 +++++- R/BatchtoolsFuture-class.R | 5 ++--- tests/future,lazy.R | 7 +++++++ 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index b63323d..3ddbd19 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -25,5 +25,5 @@ License: LGPL (>= 2.1) LazyLoad: TRUE URL: https://github.com/HenrikBengtsson/future.batchtools BugReports: https://github.com/HenrikBengtsson/future.batchtools/issues -RoxygenNote: 6.1.0 +RoxygenNote: 6.1.1 Roxygen: list(markdown = TRUE) diff --git a/NAMESPACE b/NAMESPACE index c72988e..595a365 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -9,6 +9,7 @@ S3method(nbrOfWorkers,batchtools) S3method(print,BatchtoolsFuture) S3method(resolved,BatchtoolsFuture) S3method(result,BatchtoolsFuture) +S3method(run,BatchtoolsFuture) S3method(status,BatchtoolsFuture) export("%resources%") export(BatchtoolsFuture) @@ -55,6 +56,7 @@ importFrom(future,nbrOfWorkers) importFrom(future,plan) importFrom(future,resolved) importFrom(future,result) +importFrom(future,run) importFrom(future,tweak) importFrom(utils,capture.output) importFrom(utils,file_test) diff --git a/NEWS b/NEWS index 6c7cbec..8a9fd8c 100644 --- a/NEWS +++ b/NEWS @@ -1,12 +1,16 @@ Package: future.batchtools ========================== -Version: 0.7.1-9000 [2018-08-12] +Version: 0.7.1-9000 [2018-12-01] DOCUMENTATION: * Add a simple example(future_custom). +BUG FIXES: + + * resolve() on a lazy batchtools future would stall and never return. + Version: 0.7.1 [2018-07-18] diff --git a/R/BatchtoolsFuture-class.R b/R/BatchtoolsFuture-class.R index 19fdac2..ee5723a 100644 --- a/R/BatchtoolsFuture-class.R +++ b/R/BatchtoolsFuture-class.R @@ -312,10 +312,9 @@ result.BatchtoolsFuture <- function(future, cleanup = TRUE, ...) { } -run <- function(...) UseMethod("run") - -#' @importFrom future getExpression +#' @importFrom future run getExpression #' @importFrom batchtools batchExport batchMap saveRegistry setJobNames +#' @export run.BatchtoolsFuture <- function(future, ...) { if (future$state != "created") { label <- future$label diff --git a/tests/future,lazy.R b/tests/future,lazy.R index 282dab3..a68aeb0 100644 --- a/tests/future,lazy.R +++ b/tests/future,lazy.R @@ -15,12 +15,19 @@ for (strategy in strategies) { a <- 42 f <- future(2 * a, lazy = TRUE) a <- 21 + stopifnot(!resolved(f)) + f <- resolve(f) + stopifnot(resolved(f)) v <- value(f) stopifnot(v == 84) a <- 42 v %<-% { 2 * a } %lazy% TRUE a <- 21 + f <- futureOf(v) + stopifnot(!resolved(f)) + f <- resolve(f) + stopifnot(resolved(f)) stopifnot(v == 84) mprintf("- plan('%s') ... DONE\n", strategy) From dd84389083a733ffc1649b5abe2f58f6cb287967 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Sat, 1 Dec 2018 14:18:21 -0800 Subject: [PATCH 05/16] Bump dependency to future (>= 1.10.0) --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3ddbd19..1c186d7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -2,7 +2,7 @@ Package: future.batchtools Version: 0.7.1-9000 Depends: R (>= 3.2.0), - future (>= 1.8.1) + future (>= 1.10.0) Imports: batchtools (>= 0.9.8) Suggests: From bf8d5d826b204543f5887c55393730830a7b6e28 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 24 Dec 2018 00:39:30 -0800 Subject: [PATCH 06/16] TESTS: Make assertion test agile to future changes where conditions incl. messages may be captured --- tests/BatchtoolsFutureError.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BatchtoolsFutureError.R b/tests/BatchtoolsFutureError.R index 8a50ea8..67510ce 100644 --- a/tests/BatchtoolsFutureError.R +++ b/tests/BatchtoolsFutureError.R @@ -57,7 +57,7 @@ message("*** BatchtoolsFuture - expired ...") plan(batchtools_local) msg <- "Abruptly terminating the future!" f <- future({ - message(msg) + cat(file = stderr(), msg) quit(save = "no") }) res <- tryCatch({ From 7b53be350e2c0cf2eb2b7d0741cfb21f2461112f Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Mon, 24 Dec 2018 00:44:23 -0800 Subject: [PATCH 07/16] TESTS: Make assertion test agile to future changes where conditions incl. messages may be captured --- tests/stdout.R | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/tests/stdout.R b/tests/stdout.R index 351a931..e590a5d 100644 --- a/tests/stdout.R +++ b/tests/stdout.R @@ -1,9 +1,5 @@ source("incl/start.R") -## Only validate 'stdout' output if future (>= 1.9.0) -test_stdout <- ("stdout" %in% names(formals(future::Future))) -if (!test_stdout) `%stdout%` <- function(x, ...) x - message("*** Standard output ...") truth_rows <- utils::capture.output({ @@ -34,16 +30,13 @@ for (stdout in c(TRUE, FALSE, NA)) { str(r) stopifnot(value(f) == 42L) - - if (test_stdout) { - if (is.na(stdout)) { - stopifnot(!"stdout" %in% names(r)) - } else if (stdout) { - print(r) - stopifnot(identical(r$stdout, truth)) - } else { - stopifnot(is.null(r$stdout)) - } + if (is.na(stdout)) { + stopifnot(!"stdout" %in% names(r)) + } else if (stdout) { + print(r) + stopifnot(identical(r$stdout, truth)) + } else { + stopifnot(is.null(r$stdout)) } v %<-% { @@ -57,13 +50,11 @@ for (stdout in c(TRUE, FALSE, NA)) { out <- utils::capture.output(y <- v) stopifnot(y == 42L) - if (test_stdout) { - if (is.na(stdout) || !stdout) { - stopifnot(out == "") - } else { - print(out) - stopifnot(identical(out, truth_rows)) - } + if (is.na(stdout) || !stdout) { + stopifnot(out == "") + } else { + print(out) + stopifnot(identical(out, truth_rows)) } } ## for (stdout ...) From 60c14b2bf701248cf977a8149250cc7087336f2f Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 28 Dec 2018 18:42:54 -0800 Subject: [PATCH 08/16] Add internal utility function result_has_errors() to encapsulate future (>= 1.11.0) updates --- R/BatchtoolsFuture-class.R | 9 +++------ R/utils.R | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/R/BatchtoolsFuture-class.R b/R/BatchtoolsFuture-class.R index ee5723a..51a590c 100644 --- a/R/BatchtoolsFuture-class.R +++ b/R/BatchtoolsFuture-class.R @@ -198,8 +198,7 @@ status.BatchtoolsFuture <- function(future, ...) { result <- future$result if (inherits(result, "FutureResult")) { - condition <- result$condition - if (inherits(condition, "error")) status <- c("error", status) + if (result_has_errors(result)) status <- unique(c("error", status)) } status @@ -503,9 +502,7 @@ await.BatchtoolsFuture <- function(future, cleanup = TRUE, result[["batchtools_log"]] <- try({ getLog(id = jobid, reg = reg) }, silent = TRUE) - if (inherits(result$condition, "error")) { - cleanup <- FALSE - } + if (result_has_errors(result)) cleanup <- FALSE } } else if ("error" %in% stat) { cleanup <- FALSE @@ -636,7 +633,7 @@ delete.BatchtoolsFuture <- function(future, status <- status(future) res <- future$result if (inherits(res, "FutureResult")) { - if (inherits(res$condition, "error")) status <- "error" + if (result_has_errors(res)) status <- unique(c("error", status)) } mdebug("delete(): status() = %s", paste(sQuote(status), collapse = ", ")) diff --git a/R/utils.R b/R/utils.R index d0959e5..e303471 100644 --- a/R/utils.R +++ b/R/utils.R @@ -170,3 +170,19 @@ tempvar <- function(prefix = "var", value = NA, envir = parent.frame()) { # Failed to find a unique temporary variable name stop(sprintf("Failed to generate a unique non-existing temporary variable with prefix '%s'", prefix)) #nolint } + + + +result_has_errors <- function(result) { + stop_if_not(inherits(result, "FutureResult")) + conditions <- result$conditions + ## BACKWARD COMPATIBILITY: future (< 1.11.0) + if (!is.list(conditions) && !is.null(result$condition)) { + conditions <- list(result$condition) + } + ## Any error:s? + for (condition in conditions) { + if (inherits(condition, "error")) return(TRUE) + } + FALSE +} From 8721ce0ba581a53b9a71b840cbceef1494ea6142 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Fri, 28 Dec 2018 20:07:55 -0800 Subject: [PATCH 09/16] Tweak result_has_errors() --- R/utils.R | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/R/utils.R b/R/utils.R index e303471..fa2681c 100644 --- a/R/utils.R +++ b/R/utils.R @@ -175,14 +175,13 @@ tempvar <- function(prefix = "var", value = NA, envir = parent.frame()) { result_has_errors <- function(result) { stop_if_not(inherits(result, "FutureResult")) - conditions <- result$conditions + ## BACKWARD COMPATIBILITY: future (< 1.11.0) - if (!is.list(conditions) && !is.null(result$condition)) { - conditions <- list(result$condition) + if (inherits(result$condition, "error")) return(TRUE) + + for (c in result$conditions) { + if (inherits(c$condition, "error")) return(TRUE) } - ## Any error:s? - for (condition in conditions) { - if (inherits(condition, "error")) return(TRUE) - } + FALSE } From d0c408f7eba40149b7a2868494fcf7ddec8c6b13 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Sun, 30 Dec 2018 20:14:41 -0800 Subject: [PATCH 10/16] TESTS: Adjust assertions for future (>= 1.11.0) --- tests/stdout.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/stdout.R b/tests/stdout.R index e590a5d..294a045 100644 --- a/tests/stdout.R +++ b/tests/stdout.R @@ -31,7 +31,7 @@ for (stdout in c(TRUE, FALSE, NA)) { stopifnot(value(f) == 42L) if (is.na(stdout)) { - stopifnot(!"stdout" %in% names(r)) + stopifnot(is.null(r$stdout) || r$stdout == "") } else if (stdout) { print(r) stopifnot(identical(r$stdout, truth)) From 303b1910e7ebbd9505f8ed56eb73b13d9e347b81 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 3 Jan 2019 15:33:22 -0800 Subject: [PATCH 11/16] REVDEP: Modernized --- revdep/run.R | 121 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 109 insertions(+), 12 deletions(-) diff --git a/revdep/run.R b/revdep/run.R index c910f81..7c3bb01 100644 --- a/revdep/run.R +++ b/revdep/run.R @@ -1,9 +1,13 @@ library("revdepcheck") options(warn = 1L) -availableCores <- function() { - getenv <- function(name) as.integer(Sys.getenv(name, NA_character_)) - getopt <- function(name) as.integer(getOption(name, NA_integer_)) +available_cores <- function() { + getenv <- function(name) { + as.integer(Sys.getenv(name, NA_character_)) + } + getopt <- function(name) { + as.integer(getOption(name, NA_integer_)) + } if (is.finite(n <- getopt("mc.cores") + 1L)) return(n) if (is.finite(n <- getopt("Ncpus") + 1L)) return(n) if (is.finite(n <- getenv("PBS_NUM_PPN"))) return(n) @@ -12,16 +16,109 @@ availableCores <- function() { 1L } -if (file_test("-f", p <- Sys.getenv("R_CHECK_ENVIRON", "~/.R/check.Renviron"))) { - cat(sprintf("R CMD check will use env vars from %s\n", sQuote(p))) - cat(sprintf("To disable, set 'R_CHECK_ENVIRON=false' (a fake pathname)\n")) +precheck <- function() { + ## WORKAROUND: Remove checked pkgs that use file links, which otherwise + ## produce warnings which are promoted to errors by revdepcheck. + unlink("revdep/checks/aroma.affymetrix", recursive = TRUE) } -envs <- grep("^_R_CHECK_", names(Sys.getenv()), value = TRUE) -if (length(envs) > 0L) { - cat(sprintf("Detected _R_CHECK_* env vars that will affect R CMD check: %s\n", - paste(sQuote(envs), collapse = ", "))) +check <- function() { + if (file_test("-f", p <- Sys.getenv("R_CHECK_ENVIRON", "~/.R/check.Renviron"))) { + cat(sprintf("R CMD check will use env vars from %s\n", sQuote(p))) + cat(sprintf("To disable, set 'R_CHECK_ENVIRON=false' (a fake pathname)\n")) + } + + envs <- grep("^_R_CHECK_", names(Sys.getenv()), value = TRUE) + if (length(envs) > 0L) { + cat(sprintf("Detected _R_CHECK_* env vars that will affect R CMD check: %s\n", + paste(sQuote(envs), collapse = ", "))) + } + + precheck() + revdep_check(bioc = TRUE, num_workers = available_cores(), + timeout = as.difftime(20, units = "mins"), quiet = FALSE) +} + +todo <- function() { + pkgs <- tryCatch(revdep_todo(), error = function(ex) NA) + if (identical(pkgs, NA)) { + cat("Revdepcheck has not been initiated\n") + } else if (length(pkgs) == 0) { + cat("There are no packages on the revdepcheck todo list\n") + } else { + cat(sprintf("%d. %s\n", seq_along(pkgs), pkgs)) + } } -revdep_check(bioc = TRUE, num_workers = availableCores(), - timeout = as.difftime(20, units = "mins"), quiet = FALSE) +parse_pkgs <- function(pkgs) { + pkgs <- unlist(strsplit(pkgs, split = ",", fixed = TRUE)) + pkgs <- gsub("[ \t'\"‘’]", "", pkgs) + sort(unique(pkgs)) +} + +revdep_init <- function() { + if (!revdepcheck:::db_exists(".")) revdepcheck:::db_setup(".") +} + +revdep_todo_reset <- function() { + revdep_init() + db <- revdepcheck:::db(".") + df <- data.frame(package = character(0L), stringsAsFactors = FALSE) + DBI::dbWriteTable(db, "todo", df, overwrite = TRUE, append = FALSE) +} + +revdep_children <- local({ + cache <- list() + function(pkg = NULL) { + if (is.null(pkg)) pkg <- desc::desc(file = "DESCRIPTION")$get("Package") + pkgs <- cache[[pkg]] + if (is.null(pkgs)) { + pkgs <- revdepcheck:::cran_revdeps(pkg) + pkgs <- setdiff(pkgs, pkg) ## WORKAROUND + cache[[pkg]] <- pkgs + } + pkgs + } +}) + +args <- base::commandArgs() +if ("--reset" %in% args) { + revdep_reset() +} else if ("--todo-reset" %in% args) { + revdep_todo_reset() + todo() +} else if ("--todo" %in% args) { + todo() +} else if ("--add" %in% args) { + pos <- which("--add" == args) + pkgs <- parse_pkgs(args[seq(from = pos + 1L, to = length(args))]) + revdep_add(packages = pkgs) + todo() +} else if ("--add-broken" %in% args) { + revdep_add_broken() + todo() +} else if ("--add-all" %in% args) { + revdep_init() + pkgs <- revdep_children() + for (pkg in pkgs) { + pkgs <- c(pkgs, revdepcheck:::cran_revdeps(pkg)) + } + pkgs <- unique(pkgs) + revdep_add(packages = pkgs) + todo() +} else if ("--add-grandchildren" %in% args) { + revdep_init() + pkgs <- NULL + for (pkg in revdep_children()) { + pkgs <- c(pkgs, revdepcheck:::cran_revdeps(pkg)) + } + pkgs <- unique(pkgs) + revdep_add(packages = pkgs) + todo() +} else if ("--install" %in% args) { + pos <- which("--install" == args) + pkgs <- parse_pkgs(args[seq(from = pos + 1L, to = length(args))]) + crancache::install_packages(pkgs) +} else { + check() +} From fb32cadcf13c8a14500aceb6b1702932bfa57d29 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 3 Jan 2019 15:33:55 -0800 Subject: [PATCH 12/16] Mention tweaks for upcoming future release --- NEWS | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 8a9fd8c..a757c6a 100644 --- a/NEWS +++ b/NEWS @@ -1,12 +1,17 @@ Package: future.batchtools ========================== -Version: 0.7.1-9000 [2018-12-01] +Version: 0.7.1-9000 [2018-12-03] DOCUMENTATION: * Add a simple example(future_custom). +FIXES: + + o Made internal code agile to upcoming changes in the future package on + how a captured error is represented. + BUG FIXES: * resolve() on a lazy batchtools future would stall and never return. From f0da2132348826c489abba9ac5d0f57f5ee12502 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 3 Jan 2019 15:34:42 -0800 Subject: [PATCH 13/16] REVDEP: Ran revdep checks on 4 CRAN packages [ci skip] --- revdep/README.md | 68 +++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/revdep/README.md b/revdep/README.md index d691697..10f557b 100644 --- a/revdep/README.md +++ b/revdep/README.md @@ -3,51 +3,53 @@ |field |value | |:--------|:----------------------------| |version |R version 3.5.1 (2018-07-02) | -|os |Ubuntu 16.04.4 LTS | +|os |Ubuntu 18.04.1 LTS | |system |x86_64, linux-gnu | |ui |X11 | |language |en | |collate |en_US.UTF-8 | -|tz |Europe/Copenhagen | -|date |2018-07-18 | +|ctype |en_US.UTF-8 | +|tz |America/Los_Angeles | +|date |2019-01-03 | # Dependencies -|package |old |new |Δ | -|:-----------------|:-------|:-------|:--| -|future.batchtools |0.7.0 |0.7.1 |* | -|assertthat |0.2.0 |0.2.0 | | -|backports |1.1.2 |1.1.2 | | -|base64url |1.4 |1.4 | | -|batchtools |0.9.10 |0.9.10 | | -|brew |1.0-6 |1.0-6 | | -|checkmate |1.8.5 |1.8.5 | | -|crayon |1.3.4 |1.3.4 | | -|data.table |1.11.4 |1.11.4 | | -|digest |0.6.15 |0.6.15 | | -|fs |1.2.3 |1.2.3 | | -|future |1.8.1 |1.8.1 | | -|globals |0.12.1 |0.12.1 | | -|hms |0.4.2 |0.4.2 | | -|listenv |0.7.0 |0.7.0 | | -|magrittr |1.5 |1.5 | | -|pkgconfig |2.0.1 |2.0.1 | | -|prettyunits |1.0.2 |1.0.2 | | -|progress |1.2.0 |1.2.0 | | -|R6 |2.2.2 |2.2.2 | | -|rappdirs |0.3.1 |0.3.1 | | -|Rcpp |0.12.17 |0.12.17 | | -|rlang |0.2.1 |0.2.1 | | -|stringi |1.2.3 |1.2.3 | | -|withr |2.1.2 |2.1.2 | | +|package |old |new |Δ | +|:-----------------|:-------|:----------|:--| +|future.batchtools |0.7.1 |0.7.1-9000 |* | +|assertthat |0.2.0 |0.2.0 | | +|backports |1.1.3 |1.1.3 | | +|base64url |1.4 |1.4 | | +|batchtools |0.9.11 |0.9.11 | | +|brew |1.0-6 |1.0-6 | | +|checkmate |1.8.5 |1.8.5 | | +|crayon |1.3.4 |1.3.4 | | +|data.table |1.11.8 |1.11.8 | | +|digest |0.6.18 |0.6.18 | | +|fs |1.2.6 |1.2.6 | | +|future |1.10.0 |1.10.0 | | +|globals |0.12.4 |0.12.4 | | +|hms |0.4.2 |0.4.2 | | +|listenv |0.7.0 |0.7.0 | | +|magrittr |1.5 |1.5 | | +|pkgconfig |2.0.2 |2.0.2 | | +|prettyunits |1.0.2 |1.0.2 | | +|progress |1.2.0 |1.2.0 | | +|R6 |2.3.0 |2.3.0 | | +|rappdirs |0.3.1 |0.3.1 | | +|Rcpp |1.0.0 |1.0.0 | | +|rlang |0.3.0.1 |0.3.0.1 | | +|stringi |1.2.4 |1.2.4 | | +|withr |2.1.2 |2.1.2 | | # Revdeps -## All (3) +## All (4) |package |version |error |warning |note | |:----------|:-------|:-----|:-------|:----| -|batchtools |0.9.10 | | | | -|drtmle |1.0.3 | | | | +|batchtools |0.9.11 | | | | +|drtmle |1.0.4 | | | | +|FAMoS |0.1.0 | | | | |origami |1.0.0 | | | | From 2ca4e5063c4982dcf8d043e9b8f930716a3a0673 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 3 Jan 2019 16:32:19 -0800 Subject: [PATCH 14/16] cleanup [ci skip] --- tests/batchtools_multicore.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/batchtools_multicore.R b/tests/batchtools_multicore.R index f4e0fd7..c9642bd 100644 --- a/tests/batchtools_multicore.R +++ b/tests/batchtools_multicore.R @@ -101,7 +101,7 @@ for (cores in 1:min(2L, availableCores("multicore"))) { ## CRAN processing times: ## On Windows 32-bit, don't run these tests if (fullTest || !isWin32) { - mprintf("*** batchtools_multicore() and errors\n", globals) + mprintf("*** batchtools_multicore() and errors\n") f <- batchtools_multicore({ stop("Whoops!") 1 From 8f97118f98cdd363f36fa48e88854bba29ea5fcb Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 3 Jan 2019 16:54:41 -0800 Subject: [PATCH 15/16] Spell check package [ci skip] --- README.md | 2 +- inst/WORDLIST | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 inst/WORDLIST diff --git a/README.md b/README.md index b7570ab..9bee446 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ Contributing to this package is easy. Just send a [pull request](https://help.g ## Software status -| Resource: | CRAN | Travis CI | Appveyor | +| Resource: | CRAN | Travis CI | AppVeyor | | ------------- | ------------------- | --------------- | ---------------- | | _Platforms:_ | _Multiple_ | _Linux & macOS_ | _Windows_ | | R CMD check | CRAN version | Build status | Build status | diff --git a/inst/WORDLIST b/inst/WORDLIST new file mode 100644 index 0000000..ff65636 --- /dev/null +++ b/inst/WORDLIST @@ -0,0 +1,24 @@ +alloted +AppVeyor +batchtools +Batchtools +BatchtoolsFuture +CMD +FASTQ +FutureError +GiB +globals +Globals +HPC +lapply +LSF +macOS +multiprocess +OpenLava +pre +Pre +se +SGE +Slurm +th +uniprocess From 2a2d13b00257833b9972bd059072d00942cd4220 Mon Sep 17 00:00:00 2001 From: Henrik Bengtsson Date: Thu, 3 Jan 2019 18:12:51 -0800 Subject: [PATCH 16/16] future.batchtools 0.7.2 --- DESCRIPTION | 2 +- NEWS | 10 ++++--- cran-comments.md | 70 +++++++++++++++++++++--------------------------- 3 files changed, 39 insertions(+), 43 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 1c186d7..ec7452d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: future.batchtools -Version: 0.7.1-9000 +Version: 0.7.2 Depends: R (>= 3.2.0), future (>= 1.10.0) diff --git a/NEWS b/NEWS index a757c6a..a678223 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ Package: future.batchtools ========================== -Version: 0.7.1-9000 [2018-12-03] +Version: 0.7.2 [2018-12-03] DOCUMENTATION: @@ -9,9 +9,13 @@ DOCUMENTATION: FIXES: - o Made internal code agile to upcoming changes in the future package on + * Made internal code agile to upcoming changes in the future package on how a captured error is represented. - + +SOFTWARE QUALITY: + + * FYI: Every release is tested against one Torque/PBS and one SGE scheduler. + BUG FIXES: * resolve() on a lazy batchtools future would stall and never return. diff --git a/cran-comments.md b/cran-comments.md index 21430f1..5be1be2 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,64 +1,56 @@ -# CRAN submission future.batchtools 0.7.1 +# CRAN submission future.batchtools 0.7.2 -on 2018-07-18 +on 2019-01-03 -## Submission 1 +Thanks in advance. -Thanks in advance - -## Submission 2 - -Resubmission of future.batchtools 0.7.1 where the overall R CMD check time has been decreased significantly. +## Notes not sent to CRAN ### R CMD check --as-cran validation The package has been verified using `R CMD check --as-cran` on: -* Platform x86_64-apple-darwin15.6.0 (64-bit) [r-hub; single-core]: - - R version 3.5.0 (2018-04-23) +* Platform x86_64-apple-darwin15.6.0 (64-bit) [Travis CI]: + - R version 3.4.4 (2017-01-27) + - R version 3.5.2 (2018-12-20) * Platform x86_64-unknown-linux-gnu (64-bit) [Travis CI]: - R version 3.4.4 (2017-01-27) - - R version 3.5.0 (2017-01-27) - - R Under development (unstable) (2018-06-20 r74923) + - R version 3.5.1 (2018-07-02) + - R Under development (unstable) (2019-01-02 r75944) * Platform x86_64-pc-linux-gnu (64-bit) [r-hub]: - R version 3.4.4 (2018-03-15) - - R Under development (unstable) (2018-07-16 r74967) - + - R Under development (unstable) (2018-12-22 r75884) + * Platform x86_64-pc-linux-gnu (64-bit): - - R version 3.2.0 (2015-04-16) - - R version 3.3.0 (2016-05-03) - - R version 3.4.0 (2017-04-21) - - R version 3.5.0 (2018-04-23) + - R version 3.2.0 (2015-04-16) w/ Torque/PBS scheduler + - R version 3.3.0 (2016-05-03) w/ Torque/PBS scheduler + - R version 3.5.0 (2018-04-23) w/ Torque/PBS scheduler - R version 3.5.1 (2018-07-02) + - R version 3.5.1 (2018-07-02) w/ SGE scheduler + - R version 3.5.2 (2018-12-20) w/ Torque/PBS scheduler -* Platform i386-w64-mingw32 (32-bit) (64-bit) [Appveyor CI]: - - R Under development (unstable) (2018-07-16 r74967) +* Platform i686-pc-linux-gnu (32-bit): + - R version 3.4.4 (2018-03-15) -* Platform x86_64-w64-mingw32/x64 (64-bit) [Appveyor CI]: - - R version 3.5.1 (2018-07-02) - - R Under development (unstable) (2018-07-16 r74967) +* Platform i386-w64-mingw32 (32-bit) [Appveyor CI]: + - R Under development (unstable) (2018-10-12 r75434) * Platform x86_64-w64-mingw32 (64-bit) [r-hub]: - - R Under development (unstable) (2018-07-16 r74967) + - R Under development (unstable) (2018-12-26 r75909) -* Platform x86_64-w64-mingw32/x64 (64-bit) [win-builder]: - - R version 3.5.1 (2018-07-02) - - R Under development (unstable) (2018-07-16 r74967) - - -The following setups were skipped due to non-availability: - -* Platform x86_64-apple-darwin13.4.0 (64-bit) [Travis CI]: - - R version 3.4.4 (2017-01-27) +* Platform x86_64-w64-mingw32/x64 (64-bit) [Appveyor CI]: + - R version 3.5.2 (2018-12-20) + - R Under development (unstable) (2018-12-29 r75924) -* Platform x86_64-apple-darwin15.6.0 (64-bit) [Travis CI]: - - R version 3.5.0 (2018-04-23) +* Platform x86_64-w64-mingw32/x64 (64-bit) [win-builder]: + - R version 3.5.2 (2018-12-20) + - R Under development (unstable) (2019-01-01 r75943) -* Platform i686-pc-linux-gnu (32-bit): - - R version 3.4.4 (2018-03-15) +Failed to test: -* Platform i686-pc-linux-gnu (32-bit): - - R version 3.4.4 (2018-03-15) +* Platform i386-pc-solaris2.10 (32-bit): + - R version 3.5.0 Patched (2018-04-30 r74674) + REASON: 'stringi' package failed to install