make noisy repls flexible

mlr-org · Nov 14, 2018 · db2239f · db2239f
1 parent 6e35606
commit db2239f
Show file tree

Hide file tree

Showing 13 changed files with 97 additions and 97 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -61,5 +61,5 @@ LazyData: yes
 Encoding: UTF-8
 ByteCompile: yes
 Version: 1.1.3
-RoxygenNote: 6.0.1
+RoxygenNote: 6.1.1
 VignetteBuilder: knitr
diff --git a/R/evalTargetFun.R b/R/evalTargetFun.R
@@ -25,60 +25,65 @@ evalTargetFun.OptState = function(opt.state, xs, extras) {
   # short names and so on
   nevals = length(xs)
   ny = control$n.objectives
+  imputeY = control$impute.y.fun
 
   # trafo X points
   xs.trafo = lapply(xs, trafoValue, par = par.set)
 
-  # handle noisy instances
-  if (isTRUE(control$noisy.instances > 1L)) {
-    nevals = nevals * control$noisy.instances
-    xs = rep(xs, each = control$noisy.instances)
-    extras = rep(extras, each = control$noisy.instances)
-    if (!control$noisy.self.replicating) {
-      xs.trafo = rep(xs.trafo, each = control$noisy.instances)
-      if (!is.na(control$noisy.instance.param)) {
-        inst.param = lapply(seq_len(control$noisy.instances), function(x) setNames(list(x), control$noisy.instance.param))
-        xs.trafo = Map(c, xs.trafo, inst.param)
-      }
-    }
-  }
-
-
-  num.format = control$output.num.format
-  num.format.string = paste("%s = ", num.format, sep = "")
-  dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer")
-  imputeY = control$impute.y.fun
-
-
   # function to measure of fun call
-    wrapFun = function(x) {
-      st = proc.time()
-      y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem)))
-      user.extras = list()
+  wrapFun = function(x) {
+    st = proc.time()
+    y = do.call(getOptProblemFun(opt.problem), insert(list(x = x), getOptProblemMoreArgs(opt.problem)))
+    user.extras = list()
       # here we extract additional stuff which the user wants to log in the opt path
-      if (hasAttributes(y, "extras")) {
-        user.extras = attr(y, "extras")
-        y = setAttribute(y, "extras", NULL)
-      }
-      if (!is.null(control$noisy.instance.param) && !is.na(control$noisy.instance.param) && !control$noisy.self.replicating) {
-        user.extras = c(user.extras, x[control$noisy.instance.param])
-      }
-      st = proc.time() - st
-      list(y = y, time = st[3], user.extras = user.extras)
+    if (hasAttributes(y, "extras")) {
+      user.extras = attr(y, "extras")
+      y = setAttribute(y, "extras", NULL)
+    }
+    if (!is.null(control$noisy.instance.param) && !is.na(control$noisy.instance.param) && !control$noisy.self.replicating) {
+      user.extras = c(user.extras, x[control$noisy.instance.param])
     }
+    st = proc.time() - st
+    list(y = y, time = st[3], user.extras = user.extras)
+  }
 
   # do we have a valid y object?
   isYValid = function(y) {
-    !is.error(y) && testNumeric(y, len = ny, any.missing = FALSE, finite = TRUE)
+    if (!isTRUE(control$noisy.self.replicating)) {
+      len = NULL
+    } else {
+      len = ny
+    }
+    !is.error(y) && testNumeric(y, len = len, any.missing = FALSE, finite = TRUE)
   }
 
   # return error objects if we impute
-  res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval",
-    impute.error = if (is.null(imputeY)) NULL else identity)
+  res = parallelMap(wrapFun, xs.trafo, level = "mlrMBO.feval", impute.error = if (is.null(imputeY)) NULL else identity)
+
+  # handle noisy instances
+  if (isTRUE(control$noisy.self.replicating)) {
+    nevals.each = lengths(extractSubList(res, "y", simplify = FALSE))
+    nevals = sum(nevals.each)
+
+    # replications for opt path stuff
+    repVec = function(x, fun = replicate) {
+      unlist(Map(fun, nevals.each, x, simplify = FALSE), recursive = FALSE)
+    }
+    xs = repVec(xs)
+    xs.trafo = repVec(xs.trafo)
+
+    #set extras to NA that are only important for the first point
+    setNAfun = function(n, xs, ...) {
+      res = replicate(n = n, expr = xs, ...)
+      res[-1] = lapply(res[-1], function(x) {
+        x[c("train.time", "error.model", "propose.time")] = NA
+        x
+      })
+      res
+    }
+    extras = repVec(extras, setNAfun)
 
-  # handle noisy instances of self.replicating functions
-  if (isTRUE(control$noisy.instances > 1L) && control$noisy.self.replicating) {
-    xs.trafo = rep(xs.trafo, each = control$noisy.instances)
+    # handle result list
     res = lapply(res, function(r) {
       if (is.error(r)) {
         rep(list(r), control$noisy.instances)
@@ -91,6 +96,9 @@ evalTargetFun.OptState = function(opt.state, xs, extras) {
     res = unlist(res, recursive = FALSE)
   }
 
+  num.format.string = paste("%s = ", control$output.num.format, sep = "")
+  dobs = ensureVector(asInteger(getOptStateLoop(opt.state)), n = nevals, cl = "integer")
+
   # loop evals and to some post-processing
   for (i in seq_len(nevals)) {
     r = res[[i]]; x = xs[[i]]; x.trafo = xs.trafo[[i]]; dob = dobs[i]

diff --git a/R/setMBOControlNoisy.R b/R/setMBOControlNoisy.R
@@ -2,29 +2,17 @@
 #' @description
 #' Extends an MBO control object with options for multipoint proposal.
 #' @template arg_control
-#' @param instances [\code{integer(1)}]\cr
-#'   How many instances of one parameter will be calculated?
-#' @param instance.param [\code{character(1)}]\cr
-#'   What is the name of the function param that defines the instance?
 #' @param self.replicating [\code{logical(1)}]\cr
 #'   TRUE if the function returns a vector of noisy results for one input. Then \code{instances} specifies the length of the result we expect.
 #' @return [\code{\link{MBOControl}}].
 #' @family MBOControl
 #' @export
-setMBOControlNoisy = function(control,
-  instances = NULL,
-  instance.param = NULL,
-  self.replicating = NULL) {
+setMBOControlNoisy = function(control, self.replicating) {
 
   assertClass(control, "MBOControl")
 
-  control$noisy.instances = assertInt(instances, lower = 1L, null.ok = TRUE, na.ok = FALSE) %??% control$noisy.instances
-  control$noisy.self.replicating = assertFlag(self.replicating, null.ok = TRUE, na.ok = FALSE) %??% control$noisy.self.replicating %??% FALSE
-  control$noisy.instance.param = assertString(instance.param, null.ok = TRUE, na.ok = TRUE) %??% control$noisy.instance.param %??% ifelse(control$noisy.self.replicating, "noisy.repl", NA_character_)
-
-  if (control$noisy.self.replicating && control$noisy.instance.param != "noisy.repl") {
-    stop("You can not change the instance.param for self replicating functions.")
-  }
+  control$noisy.self.replicating = assertFlag(self.replicating %??% control$noisy.self.replicating %??% TRUE, na.ok = FALSE)
+  control$noisy.instance.param = "noisy.repl"
 
   return(control)
 }
diff --git a/man/makeMBOControl.Rd b/man/makeMBOControl.Rd
diff --git a/man/plotMBOResult.Rd b/man/plotMBOResult.Rd
diff --git a/man/renderExampleRunPlot.Rd b/man/renderExampleRunPlot.Rd
diff --git a/man/setMBOControlInfill.Rd b/man/setMBOControlInfill.Rd
diff --git a/man/setMBOControlMultiObj.Rd b/man/setMBOControlMultiObj.Rd
diff --git a/man/setMBOControlMultiPoint.Rd b/man/setMBOControlMultiPoint.Rd
diff --git a/man/setMBOControlNoisy.Rd b/man/setMBOControlNoisy.Rd
diff --git a/man/setMBOControlTermination.Rd b/man/setMBOControlTermination.Rd
diff --git a/man/updateSMBO.Rd b/man/updateSMBO.Rd
diff --git a/tests/testthat/test_mbo_noisy.R b/tests/testthat/test_mbo_noisy.R
@@ -16,37 +16,37 @@ test_that("mbo works with multiple instances of noisy problems", {
   expect_true(all(table(opdf$x) == 5))
 })
 
-test_that("mbo works with multiple fixed instances of noisy problems", {
+test_that("mbo works with self replicating instances of noisy problems", {
   ps = makeNumericParamSet("x", 1, -7, 7)
   fun = smoof::makeSingleObjectiveFunction(
-    fn = function(x) x$x^2 + rnorm(1, (x$i - 2)/100, 0.05),
+    fn = function(x) x^2 + rnorm(5, 0.01),
     par.set = ps,
-    noisy = TRUE,
-    has.simple.signature = FALSE
+    noisy = TRUE
   )
   ctrl = makeMBOControl()
   ctrl = setMBOControlTermination(ctrl, iters = 5L)
   ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L)
-  ctrl = setMBOControlNoisy(ctrl, instances = 5L, instance.param = "i")
+  ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE)
   or = mbo(fun, control = ctrl)
   opdf = as.data.frame(or$opt.path)
-  expect_true(all(opdf$i %in% 1:5))
+  expect_true(all(opdf$noisy.repl %in% 1:5))
   expect_true(all(table(opdf$x) == 5))
-})
 
-test_that("mbo works with self replicating instances of noisy problems", {
-  ps = makeNumericParamSet("x", 1, -7, 7)
+  # now the function has varying n results
+  i = 0
   fun = smoof::makeSingleObjectiveFunction(
-    fn = function(x) x^2 + rnorm(5, 0.01),
+    fn = function(x) {
+      i <<- i + 1
+      x^2 + rnorm(i, 0.01)
+    },
     par.set = ps,
     noisy = TRUE
   )
-  ctrl = makeMBOControl()
-  ctrl = setMBOControlTermination(ctrl, iters = 5L)
+  ctrl = setMBOControlTermination(ctrl, iters = 2L)
   ctrl = setMBOControlInfill(ctrl, crit = crit.aei, opt.focussearch.points = 100L)
-  ctrl = setMBOControlNoisy(ctrl, instances = 5L, self.replicating = TRUE)
+  ctrl = setMBOControlNoisy(ctrl, self.replicating = TRUE)
   or = mbo(fun, control = ctrl)
   opdf = as.data.frame(or$opt.path)
-  expect_true(all(opdf$noisy.repl %in% 1:5))
-  expect_true(all(table(opdf$x) == 5))
+  expect_true(all(opdf$noisy.repl == unlist(lapply(1:6, function(x) head(1:6,x)))))
 })
+