BREAKING_CHANGE: change format for binary classification heads (#385)

Author: sebffischer

NAMESPACE

@@ -36,6 +36,8 @@ S3method(marshal_model,learner_torch_model)
 S3method(materialize,data.frame)
 S3method(materialize,lazy_tensor)
 S3method(materialize,list)
+S3method(output_dim_for,TaskClassif)
+S3method(output_dim_for,TaskRegr)
 S3method(print,ModelDescriptor)
 S3method(print,Select)
 S3method(print,TorchIngressToken)
@@ -194,6 +196,7 @@ export(nn_squeeze)
 export(nn_tokenizer_categ)
 export(nn_tokenizer_num)
 export(nn_unsqueeze)
+export(output_dim_for)
 export(pipeop_preproc_torch)
 export(replace_head)
 export(select_all)

NEWS.md

@@ -6,6 +6,11 @@
 * feat: Added encoders for numericals and categoricals
 * feat: Added `po("nn_fn")` for calling custom functions in a network.
 * feat: Added `po("nn_ft_cls")` for concatenating a CLS token to a tokenized input.
+* BREAKING_CHANGE: The output dimension of neural networks for binary classification tasks is now
+  expected to be 1 and not 2 as before. The behavior of `nn("head")` was also changed to match this.
+  This means that for binary classification tasks, `t_loss("cross_entropy")` now generates
+  `nn_bce_with_logits_loss` instead of `nn_cross_entropy_loss`.
+  This also came with a reparametrization of the `t_loss("cross_entropy")` loss (thanks to @tdhock, #374).
 
 # mlr3torch 0.2.1
 

R/LearnerTorch.R

@@ -29,6 +29,21 @@
 #' To do so, you just need to include `epochs = to_tune(upper = <upper>, internal = TRUE)` in the search space,
 #' where `<upper>` is the maximally allowed number of epochs, and configure the early stopping.
 #'
+#' @section Network Head and Target Encoding:
+#' Torch learners are expected to have the following output:
+#' * binary classification: `c(batch_size, 1)`, representing the logits for the positive class.
+#' * multiclass classification: `c(batch_size, n_classes)`, representing the logits for all classes.
+#' * regression: `c(batch_size, 1)` representing the response prediction.
+#'
+#' Furthermore, the target encoding is expected to be as follows:
+#' * regression: The `numeric` target variable of a [`TaskRegr`][mlr3::TaskRegr] is encoded as a
+#'   [`torch_float`][torch::torch_float] with shape `c(batch_size, 1)`.
+#' * binary classification: The `factor` target variable of a [`TaskClassif`][mlr3::TaskClassif] is encoded as a
+#'   [`torch_float`][torch::torch_float] with shape `(batch_size, 1)` where the positive class is `1` and the negative
+#'   class is `0`.
+#' * multi-class classification: The `factor` target variable of a [`TaskClassif`][mlr3::TaskClassif] is a label-encoded
+#'   [`torch_long`][torch::torch_long] with shape `(batch_size, n_classes)` starting at `1` and ending at `n_classes`.
+#'
 #' @template param_id
 #' @template param_task_type
 #' @template param_param_vals
@@ -81,8 +96,8 @@
 #'   ([`Task`][mlr3::Task], `list()`) -> [`nn_module`][torch::nn_module]\cr
 #'   Construct a [`torch::nn_module`] object for the given task and parameter values, i.e. the neural network that
 #'   is trained by the learner.
-#'   For classification, the output of this network are expected to be the scores before the application of the
-#'   final softmax layer.
+#'   Note that a specific output shape is expected from the returned network, see section *Network Head and Target Encoding*.
+#'   You can use [`output_dim_for()`] to obtain the correct output dimension for a given task.
 #' * `.dataset(task, param_vals)`\cr
 #'   ([`Task`][mlr3::Task], `list()`) -> [`torch::dataset`]\cr
 #'   Create the dataset for the task.
@@ -92,17 +107,19 @@
 #'   * `y` is the target tensor.
 #'   * `.index` are the indices of the batch (`integer()` or a `torch_int()`).
 #'
+#'   For information on the expected target encoding of `y`, see section *Network Head and Target Encoding*.
 #'   Moreover, one needs to pay attention respect the row ids of the provided task.
+#'   It is recommended to relu on [`task_dataset`] for creating the [`dataset`][torch::dataset].
 #'
 #' It is also possible to overwrite the private `.dataloader()` method.
 #' This must respect the dataloader parameters from the [`ParamSet`][paradox::ParamSet].
 #'
 #' * `.dataloader(dataset, param_vals)`\cr
 #'   ([`Task`][mlr3::Task], `list()`) -> [`torch::dataloader`]\cr
 #'   Create a dataloader from the task.
-#'   Needs to respect at least `batch_size` and `shuffle` (otherwise predictions can be permuted).
+#'   Needs to respect at least `batch_size` and `shuffle` (otherwise predictions will be incorrectly ordered).
 #'
-#' To change the predict types, the it is possible to overwrite the method below:
+#' To change the predict types, it is possible to overwrite the method below:
 #'
 #' * `.encode_prediction(predict_tensor, task)`\cr
 #'   ([`torch_tensor`][torch::torch_tensor], [`Task`][mlr3::Task]) -> `list()`\cr

R/LearnerTorchFeatureless.R

@@ -41,7 +41,7 @@ LearnerTorchFeatureless = R6Class("LearnerTorchFeatureless",
   ),
   private = list(
     .network = function(task, param_vals) {
-      nn_featureless(nout = get_nout(task))
+      nn_featureless(nout = output_dim_for(task))
     },
     .dataset = function(task, dataset) {
       dataset_featureless(task)
@@ -52,7 +52,7 @@ LearnerTorchFeatureless = R6Class("LearnerTorchFeatureless",
 dataset_featureless = dataset(
   initialize = function(task) {
     self$task = task
-    self$target_batchgetter = get_target_batchgetter(task$task_type)
+    self$target_batchgetter = get_target_batchgetter(task)
   },
   .getbatch = function(index) {
     target = self$task$data(rows = self$task$row_ids[index], cols = self$task$target_names)

R/LearnerTorchMLP.R

@@ -87,7 +87,7 @@ LearnerTorchMLP = R6Class("LearnerTorchMLP",
   private = list(
     .network = function(task, param_vals) {
       # verify_train_task was already called beforehand, so we can make some assumptions
-      d_out = get_nout(task)
+      d_out = output_dim_for(task)
       d_in = if (single_lazy_tensor(task)) {
         private$.get_input_shape(task, param_vals$shape)[2L]
       } else {

R/LearnerTorchModel.R

@@ -129,7 +129,7 @@ LearnerTorchModel = R6Class("LearnerTorchModel",
       dataset = task_dataset(
         task,
         feature_ingress_tokens = ingress_tokens,
-        target_batchgetter = get_target_batchgetter(self$task_type)
+        target_batchgetter = get_target_batchgetter(task)
       )
     },
     .network_stored = NULL,

R/LearnerTorchModule.R

@@ -35,7 +35,7 @@
 #' nn_one_layer = nn_module("nn_one_layer",
 #'   initialize = function(task, size_hidden) {
 #'     self$first = nn_linear(task$n_features, size_hidden)
-#'     self$second = nn_linear(size_hidden, length(task$class_names))
+#'     self$second = nn_linear(size_hidden, output_dim_for(task))
 #'   },
 #'   # argument x corresponds to the ingress token x
 #'   forward = function(x) {
@@ -117,7 +117,7 @@ LearnerTorchModule = R6Class("LearnerTorchModule",
       dataset = task_dataset(
         task,
         feature_ingress_tokens = ingress_tokens,
-        target_batchgetter = get_target_batchgetter(self$task_type)
+        target_batchgetter = get_target_batchgetter(task)
       )
     },
 

R/LearnerTorchTabResNet.R

@@ -43,7 +43,7 @@ LearnerTorchTabResNet = R6Class("LearnerTorchTabResNet",
       )
 
       private$.param_set_base =  ps(
-        n_blocks = p_int(1, tags = c("train", "required")),
+        n_blocks = p_int(0, tags = c("train", "required")),
         d_block = p_int(1, tags = c("train", "required"))
       )
       param_set = alist(private$.block$param_set, private$.param_set_base)
@@ -65,10 +65,10 @@ LearnerTorchTabResNet = R6Class("LearnerTorchTabResNet",
   private = list(
     .block = NULL,
     .dataset = function(task, param_vals) {
-      dataset_num(task, param_vals)
+      dataset_num(task, param_vals, argname = "num.input")
     },
     .network = function(task, param_vals) {
-      graph = po("torch_ingress_num") %>>%
+      graph = po("torch_ingress_num", id = "num") %>>%
         po("nn_linear", out_features = param_vals$d_block) %>>%
         po("nn_block", private$.block, n_blocks = param_vals$n_blocks) %>>%
         po("nn_head")

R/LearnerTorchVision.R

@@ -58,7 +58,7 @@ LearnerTorchVision = R6Class("LearnerTorchVision",
   private = list(
     .module_generator = NULL,
     .network = function(task, param_vals) {
-      nout = get_nout(task)
+      nout = output_dim_for(task)
       if (param_vals$pretrained) {
         network = replace_head(private$.module_generator(pretrained = TRUE), nout)
         return(network)
@@ -107,126 +107,126 @@ replace_head.VGG = function(network, d_out) {
 }
 
 #' @include aaa.R
-register_learner("classif.alexnet", 
+register_learner("classif.alexnet",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("alexnet", torchvision::model_alexnet, "AlexNet",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-# register_learner("classif.inception_v3", 
+# register_learner("classif.inception_v3",
 #   function(loss = NULL, optimizer = NULL, callbacks = list()) {
 #     LearnerTorchVision$new("inception_v3", torchvision::model_inception_v3, "Inception V3",
 #     loss = loss, optimizer = optimizer, callbacks = callbacks)
 #   }
 # )
 
-register_learner("classif.mobilenet_v2", 
+register_learner("classif.mobilenet_v2",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("mobilenet_v2", torchvision::model_mobilenet_v2, "Mobilenet V2",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.resnet18", 
+register_learner("classif.resnet18",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("resnet18", torchvision::model_resnet18, "ResNet-18",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.resnet34", 
+register_learner("classif.resnet34",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("resnet34", torchvision::model_resnet34, "ResNet-34",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.resnet50", 
+register_learner("classif.resnet50",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("resnet50", torchvision::model_resnet50, "ResNet-50",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.resnet101", 
+register_learner("classif.resnet101",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("resnet101", torchvision::model_resnet101, "ResNet-101",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.resnet152", 
+register_learner("classif.resnet152",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("resnet152", torchvision::model_resnet152, "ResNet-152",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.resnext101_32x8d", 
+register_learner("classif.resnext101_32x8d",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("resnext101_32x8d", torchvision::model_resnext101_32x8d, "ResNeXt-101 32x8d",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.resnext50_32x4d", 
+register_learner("classif.resnext50_32x4d",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("resnext50_32x4d", torchvision::model_resnext50_32x4d, "ResNeXt-50 32x4d",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg11", 
+register_learner("classif.vgg11",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg11", torchvision::model_vgg11, "VGG 11",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg11_bn", 
+register_learner("classif.vgg11_bn",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg11_bn", torchvision::model_vgg11_bn, "VGG 11",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg13", 
+register_learner("classif.vgg13",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg13", torchvision::model_vgg13, "VGG 13",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg13_bn", 
+register_learner("classif.vgg13_bn",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg13_bn", torchvision::model_vgg13_bn, "VGG 13",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg16", 
+register_learner("classif.vgg16",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg16", torchvision::model_vgg16, "VGG 16",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg16_bn", 
+register_learner("classif.vgg16_bn",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg16_bn", torchvision::model_vgg16_bn, "VGG 16",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg19", 
+register_learner("classif.vgg19",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg19", torchvision::model_vgg19, "VGG 19",
       loss = loss, optimizer = optimizer, callbacks = callbacks)
   }
 )
 
-register_learner("classif.vgg19_bn", 
+register_learner("classif.vgg19_bn",
   function(loss = NULL, optimizer = NULL, callbacks = list()) {
     LearnerTorchVision$new("vgg19_bn", torchvision::model_vgg19_bn, "VGG 19",
       loss = loss, optimizer = optimizer, callbacks = callbacks)

R/PipeOpTorch.R

@@ -98,7 +98,7 @@
 #'     self$output = if (task$task_type == "regr") {
 #'       torch::nn_linear(d_hidden, 1)
 #'     } else if (task$task_type == "classif") {
-#'       torch::nn_linear(d_hidden, length(task$class_names))
+#'       torch::nn_linear(d_hidden, output_dim_for(task))
 #'     }
 #'   },
 #'   forward = function(x) {