Examples }
data_ts[n, ] <- m_1[-1]
}
-data_ts <- data.table::as.data.table(data_ts)
+data_ts <- data.table::as.data.table(data_ts)
x_var_ts <- paste0("X", 1:40)
y_var_ts <- "X41"
@@ -642,6 +643,307 @@ Examples group = group_ts
)
+MSEv evaluation criterion
+
+We can use the \(\operatorname{MSE}_{v}\) criterion proposed
+by Frye et al. (2021), and later used by,
+e.g., Olsen et al. (2022) and Olsen et al. (2023), to evaluate and rank the
+approaches/methods. The \(\operatorname{MSE}_{v}\) is given by
+\[\begin{align}
+ \label{eq:MSE_v}
+ \operatorname{MSE}_{v} = \operatorname{MSE}_{v}(\text{method }
+\texttt{q})
+ =
+ \frac{1}{N_\mathcal{S}} \sum_{\mathcal{S} \in
+\mathcal{P}^*(\mathcal{M})} \frac{1}{N_\text{explain}}
+ \sum_{i=1}^{N_\text{explain}} \left( f(\boldsymbol{x}^{[i]}) -
+{\hat{v}}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}^{[i]})\right)^2\!,
+\end{align}\]
+where \({\hat{v}}_{\texttt{q}}\) is
+the estimated contribution function using method \(\texttt{q}\) and \(N_\mathcal{S} = |\mathcal{P}^*(\mathcal{M})| =
+2^M-2\), i.e., we have removed the empty (\(\mathcal{S} = \emptyset\)) and the grand
+combinations (\(\mathcal{S} =
+\mathcal{M}\)) as they are method independent. Meaning that these
+two combinations do not influence the ranking of the methods as the
+methods are not used to compute the contribution function for them.
+The motivation behind the \(\operatorname{MSE}_{v}\) criterion is that
+\(\mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}}
+(v_{\texttt{true}}(\mathcal{S},\boldsymbol{x}) -
+\hat{v}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}))^2\) can be
+decomposed as
+\[\begin{align}
+ \label{eq:expectation_decomposition}
+ \begin{split}
+ \mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}}
+(v_{\texttt{true}}(\mathcal{S}, \boldsymbol{x})-
+\hat{v}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}))^2
+ &=
+ \mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}} (f(\boldsymbol{x})
+- \hat{v}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}))^2 \\
+ &\phantom{\,\,\,\,\,\,\,}-
+\mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}}
+(f(\boldsymbol{x})-v_{\texttt{true}}(\mathcal{S}, \boldsymbol{x}))^2,
+ \end{split}
+\end{align}\]
+see Appendix A in Covert, Lundberg, and Lee
+(2020). The first term on the right-hand side of the equation
+above can be estimated by \(\operatorname{MSE}_{v}\), while the second
+term is a fixed (unknown) constant not influenced by the approach .
+Thus, a low value of \(\operatorname{MSE}_{v}\) indicates that the
+estimated contribution function \(\hat{v}_{\texttt{q}}\) is closer to the
+true counterpart \(v_{\texttt{true}}\)
+than a high value.
+In shapr, we allow for weighting the combinations in the
+\(\operatorname{MSE}_{v}\) evaluation
+criterion either uniformly or by using the corresponding Shapley kernel
+weights (or the sampling frequencies when sampling of combinations is
+used). This is determined by the logical parameter
+MSEv_uniform_comb_weights in the explain()
+function, and the default is to do uniform weighting, that is,
+MSEv_uniform_comb_weights = TRUE.
+
+Advantage:
+
+An advantage of the \(\operatorname{MSE}_{v}\) criterion is that
+\(v_\texttt{true}\) is not involved.
+Thus, we can apply it as an evaluation criterion to real-world data sets
+where the true Shapley values are unknown.
+
+
+Disadvantages:
+
+First, we can only use the \(\operatorname{MSE}_{v}\) criterion to rank
+the methods and not assess their closeness to the optimum since the
+minimum value of the \(\operatorname{MSE}_{v}\) criterion is
+unknown. Second, the criterion evaluates the contribution functions and
+not the Shapley values.
+Note that Olsen et al. (2023) observed
+a relatively linear relationship between the \(\operatorname{MSE}_{v}\) criterion and the
+mean absolute error \((\operatorname{MAE})\) between the true and
+estimated Shapley values in extensive simulation studies where the true
+Shapley values were known. That is, a method that achieves a low \(\operatorname{MSE}_{v}\) score also tends
+to obtain a low \(\operatorname{MAE}\)
+score, and vice versa.
+
+
+Confidence intervals
+
+The \(\operatorname{MSE}_{v}\)
+criterion can be written as \(\operatorname{MSE}_{v} =
+\frac{1}{N_\text{explain}}\sum_{i=1}^{N_\text{explain}}
+\operatorname{MSE}_{v,\text{explain }i}\). We can therefore use
+the central limit theorem to compute an approximate confidence interval
+for the \(\operatorname{MSE}_{v}\)
+criterion. We have that \(\operatorname{MSE}_{v} \pm
+t_{\alpha/2}\frac{\operatorname{SD}(\operatorname{MSE}_{v})}{\sqrt{N_\text{explain}}}\)
+is a \((1-\alpha/2)\%\) approximate
+confidence interval for the evaluation criterion, where \(t_{\alpha/2}\) is the \(\alpha/2\) percentile of the \(T_{N_\text{explain}-1}\) distribution. Note
+that \(N_\text{explain}\) should be
+large (rule of thumb is at least \(30\)) for the central limit theorem to be
+valid. The quantities \(\operatorname{MSE}_{v}\) and \(\frac{\operatorname{SD}(\operatorname{MSE}_{v})}{\sqrt{N_\text{explain}}}\)
+are returned by the explain() function in the
+MSEv list of data tables. We can also compute similar
+approximate confidence interval for \(\operatorname{MSE}_{v}\) criterion for each
+combination/coalition when only averaging over the observations.
+However, it does not make sense in the other direction, i.e., when only
+averaging over the combinations for each observation, as each
+combination is a different prediction tasks.
+
+
+MSEv examples
+
+Start by explaining the predictions by using different methods and
+combining them into lists.
+
+# We use more explicands here for more stable confidence intervals
+ind_x_explain <- 1:25
+x_train <- data[-ind_x_explain, ..x_var]
+y_train <- data[-ind_x_explain, get(y_var)]
+x_explain <- data[ind_x_explain, ..x_var]
+
+# Fitting a basic xgboost model to the training data
+model <- xgboost::xgboost(
+ data = as.matrix(x_train),
+ label = y_train,
+ nround = 20,
+ verbose = FALSE
+)
+
+# Specifying the phi_0, i.e. the expected prediction without any features
+p0 <- mean(y_train)
+
+# Independence approach
+explanation_independence <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "independence",
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Empirical approach
+explanation_empirical <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "empirical",
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Gaussian 1e1 approach
+explanation_gaussian_1e1 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = p0,
+ n_samples = 1e1,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Gaussian 1e2 approach
+explanation_gaussian_1e2 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Combined approach
+explanation_combined <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = c("gaussian", "empirical", "independence"),
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Create a list of explanations with names
+explanation_list_named <- list(
+ "Ind." = explanation_independence,
+ "Emp." = explanation_empirical,
+ "Gaus. 1e1" = explanation_gaussian_1e1,
+ "Gaus. 1e2" = explanation_gaussian_1e2,
+ "Combined" = explanation_combined
+)
+We can then compare the different approaches by creating plots of the
+\(\operatorname{MSE}_{v}\) evaluation
+criterion.
+
+# Create the MSEv plots with approximate 95% confidence intervals
+MSEv_plots <- plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = c("overall", "comb", "explicand"),
+ CI_level = 0.95
+)
+
+# 5 plots are made
+names(MSEv_plots)
+#> [1] "MSEv_explicand_bar" "MSEv_explicand_line_point"
+#> [3] "MSEv_combination_bar" "MSEv_combination_line_point"
+#> [5] "MSEv_bar"
+The main plot if interest is the MSEv_bar, which
+displays the \(\operatorname{MSE}_{v}\)
+evaluation criterion for each method averaged over both the
+combinations/coalitions and test observations/explicands. However, we
+can also look at the other plots where we have only averaged over the
+observations or the combinations (both as bar and line plots).
+
+# The main plot of the overall MSEv averaged over both the combinations and observations
+MSEv_plots$MSEv_bar
+
+
+
+# The MSEv averaged over only the explicands for each combinations
+MSEv_plots$MSEv_combination_bar
+
+
+
+# The MSEv averaged over only the combinations for each observation/explicand
+MSEv_plots$MSEv_explicand_bar
+
+
+
+# To see which coalition S each of the `id_combination` corresponds to,
+# i.e., which features that are conditions on.
+explanation_list_named[[1]]$MSEv$MSEv_combination[, c("id_combination", "features")]
+#> id_combination features
+#> 1: 2 1
+#> 2: 3 2
+#> 3: 4 3
+#> 4: 5 4
+#> 5: 6 1,2
+#> 6: 7 1,3
+#> 7: 8 1,4
+#> 8: 9 2,3
+#> 9: 10 2,4
+#> 10: 11 3,4
+#> 11: 12 1,2,3
+#> 12: 13 1,2,4
+#> 13: 14 1,3,4
+#> 14: 15 2,3,4
+We can specify the index_x_explain and
+id_combination parameters in
+plot_MSEv_eval_crit() to only plot certain test
+observations and combinations, respectively.
+
+# We can specify which test observations or combinations to plot
+plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "explicand",
+ index_x_explain = c(1, 3:4, 6),
+ CI_level = 0.95
+)$MSEv_explicand_bar
+
+
+plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "comb",
+ id_combination = c(3, 4, 9, 13:15),
+ CI_level = 0.95
+)$MSEv_combination_bar
+#> NULL
+We can also alter the plots design-wise as we do in the code
+below.
+
+bar_text_n_decimals <- 1
+CI_level <- 0.95
+MSEv_plot <- plot_MSEv_eval_crit(explanation_list_named, CI_level = CI_level)$MSEv_bar
+MSEv_plot +
+ ggplot2::scale_x_discrete(limits = rev(levels(MSEv_plot$data$Method))) +
+ ggplot2::coord_flip() +
+ ggplot2::scale_fill_brewer(palette = "Paired") +
+ ggplot2::theme_minimal() + # This must be set before other theme calls
+ ggplot2::theme(
+ plot.title = ggplot2::element_text(size = 10),
+ legend.position = "bottom"
+ ) +
+ ggplot2::geom_text(
+ ggplot2::aes(label = sprintf(
+ paste("%.", sprintf("%d", bar_text_n_decimals), "f", sep = ""),
+ round(MSEv, bar_text_n_decimals)
+ )),
+ vjust = -0.35, # This number might need altering for different plots sizes
+ hjust = 1.1, # This number might need altering for different plots sizes
+ color = "black",
+ position = ggplot2::position_dodge(0.9),
+ size = 4
+ )
+
+
+
+
Main arguments in explain
When using explain, the default behavior is to use all
@@ -726,11 +1028,11 @@
Explaining a fore
it may be of more interesting to explain the impact of each variable,
rather than each lag of each variable. This can be done by setting
group_lags = TRUE.
-
+
# Simulate time series data with AR(1)-structure.
set.seed(1)
data_ts <- data.frame(Y = arima.sim(list(order = c(1, 0, 0), ar = .5), n = 500))
-data_ts <- data.table::as.data.table(data_ts)
+data_ts <- data.table::as.data.table(data_ts)
# Fit an ARIMA(2, 0, 0) model.
arima_model <- arima(data_ts, order = c(2, 0, 0))
@@ -778,8 +1080,8 @@ Explaining a fore
model_ar_temp. This model is then used to make a forecast
of the temperature of the day that comes after the last day in the data,
this forecast starts from index 153.
-
-data <- data.table::as.data.table(airquality)
+
+data <- data.table::as.data.table(airquality)
model_ar_temp <- ar(data$Temp, order = 2)
@@ -812,7 +1114,7 @@ Explaining a fore
separately. Grouping lags may be more interesting to do in a model with
multiple variables, as it is then possible to explain each variable
separately.
-
+
explanation <- explain_forecast(
model = model_ar_temp,
y = data[, "Temp"],
@@ -844,8 +1146,8 @@ Explaining a fore
points, the model is just fit on the 151 first observations, leaving two
observations of Wind to be used as exogenous values during
the prediction phase.
-
-data <- data.table::as.data.table(airquality)
+
+data <- data.table::as.data.table(airquality)
data_fit <- data[seq_len(151), ]
@@ -868,7 +1170,7 @@ Explaining a fore
possible for shapr to not only explain the effect of the
first lag of the exogenous variable, but also the contemporary effect
during the forecasting period.
-
+
explanation <- explain_forecast(
model = model_ar_temp,
y = data_fit[, "Temp"],
@@ -928,7 +1230,7 @@ Combined approach and
"gaussian" when conditioning on respectively 1, 2 and 3
features.
-
+
# Use the combined approach
explanation_combined <- explain(
model = model,
@@ -940,11 +1242,11 @@ Combined approach# Plot the resulting explanations for observations 1 and 6, excluding
# the no-covariate effect
plot(explanation_combined, bar_plot_phi0 = FALSE, index_x_explain = c(1, 6))
-
+
As a second example using "ctree" to conditin on 1 and 2
features, and "empirical" when conditioning on 3
features:
-
+
# Use the combined approach
explanation_combined <- explain(
model = model,
@@ -963,7 +1265,7 @@ Explain groups of features) for intuition and real world
examples. Explaining prediction in terms of groups of features is very
easy using shapr:
-
+
# Define the feature groups
group_list <- list(
A = c("Temp", "Month"),
@@ -981,16 +1283,36 @@ Explain groups of features)
# Prints the group-wise explanations
explanation_group
-#> none A B
-#> 1: 43.09 -29.25 16.0731
-#> 2: 43.09 -15.17 -7.8373
-#> 3: 43.09 -13.07 -10.8778
-#> 4: 43.09 -17.47 0.6653
-#> 5: 43.09 -28.27 3.5289
-#> 6: 43.09 -20.59 -3.3793
+#> none A B
+#> 1: 47.27 -29.588 13.1628
+#> 2: 47.27 -11.834 -15.7011
+#> 3: 47.27 -15.976 -17.5729
+#> 4: 47.27 -25.067 -5.1374
+#> 5: 47.27 -35.848 20.2892
+#> 6: 47.27 -27.257 -8.4830
+#> 7: 47.27 -14.960 -21.3995
+#> 8: 47.27 -18.325 7.3791
+#> 9: 47.27 -23.012 9.6591
+#> 10: 47.27 -16.189 -5.6100
+#> 11: 47.27 -25.607 -10.1334
+#> 12: 47.27 -25.065 -5.1394
+#> 13: 47.27 -25.841 -0.7281
+#> 14: 47.27 -21.518 -13.3293
+#> 15: 47.27 -21.248 -1.3199
+#> 16: 47.27 -13.676 -16.9497
+#> 17: 47.27 -13.899 -14.8890
+#> 18: 47.27 -12.276 -8.2472
+#> 19: 47.27 -13.768 -13.5242
+#> 20: 47.27 -24.866 -10.8744
+#> 21: 47.27 -14.486 -22.7674
+#> 22: 47.27 -4.122 -14.2893
+#> 23: 47.27 -11.218 22.4682
+#> 24: 47.27 -33.002 14.2114
+#> 25: 47.27 -16.251 -8.6796
+#> none A B
# Plots the group-wise explanations
plot(explanation_group, bar_plot_phi0 = TRUE, index_x_explain = c(1, 6))
-
+
Explain custom models
@@ -1044,7 +1366,7 @@ Explain custom modelsget_model_specs function. We do this for the
gbm model class from the gbm package, fitted
to the same airquality data set as used above.
-
+
library(gbm)
#> Loaded gbm 2.1.8.1
@@ -1100,8 +1422,8 @@ Explain custom models
# Plot results
plot(explanation_custom, index_x_explain = c(1, 6))
-
-
+
+
#### Minimal version of the three required model functions ####
@@ -1129,7 +1451,7 @@ Explain custom models
# Plot results
plot(explanation_custom_minimal, index_x_explain = c(1, 6))
-
+
@@ -1163,7 +1485,7 @@ Batch computation
+
explanation_batch <- explain(
model = model,
x_explain = x_explain,
@@ -1197,7 +1519,7 @@ Parallelized computationn_batches
equals some positive integer multiplied by the
number of workers. Below is a basic example of a parallelization with
two workers.
-
+
library(future)
future::plan(multisession, workers = 2)
@@ -1246,7 +1568,7 @@ Progress updatesprogressr
vignette.
A full code example of using progressr with
shapr is shown below:
-
+
library(progressr)
progressr::handlers(global = TRUE)
# If no progression handler is specified, the txtprogressbar is used
@@ -1311,6 +1633,17 @@ References
+Covert, Ian, Scott M Lundberg, and Su-In Lee. 2020. “Understanding
+Global Feature Contributions with Additive Importance Measures.”
+Advances in Neural Information Processing Systems 33: 17212–23.
+
+
+Frye, Christopher, Damien de Mijolla, Tom Begley, Laurence Cowton, Megan
+Stanley, and Ilya Feige. 2021. “Shapley Explainability on the Data
+Manifold.” In International Conference on Learning
+Representations.
+
Hothorn, Torsten, Kurt Hornik, and Achim Zeileis. 2006. “Unbiased
Recursive Partitioning: A Conditional Inference Framework.”
@@ -1334,6 +1667,18 @@ References
+Olsen, Lars Henry Berge, Ingrid Kristine Glad, Martin Jullum, and
+Kjersti Aas. 2022. “Using Shapley Values and Variational
+Autoencoders to Explain Predictive Models with Dependent Mixed
+Features.” Journal of Machine Learning Research 23
+(213): 1–51.
+
+
+———. 2023. “A Comparative Study of Methods for Estimating
+Conditional Shapley Values and When to Use Them.” arXiv
+Preprint arXiv:2305.09536.
+
Redelmeier, Annabelle, Martin Jullum, and Kjersti Aas. 2020.
“Explaining Predictive Models with Mixed Features Using Shapley
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png
index d2ae3cc9d..bfa327d0e 100644
Binary files a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png
new file mode 100644
index 000000000..46339f4b1
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png
new file mode 100644
index 000000000..fb5790961
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png
index b50379788..ad92aaf01 100644
Binary files a/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png
new file mode 100644
index 000000000..021decff0
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png
new file mode 100644
index 000000000..70d491401
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png
new file mode 100644
index 000000000..e137d9551
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png
new file mode 100644
index 000000000..0046444c7
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png
new file mode 100644
index 000000000..0046444c7
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png differ
diff --git a/index.html b/index.html
index d5b870abc..3086e27cc 100644
--- a/index.html
+++ b/index.html
@@ -174,7 +174,7 @@ Examplelibrary(shapr)
data("airquality")
-data <- data.table::as.data.table(airquality)
+data <- data.table::as.data.table(airquality)
data <- data[complete.cases(data), ]
x_var <- c("Solar.R", "Wind", "Temp", "Month")
diff --git a/pkgdown.yml b/pkgdown.yml
index fe9cff91e..c63ba3767 100644
--- a/pkgdown.yml
+++ b/pkgdown.yml
@@ -3,7 +3,7 @@ pkgdown: 2.0.7
pkgdown_sha: ~
articles:
understanding_shapr: understanding_shapr.html
-last_built: 2023-11-20T16:01Z
+last_built: 2023-12-11T14:36Z
urls:
reference: https://norskregnesentral.github.io/shapr/reference
article: https://norskregnesentral.github.io/shapr/articles
diff --git a/reference/compute_MSEv_eval_crit.html b/reference/compute_MSEv_eval_crit.html
new file mode 100644
index 000000000..f394f01dd
--- /dev/null
+++ b/reference/compute_MSEv_eval_crit.html
@@ -0,0 +1,169 @@
+
+Mean Squared Error of the Contribution Function v(S) — compute_MSEv_eval_crit • shapr
+
+
+
+
+
+
+
+
+
+
+ Mean Squared Error of the Contribution Function v(S)
+ Source: R/finalize_explanation.R
+ compute_MSEv_eval_crit.Rd
+
+
+
+ Function that computes the Mean Squared Error (MSEv) of the contribution function
+v(s) as proposed by Frye et al. (2019) and used by
+Olsen et al. (2022).
+
+
+
+ compute_MSEv_eval_crit(
+ internal,
+ dt_vS,
+ MSEv_uniform_comb_weights,
+ MSEv_skip_empty_full_comb = TRUE
+)
+
+
+
+ Arguments
+ - internal
+List.
+Holds all parameters, data, functions and computed objects used within explain()
+The list contains one or more of the elements parameters, data, objects, output.
- dt_vS
+Data.table of dimension n_combinations times n_explain + 1 containing the contribution function
+estimates. The first column is assumed to be named id_combination and containing the ids of the combinations.
+The last row is assumed to be the full combination, i.e., it contains the predicted responses for the observations
+which are to be explained.
- MSEv_uniform_comb_weights
+Logical. If TRUE (default), then the function weights the combinations
+uniformly when computing the MSEv criterion. If FALSE, then the function use the Shapley kernel weights to
+weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the
+sampling frequency when not all combinations are considered.
- MSEv_skip_empty_full_comb
+Logical. If TRUE (default), we exclude the empty and grand
+combinations/coalitions when computing the MSEv evaluation criterion. This is reasonable as they are identical
+for all methods, i.e., their contribution function is independent of the used method as they are special cases not
+effected by the used method. If FALSE, we include the empty and grand combinations/coalitions. In this situation,
+we also recommend setting MSEv_uniform_comb_weights = TRUE, as otherwise the large weights for the empty and
+grand combinations/coalitions will outweigh all other combinations and make the MSEv criterion uninformative.
+
+ Value
+
+
+List containing:
MSEvA data.table with the overall MSEv evaluation criterion averaged
+over both the combinations/coalitions and observations/explicands. The data.table
+also contains the standard deviation of the MSEv values for each explicand (only averaged over the combinations)
+divided by the square root of the number of explicands.
MSEv_explicandA data.table with the mean squared error for each
+explicand, i.e., only averaged over the combinations/coalitions.
MSEv_combinationA data.table with the mean squared error for each
+combination/coalition, i.e., only averaged over the explicands/observations.
+The data.table also contains the standard deviation of the MSEv values for
+each combination divided by the square root of the number of explicands.
+
+ Details
+ The MSEv evaluation criterion does not rely on access to the true contribution functions nor the
+true Shapley values to be computed. A lower value indicates better approximations, however, the
+scale and magnitude of the MSEv criterion is not directly interpretable in regard to the precision
+of the final estimated Shapley values. Olsen et al. (2022)
+illustrates in Figure 11 a fairly strong linear relationship between the MSEv criterion and the
+MAE between the estimated and true Shapley values in a simulation study. Note that explicands
+refer to the observations whose predictions we are to explain.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/reference/compute_shapley_new.html b/reference/compute_shapley_new.html
index 5e44a3bc7..96820fa22 100644
--- a/reference/compute_shapley_new.html
+++ b/reference/compute_shapley_new.html
@@ -80,7 +80,7 @@ Arguments
Value
-A data.table with shapley values for each test observation.
+A data.table with Shapley values for each test observation.
diff --git a/reference/explain.html b/reference/explain.html
index 2ad8078c5..8058e0b0b 100644
--- a/reference/explain.html
+++ b/reference/explain.html
@@ -79,6 +79,7 @@ Explain the output of machine learning models with more accurately estimated
keep_samp_for_vS = FALSE,
predict_model = NULL,
get_model_specs = NULL,
+ MSEv_uniform_comb_weights = TRUE,
timing = TRUE,
...
)
@@ -195,6 +196,13 @@ Arguments
Can also be used to override the default function for natively supported model classes.
+
MSEv_uniform_comb_weights
+Logical. If TRUE (default), then the function weights the combinations
+uniformly when computing the MSEv criterion. If FALSE, then the function use the Shapley kernel weights to
+weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the
+sampling frequency when not all combinations are considered.
timing
Logical.
Whether the timing of the different parts of the explain() should saved in the model object.
Value
List with the different parameters, data and functions used internally
pred_explain
-Numeric vector with the predictions for the explained observations.
Numeric vector with the predictions for the explained observations
MSEv
+List with the values of the MSEv evaluation criterion for the approach.
shapley_values is a data.table where the number of rows equals
diff --git a/reference/explain_forecast.html b/reference/explain_forecast.html
index 23fa8f411..a2d1b7a98 100644
--- a/reference/explain_forecast.html
+++ b/reference/explain_forecast.html
@@ -358,7 +358,10 @@
Value
List with the different parameters, data and functions used internally
pred_explain
-Numeric vector with the predictions for the explained observations.
Numeric vector with the predictions for the explained observations
MSEv
+List with the values of the MSEv evaluation criterion for the approach.
shapley_values is a data.table where the number of rows equals
@@ -414,7 +417,7 @@
Examples
# Load example data
data("airquality")
-data <- data.table::as.data.table(airquality)
+data <- data.table::as.data.table(airquality)
# Fit an AR(2) model.
model_ar_temp <- ar(data$Temp, order = 2)
diff --git a/reference/finalize_explanation.html b/reference/finalize_explanation.html
index 7d35b7de6..2be5b8279 100644
--- a/reference/finalize_explanation.html
+++ b/reference/finalize_explanation.html
@@ -92,7 +92,10 @@ Value
List with the different parameters, data and functions used internally
pred_explain
-Numeric vector with the predictions for the explained observations.
Numeric vector with the predictions for the explained observations
MSEv
+List with the values of the MSEv evaluation criterion for the approach.
shapley_values is a data.table where the number of rows equals
diff --git a/reference/get_data_specs.html b/reference/get_data_specs.html
index 6e74c496f..ca39f21aa 100644
--- a/reference/get_data_specs.html
+++ b/reference/get_data_specs.html
@@ -107,7 +107,7 @@
Examples
x_train <- head(airquality, -3)
x_explain <- tail(airquality, 3)
# Split data into test- and training data
-x_train <- data.table::as.data.table(head(airquality))
+x_train <- data.table::as.data.table(head(airquality))
x_train[, Temp := as.factor(Temp)]
#> Ozone Solar.R Wind Temp Month Day
#> 1: 41 190 7.4 67 5 1
diff --git a/reference/index.html b/reference/index.html
index c64d637a9..b64fbcf94 100644
--- a/reference/index.html
+++ b/reference/index.html
@@ -104,6 +104,10 @@ All functions plot(<shapr>)
Plot of the Shapley value explanations
+
+
+ Plots of the MSEv Evaluation Criterion
diff --git a/reference/plot_MSEv_eval_crit-1.png b/reference/plot_MSEv_eval_crit-1.png
new file mode 100644
index 000000000..ef2e30987
Binary files /dev/null and b/reference/plot_MSEv_eval_crit-1.png differ
diff --git a/reference/plot_MSEv_eval_crit.html b/reference/plot_MSEv_eval_crit.html
new file mode 100644
index 000000000..5bf0cdb30
--- /dev/null
+++ b/reference/plot_MSEv_eval_crit.html
@@ -0,0 +1,338 @@
+
+Plots of the MSEv Evaluation Criterion — plot_MSEv_eval_crit • shapr
+
+
+
+
+
+
+
+
+
+
+
+
+ Make plots to visualize and compare the MSEv evaluation criterion for a list of
+explain() objects applied to the same data and model. The function creates
+bar plots and line plots with points to illustrate the overall MSEv evaluation
+criterion, but also for each observation/explicand and combination by only averaging over
+the combinations and observations/explicands, respectively.
+
+
+
+ plot_MSEv_eval_crit(
+ explanation_list,
+ index_x_explain = NULL,
+ id_combination = NULL,
+ CI_level = if (length(explanation_list[[1]]$pred_explain) < 20) NULL else 0.95,
+ geom_col_width = 0.9,
+ plot_type = "overall"
+)
+
+
+
+ Arguments
+ - explanation_list
+A list of explain() objects applied to the same data and model.
+If the entries in the list are named, then the function use these names. Otherwise, they default to
+the approach names (with integer suffix for duplicates) for the explanation objects in explanation_list.
- index_x_explain
+Integer vector.
+Which of the test observations to plot. E.g. if you have
+explained 10 observations using explain(), you can generate a plot for the first 5
+observations by setting index_x_explain = 1:5.
- id_combination
+Integer vector. Which of the combinations (coalitions) to plot.
+E.g. if you used n_combinations = 16 in explain(), you can generate a plot for the
+first 5 combinations and the 10th by setting id_combination = c(1:5, 10).
- CI_level
+Positive numeric between zero and one. Default is 0.95 if the number of observations to explain is
+larger than 20, otherwise CI_level = NULL, which removes the confidence intervals. The level of the approximate
+confidence intervals for the overall MSEv and the MSEv_combination. The confidence intervals are based on that
+the MSEv scores are means over the observations/explicands, and that means are approximation normal. Since the
+standard deviations are estimated, we use the quantile t from the T distribution with N_explicands - 1 degrees of
+freedom corresponding to the provided level. Here, N_explicands is the number of observations/explicands.
+MSEv ± tSD(MSEv)/sqrt(N_explicands). Note that the explain() function already scales the standard deviation by
+sqrt(N_explicands), thus, the CI are MSEv ± tMSEv_sd, where the values MSEv and MSEv_sd are extracted from the
+MSEv data.tables in the objects in the explanation_list.
- geom_col_width
+Numeric. Bar width. By default, set to 90% of the ggplot2::resolution() of the data.
- plot_type
+Character vector. The possible options are "overall" (default), "comb", and "explicand".
+If plot_type = "overall", then the plot (one bar plot) associated with the overall MSEv evaluation criterion
+for each method is created, i.e., when averaging over both the combinations/coalitions and observations/explicands.
+If plot_type = "comb", then the plots (one line plot and one bar plot) associated with the MSEv evaluation
+criterion for each combination/coalition are created, i.e., when we only average over the observations/explicands.
+If plot_type = "explicand", then the plots (one line plot and one bar plot) associated with the MSEv evaluation
+criterion for each observations/explicands are created, i.e., when we only average over the combinations/coalitions.
+If plot_type is a vector of one or several of "overall", "comb", and "explicand", then the associated plots are
+created.
+
+ Value
+
+
+Either a single ggplot2::ggplot() object of the MSEv criterion when plot_type = "overall", or a list
+of ggplot2::ggplot() objects based on the plot_type parameter.
+
+
+
+
+ Examples
+ # Load necessary librarieslibrary(xgboost)
+library(data.table)
+library(shapr)
+library(ggplot2)
+
+# Get the data
+data("airquality")
+data <- data.table::as.data.table(airquality)
+data <- data[complete.cases(data), ]
+
+#' Define the features and the response
+x_var <- c("Solar.R", "Wind", "Temp", "Month")
+y_var <- "Ozone"
+
+# Split data into test and training data set
+ind_x_explain <- 1:25
+x_train <- data[-ind_x_explain, ..x_var]
+y_train <- data[-ind_x_explain, get(y_var)]
+x_explain <- data[ind_x_explain, ..x_var]
+
+# Fitting a basic xgboost model to the training data
+model <- xgboost::xgboost(
+ data = as.matrix(x_train),
+ label = y_train,
+ nround = 20,
+ verbose = FALSE
+)
+
+# Specifying the phi_0, i.e. the expected prediction without any features
+prediction_zero <- mean(y_train)
+
+# Independence approach
+explanation_independence <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "independence",
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 2 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Gaussian 1e1 approach
+explanation_gaussian_1e1 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = prediction_zero,
+ n_samples = 1e1
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Gaussian 1e2 approach
+explanation_gaussian_1e2 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# ctree approach
+explanation_ctree <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "ctree",
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Combined approach
+explanation_combined <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = c("gaussian", "independence", "ctree"),
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Create a list of explanations with names
+explanation_list_named <- list(
+ "Ind." = explanation_independence,
+ "Gaus. 1e1" = explanation_gaussian_1e1,
+ "Gaus. 1e2" = explanation_gaussian_1e2,
+ "Ctree" = explanation_ctree,
+ "Combined" = explanation_combined
+)
+
+if (requireNamespace("ggplot2", quietly = TRUE)) {
+ # Create the default MSEv plot where we average over both the combinations and observations
+ # with approximate 95% confidence intervals
+ plot_MSEv_eval_crit(explanation_list_named, CI_level = 0.95, plot_type = "overall")
+
+ # Can also create plots of the MSEv criterion averaged only over the combinations or observations.
+ MSEv_figures <- plot_MSEv_eval_crit(explanation_list_named,
+ CI_level = 0.95,
+ plot_type = c("overall", "comb", "explicand")
+ )
+ MSEv_figures$MSEv_bar
+ MSEv_figures$MSEv_combination_bar
+ MSEv_figures$MSEv_explicand_bar
+
+ # When there are many combinations or observations, then it can be easier to look at line plots
+ MSEv_figures$MSEv_combination_line_point
+ MSEv_figures$MSEv_explicand_line_point
+
+ # We can specify which observations or combinations to plot
+ plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "explicand",
+ index_x_explain = c(1, 3:4, 6),
+ CI_level = 0.95
+ )$MSEv_explicand_bar
+ plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "comb",
+ id_combination = c(3, 4, 9, 13:15),
+ CI_level = 0.95
+ )$MSEv_combination_bar
+
+ # We can alter the figures if other palette schemes or design is wanted
+ bar_text_n_decimals <- 1
+ MSEv_figures$MSEv_bar +
+ ggplot2::scale_x_discrete(limits = rev(levels(MSEv_figures$MSEv_bar$data$Method))) +
+ ggplot2::coord_flip() +
+ ggplot2::scale_fill_discrete() + #' Default ggplot2 palette
+ ggplot2::theme_minimal() + #' This must be set before the other theme call
+ ggplot2::theme(
+ plot.title = ggplot2::element_text(size = 10),
+ legend.position = "bottom"
+ ) +
+ ggplot2::guides(fill = ggplot2::guide_legend(nrow = 1, ncol = 6)) +
+ ggplot2::geom_text(
+ ggplot2::aes(label = sprintf(
+ paste("%.", sprintf("%d", bar_text_n_decimals), "f", sep = ""),
+ round(MSEv, bar_text_n_decimals)
+ )),
+ vjust = -1.1, # This value must be altered based on the plot dimension
+ hjust = 1.1, # This value must be altered based on the plot dimension
+ color = "black",
+ position = ggplot2::position_dodge(0.9),
+ size = 5
+ )
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/reference/setup.html b/reference/setup.html
index 90328d104..1522ec73a 100644
--- a/reference/setup.html
+++ b/reference/setup.html
@@ -76,6 +76,7 @@ check_setup
seed,
keep_samp_for_vS,
feature_specs,
+ MSEv_uniform_comb_weights = TRUE,
type = "normal",
horizon = NULL,
y = NULL,
@@ -182,6 +183,13 @@ Arguments
+MSEv_uniform_comb_weights
+Logical. If TRUE (default), then the function weights the combinations
+uniformly when computing the MSEv criterion. If FALSE, then the function use the Shapley kernel weights to
+weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the
+sampling frequency when not all combinations are considered.
type
Character.
Either "normal" or "forecast" corresponding to function setup() is called from,
@@ -249,7 +257,7 @@
Arguments
...
-Further arguments passed to approach-specific functions.
Further arguments passed to specific approaches
https://norskregnesentral.github.io/shapr/reference/check_groups.html
+
+ https://norskregnesentral.github.io/shapr/reference/compute_MSEv_eval_crit.html
+
https://norskregnesentral.github.io/shapr/reference/compute_shapley_new.html
@@ -138,6 +141,9 @@
https://norskregnesentral.github.io/shapr/reference/plot.shapr.html
+
+ https://norskregnesentral.github.io/shapr/reference/plot_MSEv_eval_crit.html
+
https://norskregnesentral.github.io/shapr/reference/predict_model.html
MSEv evaluation criterion +
+We can use the \(\operatorname{MSE}_{v}\) criterion proposed +by Frye et al. (2021), and later used by, +e.g., Olsen et al. (2022) and Olsen et al. (2023), to evaluate and rank the +approaches/methods. The \(\operatorname{MSE}_{v}\) is given by
+\[\begin{align} + \label{eq:MSE_v} + \operatorname{MSE}_{v} = \operatorname{MSE}_{v}(\text{method } +\texttt{q}) + = + \frac{1}{N_\mathcal{S}} \sum_{\mathcal{S} \in +\mathcal{P}^*(\mathcal{M})} \frac{1}{N_\text{explain}} + \sum_{i=1}^{N_\text{explain}} \left( f(\boldsymbol{x}^{[i]}) - +{\hat{v}}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}^{[i]})\right)^2\!, +\end{align}\] +where \({\hat{v}}_{\texttt{q}}\) is +the estimated contribution function using method \(\texttt{q}\) and \(N_\mathcal{S} = |\mathcal{P}^*(\mathcal{M})| = +2^M-2\), i.e., we have removed the empty (\(\mathcal{S} = \emptyset\)) and the grand +combinations (\(\mathcal{S} = +\mathcal{M}\)) as they are method independent. Meaning that these +two combinations do not influence the ranking of the methods as the +methods are not used to compute the contribution function for them.
+The motivation behind the \(\operatorname{MSE}_{v}\) criterion is that +\(\mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}} +(v_{\texttt{true}}(\mathcal{S},\boldsymbol{x}) - +\hat{v}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}))^2\) can be +decomposed as
+\[\begin{align} + \label{eq:expectation_decomposition} + \begin{split} + \mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}} +(v_{\texttt{true}}(\mathcal{S}, \boldsymbol{x})- +\hat{v}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}))^2 + &= + \mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}} (f(\boldsymbol{x}) +- \hat{v}_{\texttt{q}}(\mathcal{S}, \boldsymbol{x}))^2 \\ + &\phantom{\,\,\,\,\,\,\,}- +\mathbb{E}_\mathcal{S}\mathbb{E}_{\boldsymbol{x}} +(f(\boldsymbol{x})-v_{\texttt{true}}(\mathcal{S}, \boldsymbol{x}))^2, + \end{split} +\end{align}\] +see Appendix A in Covert, Lundberg, and Lee +(2020). The first term on the right-hand side of the equation +above can be estimated by \(\operatorname{MSE}_{v}\), while the second +term is a fixed (unknown) constant not influenced by the approach . +Thus, a low value of \(\operatorname{MSE}_{v}\) indicates that the +estimated contribution function \(\hat{v}_{\texttt{q}}\) is closer to the +true counterpart \(v_{\texttt{true}}\) +than a high value.
+In shapr, we allow for weighting the combinations in the
+\(\operatorname{MSE}_{v}\) evaluation
+criterion either uniformly or by using the corresponding Shapley kernel
+weights (or the sampling frequencies when sampling of combinations is
+used). This is determined by the logical parameter
+MSEv_uniform_comb_weights in the explain()
+function, and the default is to do uniform weighting, that is,
+MSEv_uniform_comb_weights = TRUE.
Advantage: +
+An advantage of the \(\operatorname{MSE}_{v}\) criterion is that +\(v_\texttt{true}\) is not involved. +Thus, we can apply it as an evaluation criterion to real-world data sets +where the true Shapley values are unknown.
+Disadvantages: +
+First, we can only use the \(\operatorname{MSE}_{v}\) criterion to rank +the methods and not assess their closeness to the optimum since the +minimum value of the \(\operatorname{MSE}_{v}\) criterion is +unknown. Second, the criterion evaluates the contribution functions and +not the Shapley values.
+Note that Olsen et al. (2023) observed +a relatively linear relationship between the \(\operatorname{MSE}_{v}\) criterion and the +mean absolute error \((\operatorname{MAE})\) between the true and +estimated Shapley values in extensive simulation studies where the true +Shapley values were known. That is, a method that achieves a low \(\operatorname{MSE}_{v}\) score also tends +to obtain a low \(\operatorname{MAE}\) +score, and vice versa.
+Confidence intervals +
+The \(\operatorname{MSE}_{v}\)
+criterion can be written as \(\operatorname{MSE}_{v} =
+\frac{1}{N_\text{explain}}\sum_{i=1}^{N_\text{explain}}
+\operatorname{MSE}_{v,\text{explain }i}\). We can therefore use
+the central limit theorem to compute an approximate confidence interval
+for the \(\operatorname{MSE}_{v}\)
+criterion. We have that \(\operatorname{MSE}_{v} \pm
+t_{\alpha/2}\frac{\operatorname{SD}(\operatorname{MSE}_{v})}{\sqrt{N_\text{explain}}}\)
+is a \((1-\alpha/2)\%\) approximate
+confidence interval for the evaluation criterion, where \(t_{\alpha/2}\) is the \(\alpha/2\) percentile of the \(T_{N_\text{explain}-1}\) distribution. Note
+that \(N_\text{explain}\) should be
+large (rule of thumb is at least \(30\)) for the central limit theorem to be
+valid. The quantities \(\operatorname{MSE}_{v}\) and \(\frac{\operatorname{SD}(\operatorname{MSE}_{v})}{\sqrt{N_\text{explain}}}\)
+are returned by the explain() function in the
+MSEv list of data tables. We can also compute similar
+approximate confidence interval for \(\operatorname{MSE}_{v}\) criterion for each
+combination/coalition when only averaging over the observations.
+However, it does not make sense in the other direction, i.e., when only
+averaging over the combinations for each observation, as each
+combination is a different prediction tasks.
MSEv examples +
+Start by explaining the predictions by using different methods and +combining them into lists.
+
+# We use more explicands here for more stable confidence intervals
+ind_x_explain <- 1:25
+x_train <- data[-ind_x_explain, ..x_var]
+y_train <- data[-ind_x_explain, get(y_var)]
+x_explain <- data[ind_x_explain, ..x_var]
+
+# Fitting a basic xgboost model to the training data
+model <- xgboost::xgboost(
+ data = as.matrix(x_train),
+ label = y_train,
+ nround = 20,
+ verbose = FALSE
+)
+
+# Specifying the phi_0, i.e. the expected prediction without any features
+p0 <- mean(y_train)
+
+# Independence approach
+explanation_independence <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "independence",
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Empirical approach
+explanation_empirical <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "empirical",
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Gaussian 1e1 approach
+explanation_gaussian_1e1 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = p0,
+ n_samples = 1e1,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Gaussian 1e2 approach
+explanation_gaussian_1e2 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Combined approach
+explanation_combined <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = c("gaussian", "empirical", "independence"),
+ prediction_zero = p0,
+ n_samples = 1e2,
+ n_batches = 5,
+ MSEv_uniform_comb_weights = TRUE
+)
+
+# Create a list of explanations with names
+explanation_list_named <- list(
+ "Ind." = explanation_independence,
+ "Emp." = explanation_empirical,
+ "Gaus. 1e1" = explanation_gaussian_1e1,
+ "Gaus. 1e2" = explanation_gaussian_1e2,
+ "Combined" = explanation_combined
+)We can then compare the different approaches by creating plots of the +\(\operatorname{MSE}_{v}\) evaluation +criterion.
+
+# Create the MSEv plots with approximate 95% confidence intervals
+MSEv_plots <- plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = c("overall", "comb", "explicand"),
+ CI_level = 0.95
+)
+
+# 5 plots are made
+names(MSEv_plots)
+#> [1] "MSEv_explicand_bar" "MSEv_explicand_line_point"
+#> [3] "MSEv_combination_bar" "MSEv_combination_line_point"
+#> [5] "MSEv_bar"The main plot if interest is the MSEv_bar, which
+displays the \(\operatorname{MSE}_{v}\)
+evaluation criterion for each method averaged over both the
+combinations/coalitions and test observations/explicands. However, we
+can also look at the other plots where we have only averaged over the
+observations or the combinations (both as bar and line plots).
+# The main plot of the overall MSEv averaged over both the combinations and observations
+MSEv_plots$MSEv_bar
+
+# The MSEv averaged over only the explicands for each combinations
+MSEv_plots$MSEv_combination_bar
+
+# The MSEv averaged over only the combinations for each observation/explicand
+MSEv_plots$MSEv_explicand_bar
+
+# To see which coalition S each of the `id_combination` corresponds to,
+# i.e., which features that are conditions on.
+explanation_list_named[[1]]$MSEv$MSEv_combination[, c("id_combination", "features")]
+#> id_combination features
+#> 1: 2 1
+#> 2: 3 2
+#> 3: 4 3
+#> 4: 5 4
+#> 5: 6 1,2
+#> 6: 7 1,3
+#> 7: 8 1,4
+#> 8: 9 2,3
+#> 9: 10 2,4
+#> 10: 11 3,4
+#> 11: 12 1,2,3
+#> 12: 13 1,2,4
+#> 13: 14 1,3,4
+#> 14: 15 2,3,4We can specify the index_x_explain and
+id_combination parameters in
+plot_MSEv_eval_crit() to only plot certain test
+observations and combinations, respectively.
+# We can specify which test observations or combinations to plot
+plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "explicand",
+ index_x_explain = c(1, 3:4, 6),
+ CI_level = 0.95
+)$MSEv_explicand_bar
+plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "comb",
+ id_combination = c(3, 4, 9, 13:15),
+ CI_level = 0.95
+)$MSEv_combination_bar
+#> NULLWe can also alter the plots design-wise as we do in the code +below.
+
+bar_text_n_decimals <- 1
+CI_level <- 0.95
+MSEv_plot <- plot_MSEv_eval_crit(explanation_list_named, CI_level = CI_level)$MSEv_bar
+MSEv_plot +
+ ggplot2::scale_x_discrete(limits = rev(levels(MSEv_plot$data$Method))) +
+ ggplot2::coord_flip() +
+ ggplot2::scale_fill_brewer(palette = "Paired") +
+ ggplot2::theme_minimal() + # This must be set before other theme calls
+ ggplot2::theme(
+ plot.title = ggplot2::element_text(size = 10),
+ legend.position = "bottom"
+ ) +
+ ggplot2::geom_text(
+ ggplot2::aes(label = sprintf(
+ paste("%.", sprintf("%d", bar_text_n_decimals), "f", sep = ""),
+ round(MSEv, bar_text_n_decimals)
+ )),
+ vjust = -0.35, # This number might need altering for different plots sizes
+ hjust = 1.1, # This number might need altering for different plots sizes
+ color = "black",
+ position = ggplot2::position_dodge(0.9),
+ size = 4
+ )
Main arguments in explain
When using explain, the default behavior is to use all
@@ -726,11 +1028,11 @@
Explaining a fore
it may be of more interesting to explain the impact of each variable,
rather than each lag of each variable. This can be done by setting
group_lags = TRUE.
-
+
# Simulate time series data with AR(1)-structure.
set.seed(1)
data_ts <- data.frame(Y = arima.sim(list(order = c(1, 0, 0), ar = .5), n = 500))
-data_ts <- data.table::as.data.table(data_ts)
+data_ts <- data.table::as.data.table(data_ts)
# Fit an ARIMA(2, 0, 0) model.
arima_model <- arima(data_ts, order = c(2, 0, 0))
@@ -778,8 +1080,8 @@ Explaining a fore
model_ar_temp. This model is then used to make a forecast
of the temperature of the day that comes after the last day in the data,
this forecast starts from index 153.
-
-data <- data.table::as.data.table(airquality)
+
+data <- data.table::as.data.table(airquality)
model_ar_temp <- ar(data$Temp, order = 2)
@@ -812,7 +1114,7 @@ Explaining a fore
separately. Grouping lags may be more interesting to do in a model with
multiple variables, as it is then possible to explain each variable
separately.
-
+
explanation <- explain_forecast(
model = model_ar_temp,
y = data[, "Temp"],
@@ -844,8 +1146,8 @@ Explaining a fore
points, the model is just fit on the 151 first observations, leaving two
observations of Wind to be used as exogenous values during
the prediction phase.
-
-data <- data.table::as.data.table(airquality)
+
+data <- data.table::as.data.table(airquality)
data_fit <- data[seq_len(151), ]
@@ -868,7 +1170,7 @@ Explaining a fore
possible for shapr to not only explain the effect of the
first lag of the exogenous variable, but also the contemporary effect
during the forecasting period.
-
+
explanation <- explain_forecast(
model = model_ar_temp,
y = data_fit[, "Temp"],
@@ -928,7 +1230,7 @@ Combined approach and
"gaussian" when conditioning on respectively 1, 2 and 3
features.
-
+
# Use the combined approach
explanation_combined <- explain(
model = model,
@@ -940,11 +1242,11 @@ Combined approach# Plot the resulting explanations for observations 1 and 6, excluding
# the no-covariate effect
plot(explanation_combined, bar_plot_phi0 = FALSE, index_x_explain = c(1, 6))
-
+
As a second example using "ctree" to conditin on 1 and 2
features, and "empirical" when conditioning on 3
features:
-
+
# Use the combined approach
explanation_combined <- explain(
model = model,
@@ -963,7 +1265,7 @@ Explain groups of features) for intuition and real world
examples. Explaining prediction in terms of groups of features is very
easy using shapr:
-
+
# Define the feature groups
group_list <- list(
A = c("Temp", "Month"),
@@ -981,16 +1283,36 @@ Explain groups of features)
# Prints the group-wise explanations
explanation_group
-#> none A B
-#> 1: 43.09 -29.25 16.0731
-#> 2: 43.09 -15.17 -7.8373
-#> 3: 43.09 -13.07 -10.8778
-#> 4: 43.09 -17.47 0.6653
-#> 5: 43.09 -28.27 3.5289
-#> 6: 43.09 -20.59 -3.3793
+#> none A B
+#> 1: 47.27 -29.588 13.1628
+#> 2: 47.27 -11.834 -15.7011
+#> 3: 47.27 -15.976 -17.5729
+#> 4: 47.27 -25.067 -5.1374
+#> 5: 47.27 -35.848 20.2892
+#> 6: 47.27 -27.257 -8.4830
+#> 7: 47.27 -14.960 -21.3995
+#> 8: 47.27 -18.325 7.3791
+#> 9: 47.27 -23.012 9.6591
+#> 10: 47.27 -16.189 -5.6100
+#> 11: 47.27 -25.607 -10.1334
+#> 12: 47.27 -25.065 -5.1394
+#> 13: 47.27 -25.841 -0.7281
+#> 14: 47.27 -21.518 -13.3293
+#> 15: 47.27 -21.248 -1.3199
+#> 16: 47.27 -13.676 -16.9497
+#> 17: 47.27 -13.899 -14.8890
+#> 18: 47.27 -12.276 -8.2472
+#> 19: 47.27 -13.768 -13.5242
+#> 20: 47.27 -24.866 -10.8744
+#> 21: 47.27 -14.486 -22.7674
+#> 22: 47.27 -4.122 -14.2893
+#> 23: 47.27 -11.218 22.4682
+#> 24: 47.27 -33.002 14.2114
+#> 25: 47.27 -16.251 -8.6796
+#> none A B
# Plots the group-wise explanations
plot(explanation_group, bar_plot_phi0 = TRUE, index_x_explain = c(1, 6))
-
+
Explain custom models
@@ -1044,7 +1366,7 @@ Explain custom modelsget_model_specs function. We do this for the
gbm model class from the gbm package, fitted
to the same airquality data set as used above.
-
+
library(gbm)
#> Loaded gbm 2.1.8.1
@@ -1100,8 +1422,8 @@ Explain custom models
# Plot results
plot(explanation_custom, index_x_explain = c(1, 6))
-
-
+
+
#### Minimal version of the three required model functions ####
@@ -1129,7 +1451,7 @@ Explain custom models
# Plot results
plot(explanation_custom_minimal, index_x_explain = c(1, 6))
-
+
@@ -1163,7 +1485,7 @@ Batch computation
+
explanation_batch <- explain(
model = model,
x_explain = x_explain,
@@ -1197,7 +1519,7 @@ Parallelized computationn_batches
equals some positive integer multiplied by the
number of workers. Below is a basic example of a parallelization with
two workers.
-
+
library(future)
future::plan(multisession, workers = 2)
@@ -1246,7 +1568,7 @@ Progress updatesprogressr
vignette.
A full code example of using progressr with
shapr is shown below:
-
+
library(progressr)
progressr::handlers(global = TRUE)
# If no progression handler is specified, the txtprogressbar is used
@@ -1311,6 +1633,17 @@ References
+Covert, Ian, Scott M Lundberg, and Su-In Lee. 2020. “Understanding
+Global Feature Contributions with Additive Importance Measures.”
+Advances in Neural Information Processing Systems 33: 17212–23.
+
+
+Frye, Christopher, Damien de Mijolla, Tom Begley, Laurence Cowton, Megan
+Stanley, and Ilya Feige. 2021. “Shapley Explainability on the Data
+Manifold.” In International Conference on Learning
+Representations.
+
Hothorn, Torsten, Kurt Hornik, and Achim Zeileis. 2006. “Unbiased
Recursive Partitioning: A Conditional Inference Framework.”
@@ -1334,6 +1667,18 @@ References
+Olsen, Lars Henry Berge, Ingrid Kristine Glad, Martin Jullum, and
+Kjersti Aas. 2022. “Using Shapley Values and Variational
+Autoencoders to Explain Predictive Models with Dependent Mixed
+Features.” Journal of Machine Learning Research 23
+(213): 1–51.
+
+
+———. 2023. “A Comparative Study of Methods for Estimating
+Conditional Shapley Values and When to Use Them.” arXiv
+Preprint arXiv:2305.09536.
+
Redelmeier, Annabelle, Martin Jullum, and Kjersti Aas. 2020.
“Explaining Predictive Models with Mixed Features Using Shapley
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png
index d2ae3cc9d..bfa327d0e 100644
Binary files a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png
new file mode 100644
index 000000000..46339f4b1
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png
new file mode 100644
index 000000000..fb5790961
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png
index b50379788..ad92aaf01 100644
Binary files a/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png
new file mode 100644
index 000000000..021decff0
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png
new file mode 100644
index 000000000..70d491401
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png
new file mode 100644
index 000000000..e137d9551
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png
new file mode 100644
index 000000000..0046444c7
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png differ
diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png
new file mode 100644
index 000000000..0046444c7
Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png differ
diff --git a/index.html b/index.html
index d5b870abc..3086e27cc 100644
--- a/index.html
+++ b/index.html
@@ -174,7 +174,7 @@ Examplelibrary(shapr)
data("airquality")
-data <- data.table::as.data.table(airquality)
+data <- data.table::as.data.table(airquality)
data <- data[complete.cases(data), ]
x_var <- c("Solar.R", "Wind", "Temp", "Month")
diff --git a/pkgdown.yml b/pkgdown.yml
index fe9cff91e..c63ba3767 100644
--- a/pkgdown.yml
+++ b/pkgdown.yml
@@ -3,7 +3,7 @@ pkgdown: 2.0.7
pkgdown_sha: ~
articles:
understanding_shapr: understanding_shapr.html
-last_built: 2023-11-20T16:01Z
+last_built: 2023-12-11T14:36Z
urls:
reference: https://norskregnesentral.github.io/shapr/reference
article: https://norskregnesentral.github.io/shapr/articles
diff --git a/reference/compute_MSEv_eval_crit.html b/reference/compute_MSEv_eval_crit.html
new file mode 100644
index 000000000..f394f01dd
--- /dev/null
+++ b/reference/compute_MSEv_eval_crit.html
@@ -0,0 +1,169 @@
+
+Mean Squared Error of the Contribution Function v(S) — compute_MSEv_eval_crit • shapr
+
+
+
+
+
+
+
+
+
+
+ Mean Squared Error of the Contribution Function v(S)
+ Source: R/finalize_explanation.R
+ compute_MSEv_eval_crit.Rd
+
+
+
+ Function that computes the Mean Squared Error (MSEv) of the contribution function
+v(s) as proposed by Frye et al. (2019) and used by
+Olsen et al. (2022).
+
+
+
+ compute_MSEv_eval_crit(
+ internal,
+ dt_vS,
+ MSEv_uniform_comb_weights,
+ MSEv_skip_empty_full_comb = TRUE
+)
+
+
+
+ Arguments
+ - internal
+List.
+Holds all parameters, data, functions and computed objects used within explain()
+The list contains one or more of the elements parameters, data, objects, output.
- dt_vS
+Data.table of dimension n_combinations times n_explain + 1 containing the contribution function
+estimates. The first column is assumed to be named id_combination and containing the ids of the combinations.
+The last row is assumed to be the full combination, i.e., it contains the predicted responses for the observations
+which are to be explained.
- MSEv_uniform_comb_weights
+Logical. If TRUE (default), then the function weights the combinations
+uniformly when computing the MSEv criterion. If FALSE, then the function use the Shapley kernel weights to
+weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the
+sampling frequency when not all combinations are considered.
- MSEv_skip_empty_full_comb
+Logical. If TRUE (default), we exclude the empty and grand
+combinations/coalitions when computing the MSEv evaluation criterion. This is reasonable as they are identical
+for all methods, i.e., their contribution function is independent of the used method as they are special cases not
+effected by the used method. If FALSE, we include the empty and grand combinations/coalitions. In this situation,
+we also recommend setting MSEv_uniform_comb_weights = TRUE, as otherwise the large weights for the empty and
+grand combinations/coalitions will outweigh all other combinations and make the MSEv criterion uninformative.
+
+ Value
+
+
+List containing:
MSEvA data.table with the overall MSEv evaluation criterion averaged
+over both the combinations/coalitions and observations/explicands. The data.table
+also contains the standard deviation of the MSEv values for each explicand (only averaged over the combinations)
+divided by the square root of the number of explicands.
MSEv_explicandA data.table with the mean squared error for each
+explicand, i.e., only averaged over the combinations/coalitions.
MSEv_combinationA data.table with the mean squared error for each
+combination/coalition, i.e., only averaged over the explicands/observations.
+The data.table also contains the standard deviation of the MSEv values for
+each combination divided by the square root of the number of explicands.
+
+ Details
+ The MSEv evaluation criterion does not rely on access to the true contribution functions nor the
+true Shapley values to be computed. A lower value indicates better approximations, however, the
+scale and magnitude of the MSEv criterion is not directly interpretable in regard to the precision
+of the final estimated Shapley values. Olsen et al. (2022)
+illustrates in Figure 11 a fairly strong linear relationship between the MSEv criterion and the
+MAE between the estimated and true Shapley values in a simulation study. Note that explicands
+refer to the observations whose predictions we are to explain.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/reference/compute_shapley_new.html b/reference/compute_shapley_new.html
index 5e44a3bc7..96820fa22 100644
--- a/reference/compute_shapley_new.html
+++ b/reference/compute_shapley_new.html
@@ -80,7 +80,7 @@ Arguments
Value
-A data.table with shapley values for each test observation.
+A data.table with Shapley values for each test observation.
diff --git a/reference/explain.html b/reference/explain.html
index 2ad8078c5..8058e0b0b 100644
--- a/reference/explain.html
+++ b/reference/explain.html
@@ -79,6 +79,7 @@ Explain the output of machine learning models with more accurately estimated
keep_samp_for_vS = FALSE,
predict_model = NULL,
get_model_specs = NULL,
+ MSEv_uniform_comb_weights = TRUE,
timing = TRUE,
...
)
@@ -195,6 +196,13 @@ Arguments
Can also be used to override the default function for natively supported model classes.
+
MSEv_uniform_comb_weights
+Logical. If TRUE (default), then the function weights the combinations
+uniformly when computing the MSEv criterion. If FALSE, then the function use the Shapley kernel weights to
+weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the
+sampling frequency when not all combinations are considered.
timing
Logical.
Whether the timing of the different parts of the explain() should saved in the model object.
Value
List with the different parameters, data and functions used internally
pred_explain
-Numeric vector with the predictions for the explained observations.
Numeric vector with the predictions for the explained observations
MSEv
+List with the values of the MSEv evaluation criterion for the approach.
shapley_values is a data.table where the number of rows equals
diff --git a/reference/explain_forecast.html b/reference/explain_forecast.html
index 23fa8f411..a2d1b7a98 100644
--- a/reference/explain_forecast.html
+++ b/reference/explain_forecast.html
@@ -358,7 +358,10 @@
Value
List with the different parameters, data and functions used internally
pred_explain
-Numeric vector with the predictions for the explained observations.
Numeric vector with the predictions for the explained observations
MSEv
+List with the values of the MSEv evaluation criterion for the approach.
shapley_values is a data.table where the number of rows equals
@@ -414,7 +417,7 @@
Examples
# Load example data
data("airquality")
-data <- data.table::as.data.table(airquality)
+data <- data.table::as.data.table(airquality)
# Fit an AR(2) model.
model_ar_temp <- ar(data$Temp, order = 2)
diff --git a/reference/finalize_explanation.html b/reference/finalize_explanation.html
index 7d35b7de6..2be5b8279 100644
--- a/reference/finalize_explanation.html
+++ b/reference/finalize_explanation.html
@@ -92,7 +92,10 @@ Value
List with the different parameters, data and functions used internally
pred_explain
-Numeric vector with the predictions for the explained observations.
Numeric vector with the predictions for the explained observations
MSEv
+List with the values of the MSEv evaluation criterion for the approach.
shapley_values is a data.table where the number of rows equals
diff --git a/reference/get_data_specs.html b/reference/get_data_specs.html
index 6e74c496f..ca39f21aa 100644
--- a/reference/get_data_specs.html
+++ b/reference/get_data_specs.html
@@ -107,7 +107,7 @@
Examples
x_train <- head(airquality, -3)
x_explain <- tail(airquality, 3)
# Split data into test- and training data
-x_train <- data.table::as.data.table(head(airquality))
+x_train <- data.table::as.data.table(head(airquality))
x_train[, Temp := as.factor(Temp)]
#> Ozone Solar.R Wind Temp Month Day
#> 1: 41 190 7.4 67 5 1
diff --git a/reference/index.html b/reference/index.html
index c64d637a9..b64fbcf94 100644
--- a/reference/index.html
+++ b/reference/index.html
@@ -104,6 +104,10 @@ All functions plot(<shapr>)
Plot of the Shapley value explanations
+
+
+ Plots of the MSEv Evaluation Criterion
diff --git a/reference/plot_MSEv_eval_crit-1.png b/reference/plot_MSEv_eval_crit-1.png
new file mode 100644
index 000000000..ef2e30987
Binary files /dev/null and b/reference/plot_MSEv_eval_crit-1.png differ
diff --git a/reference/plot_MSEv_eval_crit.html b/reference/plot_MSEv_eval_crit.html
new file mode 100644
index 000000000..5bf0cdb30
--- /dev/null
+++ b/reference/plot_MSEv_eval_crit.html
@@ -0,0 +1,338 @@
+
+Plots of the MSEv Evaluation Criterion — plot_MSEv_eval_crit • shapr
+
+
+
+
+
+
+
+
+
+
+
+
+ Make plots to visualize and compare the MSEv evaluation criterion for a list of
+explain() objects applied to the same data and model. The function creates
+bar plots and line plots with points to illustrate the overall MSEv evaluation
+criterion, but also for each observation/explicand and combination by only averaging over
+the combinations and observations/explicands, respectively.
+
+
+
+ plot_MSEv_eval_crit(
+ explanation_list,
+ index_x_explain = NULL,
+ id_combination = NULL,
+ CI_level = if (length(explanation_list[[1]]$pred_explain) < 20) NULL else 0.95,
+ geom_col_width = 0.9,
+ plot_type = "overall"
+)
+
+
+
+ Arguments
+ - explanation_list
+A list of explain() objects applied to the same data and model.
+If the entries in the list are named, then the function use these names. Otherwise, they default to
+the approach names (with integer suffix for duplicates) for the explanation objects in explanation_list.
- index_x_explain
+Integer vector.
+Which of the test observations to plot. E.g. if you have
+explained 10 observations using explain(), you can generate a plot for the first 5
+observations by setting index_x_explain = 1:5.
- id_combination
+Integer vector. Which of the combinations (coalitions) to plot.
+E.g. if you used n_combinations = 16 in explain(), you can generate a plot for the
+first 5 combinations and the 10th by setting id_combination = c(1:5, 10).
- CI_level
+Positive numeric between zero and one. Default is 0.95 if the number of observations to explain is
+larger than 20, otherwise CI_level = NULL, which removes the confidence intervals. The level of the approximate
+confidence intervals for the overall MSEv and the MSEv_combination. The confidence intervals are based on that
+the MSEv scores are means over the observations/explicands, and that means are approximation normal. Since the
+standard deviations are estimated, we use the quantile t from the T distribution with N_explicands - 1 degrees of
+freedom corresponding to the provided level. Here, N_explicands is the number of observations/explicands.
+MSEv ± tSD(MSEv)/sqrt(N_explicands). Note that the explain() function already scales the standard deviation by
+sqrt(N_explicands), thus, the CI are MSEv ± tMSEv_sd, where the values MSEv and MSEv_sd are extracted from the
+MSEv data.tables in the objects in the explanation_list.
- geom_col_width
+Numeric. Bar width. By default, set to 90% of the ggplot2::resolution() of the data.
- plot_type
+Character vector. The possible options are "overall" (default), "comb", and "explicand".
+If plot_type = "overall", then the plot (one bar plot) associated with the overall MSEv evaluation criterion
+for each method is created, i.e., when averaging over both the combinations/coalitions and observations/explicands.
+If plot_type = "comb", then the plots (one line plot and one bar plot) associated with the MSEv evaluation
+criterion for each combination/coalition are created, i.e., when we only average over the observations/explicands.
+If plot_type = "explicand", then the plots (one line plot and one bar plot) associated with the MSEv evaluation
+criterion for each observations/explicands are created, i.e., when we only average over the combinations/coalitions.
+If plot_type is a vector of one or several of "overall", "comb", and "explicand", then the associated plots are
+created.
+
+ Value
+
+
+Either a single ggplot2::ggplot() object of the MSEv criterion when plot_type = "overall", or a list
+of ggplot2::ggplot() objects based on the plot_type parameter.
+
+
+
+
+ Examples
+ # Load necessary librarieslibrary(xgboost)
+library(data.table)
+library(shapr)
+library(ggplot2)
+
+# Get the data
+data("airquality")
+data <- data.table::as.data.table(airquality)
+data <- data[complete.cases(data), ]
+
+#' Define the features and the response
+x_var <- c("Solar.R", "Wind", "Temp", "Month")
+y_var <- "Ozone"
+
+# Split data into test and training data set
+ind_x_explain <- 1:25
+x_train <- data[-ind_x_explain, ..x_var]
+y_train <- data[-ind_x_explain, get(y_var)]
+x_explain <- data[ind_x_explain, ..x_var]
+
+# Fitting a basic xgboost model to the training data
+model <- xgboost::xgboost(
+ data = as.matrix(x_train),
+ label = y_train,
+ nround = 20,
+ verbose = FALSE
+)
+
+# Specifying the phi_0, i.e. the expected prediction without any features
+prediction_zero <- mean(y_train)
+
+# Independence approach
+explanation_independence <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "independence",
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 2 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Gaussian 1e1 approach
+explanation_gaussian_1e1 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = prediction_zero,
+ n_samples = 1e1
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Gaussian 1e2 approach
+explanation_gaussian_1e2 <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "gaussian",
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# ctree approach
+explanation_ctree <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = "ctree",
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Combined approach
+explanation_combined <- explain(
+ model = model,
+ x_explain = x_explain,
+ x_train = x_train,
+ approach = c("gaussian", "independence", "ctree"),
+ prediction_zero = prediction_zero,
+ n_samples = 1e2
+)
+#> Note: Feature classes extracted from the model contains NA.
+#> Assuming feature classes from the data are correct.
+#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time.
+#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption.
+
+# Create a list of explanations with names
+explanation_list_named <- list(
+ "Ind." = explanation_independence,
+ "Gaus. 1e1" = explanation_gaussian_1e1,
+ "Gaus. 1e2" = explanation_gaussian_1e2,
+ "Ctree" = explanation_ctree,
+ "Combined" = explanation_combined
+)
+
+if (requireNamespace("ggplot2", quietly = TRUE)) {
+ # Create the default MSEv plot where we average over both the combinations and observations
+ # with approximate 95% confidence intervals
+ plot_MSEv_eval_crit(explanation_list_named, CI_level = 0.95, plot_type = "overall")
+
+ # Can also create plots of the MSEv criterion averaged only over the combinations or observations.
+ MSEv_figures <- plot_MSEv_eval_crit(explanation_list_named,
+ CI_level = 0.95,
+ plot_type = c("overall", "comb", "explicand")
+ )
+ MSEv_figures$MSEv_bar
+ MSEv_figures$MSEv_combination_bar
+ MSEv_figures$MSEv_explicand_bar
+
+ # When there are many combinations or observations, then it can be easier to look at line plots
+ MSEv_figures$MSEv_combination_line_point
+ MSEv_figures$MSEv_explicand_line_point
+
+ # We can specify which observations or combinations to plot
+ plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "explicand",
+ index_x_explain = c(1, 3:4, 6),
+ CI_level = 0.95
+ )$MSEv_explicand_bar
+ plot_MSEv_eval_crit(explanation_list_named,
+ plot_type = "comb",
+ id_combination = c(3, 4, 9, 13:15),
+ CI_level = 0.95
+ )$MSEv_combination_bar
+
+ # We can alter the figures if other palette schemes or design is wanted
+ bar_text_n_decimals <- 1
+ MSEv_figures$MSEv_bar +
+ ggplot2::scale_x_discrete(limits = rev(levels(MSEv_figures$MSEv_bar$data$Method))) +
+ ggplot2::coord_flip() +
+ ggplot2::scale_fill_discrete() + #' Default ggplot2 palette
+ ggplot2::theme_minimal() + #' This must be set before the other theme call
+ ggplot2::theme(
+ plot.title = ggplot2::element_text(size = 10),
+ legend.position = "bottom"
+ ) +
+ ggplot2::guides(fill = ggplot2::guide_legend(nrow = 1, ncol = 6)) +
+ ggplot2::geom_text(
+ ggplot2::aes(label = sprintf(
+ paste("%.", sprintf("%d", bar_text_n_decimals), "f", sep = ""),
+ round(MSEv, bar_text_n_decimals)
+ )),
+ vjust = -1.1, # This value must be altered based on the plot dimension
+ hjust = 1.1, # This value must be altered based on the plot dimension
+ color = "black",
+ position = ggplot2::position_dodge(0.9),
+ size = 5
+ )
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/reference/setup.html b/reference/setup.html
index 90328d104..1522ec73a 100644
--- a/reference/setup.html
+++ b/reference/setup.html
@@ -76,6 +76,7 @@ check_setup
seed,
keep_samp_for_vS,
feature_specs,
+ MSEv_uniform_comb_weights = TRUE,
type = "normal",
horizon = NULL,
y = NULL,
@@ -182,6 +183,13 @@ Arguments
+MSEv_uniform_comb_weights
+Logical. If TRUE (default), then the function weights the combinations
+uniformly when computing the MSEv criterion. If FALSE, then the function use the Shapley kernel weights to
+weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the
+sampling frequency when not all combinations are considered.
type
Character.
Either "normal" or "forecast" corresponding to function setup() is called from,
@@ -249,7 +257,7 @@
Arguments
...
-Further arguments passed to approach-specific functions.
Further arguments passed to specific approaches
https://norskregnesentral.github.io/shapr/reference/check_groups.html
+
+ https://norskregnesentral.github.io/shapr/reference/compute_MSEv_eval_crit.html
+
https://norskregnesentral.github.io/shapr/reference/compute_shapley_new.html
@@ -138,6 +141,9 @@
https://norskregnesentral.github.io/shapr/reference/plot.shapr.html
+
+ https://norskregnesentral.github.io/shapr/reference/plot_MSEv_eval_crit.html
+
https://norskregnesentral.github.io/shapr/reference/predict_model.html
+# Simulate time series data with AR(1)-structure. set.seed(1) data_ts <- data.frame(Y = arima.sim(list(order = c(1, 0, 0), ar = .5), n = 500)) -data_ts <- data.table::as.data.table(data_ts) +data_ts <- data.table::as.data.table(data_ts) # Fit an ARIMA(2, 0, 0) model. arima_model <- arima(data_ts, order = c(2, 0, 0)) @@ -778,8 +1080,8 @@Explaining a fore
model_ar_temp. This model is then used to make a forecast of the temperature of the day that comes after the last day in the data, this forecast starts from index 153. --data <- data.table::as.data.table(airquality) ++data <- data.table::as.data.table(airquality) model_ar_temp <- ar(data$Temp, order = 2) @@ -812,7 +1114,7 @@Explaining a fore separately. Grouping lags may be more interesting to do in a model with multiple variables, as it is then possible to explain each variable separately. -
+explanation <- explain_forecast( model = model_ar_temp, y = data[, "Temp"], @@ -844,8 +1146,8 @@Explaining a fore points, the model is just fit on the 151 first observations, leaving two observations of
Windto be used as exogenous values during the prediction phase. --data <- data.table::as.data.table(airquality) ++data <- data.table::as.data.table(airquality) data_fit <- data[seq_len(151), ] @@ -868,7 +1170,7 @@Explaining a fore possible for
shaprto not only explain the effect of the first lag of the exogenous variable, but also the contemporary effect during the forecasting period. -+explanation <- explain_forecast( model = model_ar_temp, y = data_fit[, "Temp"], @@ -928,7 +1230,7 @@Combined approach and
"gaussian"when conditioning on respectively 1, 2 and 3 features. -+-# Use the combined approach explanation_combined <- explain( model = model, @@ -940,11 +1242,11 @@Combined approach# Plot the resulting explanations for observations 1 and 6, excluding # the no-covariate effect plot(explanation_combined, bar_plot_phi0 = FALSE, index_x_explain = c(1, 6))
+
As a second example using
-"ctree"to conditin on 1 and 2 features, and"empirical"when conditioning on 3 features:+# Use the combined approach explanation_combined <- explain( model = model, @@ -963,7 +1265,7 @@Explain groups of features) for intuition and real world examples. Explaining prediction in terms of groups of features is very easy using
shapr: -+-# Define the feature groups group_list <- list( A = c("Temp", "Month"), @@ -981,16 +1283,36 @@Explain groups of features) # Prints the group-wise explanations explanation_group -#> none A B -#> 1: 43.09 -29.25 16.0731 -#> 2: 43.09 -15.17 -7.8373 -#> 3: 43.09 -13.07 -10.8778 -#> 4: 43.09 -17.47 0.6653 -#> 5: 43.09 -28.27 3.5289 -#> 6: 43.09 -20.59 -3.3793 +#> none A B +#> 1: 47.27 -29.588 13.1628 +#> 2: 47.27 -11.834 -15.7011 +#> 3: 47.27 -15.976 -17.5729 +#> 4: 47.27 -25.067 -5.1374 +#> 5: 47.27 -35.848 20.2892 +#> 6: 47.27 -27.257 -8.4830 +#> 7: 47.27 -14.960 -21.3995 +#> 8: 47.27 -18.325 7.3791 +#> 9: 47.27 -23.012 9.6591 +#> 10: 47.27 -16.189 -5.6100 +#> 11: 47.27 -25.607 -10.1334 +#> 12: 47.27 -25.065 -5.1394 +#> 13: 47.27 -25.841 -0.7281 +#> 14: 47.27 -21.518 -13.3293 +#> 15: 47.27 -21.248 -1.3199 +#> 16: 47.27 -13.676 -16.9497 +#> 17: 47.27 -13.899 -14.8890 +#> 18: 47.27 -12.276 -8.2472 +#> 19: 47.27 -13.768 -13.5242 +#> 20: 47.27 -24.866 -10.8744 +#> 21: 47.27 -14.486 -22.7674 +#> 22: 47.27 -4.122 -14.2893 +#> 23: 47.27 -11.218 22.4682 +#> 24: 47.27 -33.002 14.2114 +#> 25: 47.27 -16.251 -8.6796 +#> none A B # Plots the group-wise explanations plot(explanation_group, bar_plot_phi0 = TRUE, index_x_explain = c(1, 6))
+
Explain custom models @@ -1044,7 +1366,7 @@
Explain custom modelsget_model_specs function. We do this for the
gbmmodel class from thegbmpackage, fitted to the same airquality data set as used above. -+-library(gbm) #> Loaded gbm 2.1.8.1 @@ -1100,8 +1422,8 @@Explain custom models # Plot results plot(explanation_custom, index_x_explain = c(1, 6))
-
@@ -1163,7 +1485,7 @@++
-#### Minimal version of the three required model functions #### @@ -1129,7 +1451,7 @@Explain custom models # Plot results plot(explanation_custom_minimal, index_x_explain = c(1, 6))
+
Batch computation
+explanation_batch <- explain( model = model, x_explain = x_explain, @@ -1197,7 +1519,7 @@equals some positive integer multiplied by the number of workers. Below is a basic example of a parallelization with two workers. -Parallelized computationn_batches
+@@ -195,6 +196,13 @@library(future) future::plan(multisession, workers = 2) @@ -1246,7 +1568,7 @@vignette. A full code example of usingProgress updatesprogressr
progressrwithshapris shown below: -diff --git a/reference/explain.html b/reference/explain.html index 2ad8078c5..8058e0b0b 100644 --- a/reference/explain.html +++ b/reference/explain.html @@ -79,6 +79,7 @@++library(progressr) progressr::handlers(global = TRUE) # If no progression handler is specified, the txtprogressbar is used @@ -1311,6 +1633,17 @@References +Covert, Ian, Scott M Lundberg, and Su-In Lee. 2020. “Understanding +Global Feature Contributions with Additive Importance Measures.” +Advances in Neural Information Processing Systems 33: 17212–23. +
+Frye, Christopher, Damien de Mijolla, Tom Begley, Laurence Cowton, Megan +Stanley, and Ilya Feige. 2021. “Shapley Explainability on the Data +Manifold.” In International Conference on Learning +Representations. +Hothorn, Torsten, Kurt Hornik, and Achim Zeileis. 2006. “Unbiased Recursive Partitioning: A Conditional Inference Framework.” @@ -1334,6 +1667,18 @@+References +Olsen, Lars Henry Berge, Ingrid Kristine Glad, Martin Jullum, and +Kjersti Aas. 2022. “Using Shapley Values and Variational +Autoencoders to Explain Predictive Models with Dependent Mixed +Features.” Journal of Machine Learning Research 23 +(213): 1–51. +
+———. 2023. “A Comparative Study of Methods for Estimating +Conditional Shapley Values and When to Use Them.” arXiv +Preprint arXiv:2305.09536. +Redelmeier, Annabelle, Martin Jullum, and Kjersti Aas. 2020. “Explaining Predictive Models with Mixed Features Using Shapley diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png index d2ae3cc9d..bfa327d0e 100644 Binary files a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-1.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png new file mode 100644 index 000000000..46339f4b1 Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-2.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png new file mode 100644 index 000000000..fb5790961 Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-12-3.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png index b50379788..ad92aaf01 100644 Binary files a/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-13-1.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png new file mode 100644 index 000000000..021decff0 Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-14-1.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png new file mode 100644 index 000000000..70d491401 Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-20-1.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png new file mode 100644 index 000000000..e137d9551 Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-22-1.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png new file mode 100644 index 000000000..0046444c7 Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-1.png differ diff --git a/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png new file mode 100644 index 000000000..0046444c7 Binary files /dev/null and b/articles/understanding_shapr_files/figure-html/unnamed-chunk-23-2.png differ diff --git a/index.html b/index.html index d5b870abc..3086e27cc 100644 --- a/index.html +++ b/index.html @@ -174,7 +174,7 @@Examplelibrary(shapr)
data("airquality") -data <- data.table::as.data.table(airquality) +data <- data.table::as.data.table(airquality) data <- data[complete.cases(data), ] x_var <- c("Solar.R", "Wind", "Temp", "Month") diff --git a/pkgdown.yml b/pkgdown.yml index fe9cff91e..c63ba3767 100644 --- a/pkgdown.yml +++ b/pkgdown.yml @@ -3,7 +3,7 @@ pkgdown: 2.0.7 pkgdown_sha: ~ articles: understanding_shapr: understanding_shapr.html -last_built: 2023-11-20T16:01Z +last_built: 2023-12-11T14:36Z urls: reference: https://norskregnesentral.github.io/shapr/reference article: https://norskregnesentral.github.io/shapr/articles diff --git a/reference/compute_MSEv_eval_crit.html b/reference/compute_MSEv_eval_crit.html new file mode 100644 index 000000000..f394f01dd --- /dev/null +++ b/reference/compute_MSEv_eval_crit.html @@ -0,0 +1,169 @@ + +Mean Squared Error of the Contribution Function v(S) — compute_MSEv_eval_crit • shapr + + +++ + + + + + + + diff --git a/reference/compute_shapley_new.html b/reference/compute_shapley_new.html index 5e44a3bc7..96820fa22 100644 --- a/reference/compute_shapley_new.html +++ b/reference/compute_shapley_new.html @@ -80,7 +80,7 @@+ + + + ++ + +++ +++ +Mean Squared Error of the Contribution Function
+ Source:v(S)R/finalize_explanation.R++compute_MSEv_eval_crit.Rd++ +Function that computes the Mean Squared Error (MSEv) of the contribution function +v(s) as proposed by Frye et al. (2019) and used by +Olsen et al. (2022).
+++ ++compute_MSEv_eval_crit( + internal, + dt_vS, + MSEv_uniform_comb_weights, + MSEv_skip_empty_full_comb = TRUE +)++Arguments
+
- internal
+- + + +
List. +Holds all parameters, data, functions and computed objects used within
explain()+The list contains one or more of the elementsparameters,data,objects,output.- dt_vS
+- + + +
Data.table of dimension
n_combinationstimesn_explain + 1containing the contribution function +estimates. The first column is assumed to be namedid_combinationand containing the ids of the combinations. +The last row is assumed to be the full combination, i.e., it contains the predicted responses for the observations +which are to be explained.- MSEv_uniform_comb_weights
+- + + +
Logical. If
TRUE(default), then the function weights the combinations +uniformly when computing the MSEv criterion. IfFALSE, then the function use the Shapley kernel weights to +weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the +sampling frequency when not all combinations are considered.- MSEv_skip_empty_full_comb
+- + +
Logical. If
TRUE(default), we exclude the empty and grand +combinations/coalitions when computing the MSEv evaluation criterion. This is reasonable as they are identical +for all methods, i.e., their contribution function is independent of the used method as they are special cases not +effected by the used method. IfFALSE, we include the empty and grand combinations/coalitions. In this situation, +we also recommend settingMSEv_uniform_comb_weights = TRUE, as otherwise the large weights for the empty and +grand combinations/coalitions will outweigh all other combinations and make the MSEv criterion uninformative.++Value
+ + +List containing:
- +
MSEv- + +
A
data.tablewith the overall MSEv evaluation criterion averaged +over both the combinations/coalitions and observations/explicands. Thedata.table+also contains the standard deviation of the MSEv values for each explicand (only averaged over the combinations) +divided by the square root of the number of explicands.- +
MSEv_explicand- + +
A
data.tablewith the mean squared error for each +explicand, i.e., only averaged over the combinations/coalitions.- +
MSEv_combination- + + +
A
data.tablewith the mean squared error for each +combination/coalition, i.e., only averaged over the explicands/observations. +Thedata.tablealso contains the standard deviation of the MSEv values for +each combination divided by the square root of the number of explicands.++ + +Details
+The MSEv evaluation criterion does not rely on access to the true contribution functions nor the +true Shapley values to be computed. A lower value indicates better approximations, however, the +scale and magnitude of the MSEv criterion is not directly interpretable in regard to the precision +of the final estimated Shapley values. Olsen et al. (2022) +illustrates in Figure 11 a fairly strong linear relationship between the MSEv criterion and the +MAE between the estimated and true Shapley values in a simulation study. Note that explicands +refer to the observations whose predictions we are to explain.
+Arguments
Value
-A
+data.tablewith shapley values for each test observation.A
data.tablewith Shapley values for each test observation.Explain the output of machine learning models with more accurately estimated keep_samp_for_vS = FALSE, predict_model = NULL, get_model_specs = NULL, + MSEv_uniform_comb_weights = TRUE, timing = TRUE, ... )
Arguments
Can also be used to override the default function for natively supported model classes. +MSEv_uniform_comb_weights ++ + Logical. If
TRUE(default), then the function weights the combinations +uniformly when computing the MSEv criterion. IfFALSE, then the function use the Shapley kernel weights to +weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the +sampling frequency when not all combinations are considered.timing @@ -314,7 +322,10 @@ Logical. Whether the timing of the different parts of the
explain()should saved in the model object.Value
List with the different parameters, data and functions used internally
pred_explain -+ Numeric vector with the predictions for the explained observations.
+ + Numeric vector with the predictions for the explained observations
MSEv +List with the values of the MSEv evaluation criterion for the approach.
shapley_valuesis a data.table where the number of rows equals diff --git a/reference/explain_forecast.html b/reference/explain_forecast.html index 23fa8f411..a2d1b7a98 100644 --- a/reference/explain_forecast.html +++ b/reference/explain_forecast.html @@ -358,7 +358,10 @@Value
List with the different parameters, data and functions used internally
pred_explain -+ Numeric vector with the predictions for the explained observations.
+ + Numeric vector with the predictions for the explained observations
MSEv +List with the values of the MSEv evaluation criterion for the approach.
shapley_valuesis a data.table where the number of rows equals @@ -414,7 +417,7 @@Examples
# Load example data data("airquality") -data <- data.table::as.data.table(airquality) +data <- data.table::as.data.table(airquality) # Fit an AR(2) model. model_ar_temp <- ar(data$Temp, order = 2) diff --git a/reference/finalize_explanation.html b/reference/finalize_explanation.html index 7d35b7de6..2be5b8279 100644 --- a/reference/finalize_explanation.html +++ b/reference/finalize_explanation.html @@ -92,7 +92,10 @@Value
List with the different parameters, data and functions used internally
pred_explain -+ Numeric vector with the predictions for the explained observations.
+ + Numeric vector with the predictions for the explained observations
MSEv +List with the values of the MSEv evaluation criterion for the approach.
shapley_valuesis a data.table where the number of rows equals diff --git a/reference/get_data_specs.html b/reference/get_data_specs.html index 6e74c496f..ca39f21aa 100644 --- a/reference/get_data_specs.html +++ b/reference/get_data_specs.html @@ -107,7 +107,7 @@Examples
x_train <- head(airquality, -3) x_explain <- tail(airquality, 3) # Split data into test- and training data -x_train <- data.table::as.data.table(head(airquality)) +x_train <- data.table::as.data.table(head(airquality)) x_train[, Temp := as.factor(Temp)] #> Ozone Solar.R Wind Temp Month Day #> 1: 41 190 7.4 67 5 1 diff --git a/reference/index.html b/reference/index.html index c64d637a9..b64fbcf94 100644 --- a/reference/index.html +++ b/reference/index.html @@ -104,6 +104,10 @@All functions
plot(<shapr>)+ Plot of the Shapley value explanations
+ + +Plots of the MSEv Evaluation Criterion
diff --git a/reference/plot_MSEv_eval_crit-1.png b/reference/plot_MSEv_eval_crit-1.png new file mode 100644 index 000000000..ef2e30987 Binary files /dev/null and b/reference/plot_MSEv_eval_crit-1.png differ diff --git a/reference/plot_MSEv_eval_crit.html b/reference/plot_MSEv_eval_crit.html new file mode 100644 index 000000000..5bf0cdb30 --- /dev/null +++ b/reference/plot_MSEv_eval_crit.html @@ -0,0 +1,338 @@ + + Plots of the MSEv Evaluation Criterion — plot_MSEv_eval_crit • shapr + + +++ + + + + + + + diff --git a/reference/setup.html b/reference/setup.html index 90328d104..1522ec73a 100644 --- a/reference/setup.html +++ b/reference/setup.html @@ -76,6 +76,7 @@+ + + + ++ + ++ + ++ +++ +Make plots to visualize and compare the MSEv evaluation criterion for a list of +
+explain()objects applied to the same data and model. The function creates +bar plots and line plots with points to illustrate the overall MSEv evaluation +criterion, but also for each observation/explicand and combination by only averaging over +the combinations and observations/explicands, respectively.++ ++plot_MSEv_eval_crit( + explanation_list, + index_x_explain = NULL, + id_combination = NULL, + CI_level = if (length(explanation_list[[1]]$pred_explain) < 20) NULL else 0.95, + geom_col_width = 0.9, + plot_type = "overall" +)++Arguments
+
- explanation_list
+- + + +
A list of
explain()objects applied to the same data and model. +If the entries in the list are named, then the function use these names. Otherwise, they default to +the approach names (with integer suffix for duplicates) for the explanation objects inexplanation_list.- index_x_explain
+- + + +
Integer vector. +Which of the test observations to plot. E.g. if you have +explained 10 observations using
explain(), you can generate a plot for the first 5 +observations by settingindex_x_explain = 1:5.- id_combination
+- + + +
Integer vector. Which of the combinations (coalitions) to plot. +E.g. if you used
n_combinations = 16inexplain(), you can generate a plot for the +first 5 combinations and the 10th by settingid_combination = c(1:5, 10).- CI_level
+- + + +
Positive numeric between zero and one. Default is
0.95if the number of observations to explain is +larger than 20, otherwiseCI_level = NULL, which removes the confidence intervals. The level of the approximate +confidence intervals for the overall MSEv and the MSEv_combination. The confidence intervals are based on that +the MSEv scores are means over the observations/explicands, and that means are approximation normal. Since the +standard deviations are estimated, we use the quantile t from the T distribution with N_explicands - 1 degrees of +freedom corresponding to the provided level. Here, N_explicands is the number of observations/explicands. +MSEv ± tSD(MSEv)/sqrt(N_explicands). Note that theexplain()function already scales the standard deviation by +sqrt(N_explicands), thus, the CI are MSEv ± tMSEv_sd, where the values MSEv and MSEv_sd are extracted from the +MSEv data.tables in the objects in theexplanation_list.- geom_col_width
+- + + +
Numeric. Bar width. By default, set to 90% of the
ggplot2::resolution()of the data.- plot_type
+- + +
Character vector. The possible options are "overall" (default), "comb", and "explicand". +If
plot_type = "overall", then the plot (one bar plot) associated with the overall MSEv evaluation criterion +for each method is created, i.e., when averaging over both the combinations/coalitions and observations/explicands. +Ifplot_type = "comb", then the plots (one line plot and one bar plot) associated with the MSEv evaluation +criterion for each combination/coalition are created, i.e., when we only average over the observations/explicands. +Ifplot_type = "explicand", then the plots (one line plot and one bar plot) associated with the MSEv evaluation +criterion for each observations/explicands are created, i.e., when we only average over the combinations/coalitions. +Ifplot_typeis a vector of one or several of "overall", "comb", and "explicand", then the associated plots are +created.++ + +Value
+ + +Either a single
+ggplot2::ggplot()object of the MSEv criterion whenplot_type = "overall", or a list +ofggplot2::ggplot()objects based on theplot_typeparameter.++Examples
++# Load necessary librarieslibrary(xgboost) +library(data.table) +library(shapr) +library(ggplot2) + +# Get the data +data("airquality") +data <- data.table::as.data.table(airquality) +data <- data[complete.cases(data), ] + +#' Define the features and the response +x_var <- c("Solar.R", "Wind", "Temp", "Month") +y_var <- "Ozone" + +# Split data into test and training data set +ind_x_explain <- 1:25 +x_train <- data[-ind_x_explain, ..x_var] +y_train <- data[-ind_x_explain, get(y_var)] +x_explain <- data[ind_x_explain, ..x_var] + +# Fitting a basic xgboost model to the training data +model <- xgboost::xgboost( + data = as.matrix(x_train), + label = y_train, + nround = 20, + verbose = FALSE +) + +# Specifying the phi_0, i.e. the expected prediction without any features +prediction_zero <- mean(y_train) + +# Independence approach +explanation_independence <- explain( + model = model, + x_explain = x_explain, + x_train = x_train, + approach = "independence", + prediction_zero = prediction_zero, + n_samples = 1e2 +) +#> Note: Feature classes extracted from the model contains NA. +#> Assuming feature classes from the data are correct. +#> Setting parameter 'n_batches' to 2 as a fair trade-off between memory consumption and computation time. +#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption. + +# Gaussian 1e1 approach +explanation_gaussian_1e1 <- explain( + model = model, + x_explain = x_explain, + x_train = x_train, + approach = "gaussian", + prediction_zero = prediction_zero, + n_samples = 1e1 +) +#> Note: Feature classes extracted from the model contains NA. +#> Assuming feature classes from the data are correct. +#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time. +#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption. + +# Gaussian 1e2 approach +explanation_gaussian_1e2 <- explain( + model = model, + x_explain = x_explain, + x_train = x_train, + approach = "gaussian", + prediction_zero = prediction_zero, + n_samples = 1e2 +) +#> Note: Feature classes extracted from the model contains NA. +#> Assuming feature classes from the data are correct. +#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time. +#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption. + +# ctree approach +explanation_ctree <- explain( + model = model, + x_explain = x_explain, + x_train = x_train, + approach = "ctree", + prediction_zero = prediction_zero, + n_samples = 1e2 +) +#> Note: Feature classes extracted from the model contains NA. +#> Assuming feature classes from the data are correct. +#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time. +#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption. + +# Combined approach +explanation_combined <- explain( + model = model, + x_explain = x_explain, + x_train = x_train, + approach = c("gaussian", "independence", "ctree"), + prediction_zero = prediction_zero, + n_samples = 1e2 +) +#> Note: Feature classes extracted from the model contains NA. +#> Assuming feature classes from the data are correct. +#> Setting parameter 'n_batches' to 10 as a fair trade-off between memory consumption and computation time. +#> Reducing 'n_batches' typically reduces the computation time at the cost of increased memory consumption. + +# Create a list of explanations with names +explanation_list_named <- list( + "Ind." = explanation_independence, + "Gaus. 1e1" = explanation_gaussian_1e1, + "Gaus. 1e2" = explanation_gaussian_1e2, + "Ctree" = explanation_ctree, + "Combined" = explanation_combined +) + +if (requireNamespace("ggplot2", quietly = TRUE)) { + # Create the default MSEv plot where we average over both the combinations and observations + # with approximate 95% confidence intervals + plot_MSEv_eval_crit(explanation_list_named, CI_level = 0.95, plot_type = "overall") + + # Can also create plots of the MSEv criterion averaged only over the combinations or observations. + MSEv_figures <- plot_MSEv_eval_crit(explanation_list_named, + CI_level = 0.95, + plot_type = c("overall", "comb", "explicand") + ) + MSEv_figures$MSEv_bar + MSEv_figures$MSEv_combination_bar + MSEv_figures$MSEv_explicand_bar + + # When there are many combinations or observations, then it can be easier to look at line plots + MSEv_figures$MSEv_combination_line_point + MSEv_figures$MSEv_explicand_line_point + + # We can specify which observations or combinations to plot + plot_MSEv_eval_crit(explanation_list_named, + plot_type = "explicand", + index_x_explain = c(1, 3:4, 6), + CI_level = 0.95 + )$MSEv_explicand_bar + plot_MSEv_eval_crit(explanation_list_named, + plot_type = "comb", + id_combination = c(3, 4, 9, 13:15), + CI_level = 0.95 + )$MSEv_combination_bar + + # We can alter the figures if other palette schemes or design is wanted + bar_text_n_decimals <- 1 + MSEv_figures$MSEv_bar + + ggplot2::scale_x_discrete(limits = rev(levels(MSEv_figures$MSEv_bar$data$Method))) + + ggplot2::coord_flip() + + ggplot2::scale_fill_discrete() + #' Default ggplot2 palette + ggplot2::theme_minimal() + #' This must be set before the other theme call + ggplot2::theme( + plot.title = ggplot2::element_text(size = 10), + legend.position = "bottom" + ) + + ggplot2::guides(fill = ggplot2::guide_legend(nrow = 1, ncol = 6)) + + ggplot2::geom_text( + ggplot2::aes(label = sprintf( + paste("%.", sprintf("%d", bar_text_n_decimals), "f", sep = ""), + round(MSEv, bar_text_n_decimals) + )), + vjust = -1.1, # This value must be altered based on the plot dimension + hjust = 1.1, # This value must be altered based on the plot dimension + color = "black", + position = ggplot2::position_dodge(0.9), + size = 5 + ) +} +check_setup
seed, keep_samp_for_vS, feature_specs, + MSEv_uniform_comb_weights = TRUE, type = "normal", horizon = NULL, y = NULL, @@ -182,6 +183,13 @@Arguments
+MSEv_uniform_comb_weights ++ + Logical. If
TRUE(default), then the function weights the combinations +uniformly when computing the MSEv criterion. IfFALSE, then the function use the Shapley kernel weights to +weight the combinations when computing the MSEv criterion. Note that the Shapley kernel weights are replaced by the +sampling frequency when not all combinations are considered.type Character. Either "normal" or "forecast" corresponding to function
setup()is called from, @@ -249,7 +257,7 @@Arguments
... -+ Further arguments passed to
approach-specific functions.diff --git a/sitemap.xml b/sitemap.xml index 083683866..c012c07bd 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -39,6 +39,9 @@ Further arguments passed to specific approaches
+ https://norskregnesentral.github.io/shapr/reference/check_groups.html + https://norskregnesentral.github.io/shapr/reference/compute_MSEv_eval_crit.html +@@ -138,6 +141,9 @@ https://norskregnesentral.github.io/shapr/reference/compute_shapley_new.html + https://norskregnesentral.github.io/shapr/reference/plot.shapr.html + https://norskregnesentral.github.io/shapr/reference/plot_MSEv_eval_crit.html +https://norskregnesentral.github.io/shapr/reference/predict_model.html