The hardware and bandwidth for this mirror is donated by METANET, the Webhosting and Full Service-Cloud Provider.
If you wish to report a bug, or if you are interested in having us mirror your free-software or open-source project, please feel free to contact us at mirror[@]metanet.ch.

ranger: Binary Classification

# nolint start
library(mlexperiments)
library(mllrnrs)

See https://github.com/kapsner/mllrnrs/blob/main/R/learner_ranger.R for implementation details.

Preprocessing

Import and Prepare Data

library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
  data.table::as.data.table() |>
  na.omit()
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"

General Configurations

seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
  # on cran
  ncores <- 2L
} else {
  ncores <- ifelse(
    test = parallel::detectCores() > 4,
    yes = 4L,
    no = ifelse(
      test = parallel::detectCores() < 2L,
      yes = 1L,
      no = parallel::detectCores()
    )
  )
}
options("mlexperiments.bayesian.max_init" = 4L)

Generate Training- and Test Data

data_split <- splitTools::partition(
  y = dataset[, get(target_col)],
  p = c(train = 0.7, test = 0.3),
  type = "stratified",
  seed = seed
)

train_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- dataset[data_split$train, get(target_col)]


test_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- dataset[data_split$test, get(target_col)]

Generate Training Data Folds

fold_list <- splitTools::create_folds(
  y = train_y,
  k = 3,
  type = "stratified",
  seed = seed
)

Experiments

Prepare Experiments

# required learner arguments, not optimized
learner_args <- list(probability = TRUE)

# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- list(prob = TRUE, positive = "malignant")
performance_metric <- mlexperiments::metric("AUC")
performance_metric_args <- list(positive = "malignant", negative = "benign")
return_models <- FALSE

# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
  num.trees = seq(500, 1000, 500),
  mtry = seq(2, 6, 2),
  min.node.size = seq(2, 9, 4),
  max.depth = seq(1, 9, 4),
  sample.fraction = seq(0.5, 0.8, 0.3)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
  set.seed(123)
  sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
  parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}

# required for bayesian optimization
parameter_bounds <- list(
  num.trees = c(100L, 1000L),
  mtry = c(2L, 9L),
  min.node.size = c(2L, 20L),
  max.depth = c(1L, 40L),
  sample.fraction = c(0.3, 1.)
)
optim_args <- list(
  n_iter = ncores,
  kappa = 3.5,
  acq = "ucb"
)

Hyperparameter Tuning

tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "grid",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_grid <- tuner$execute(k = 3)
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [================================>------------------------------------------------------------------------------] 3/10 ( 30%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.

head(tuner_results_grid)
#>    setting_id metric_optim_mean num.trees  mtry min.node.size max.depth sample.fraction probability
#>         <int>             <num>     <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:          1        0.04398668       500     2             6         9             0.5        TRUE
#> 2:          2        0.04189024       500     4             2         5             0.8        TRUE
#> 3:          3        0.04398668      1000     2             2         5             0.5        TRUE
#> 4:          4        0.04189024       500     2             6         9             0.8        TRUE
#> 5:          5        0.04613602      1000     6             2         1             0.8        TRUE
#> 6:          6        0.04398668      1000     2             2         5             0.8        TRUE

Bayesian Optimization

tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "bayesian",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds

tuner$learner_args <- learner_args
tuner$optim_args <- optim_args

tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.169  Round = 1   num.trees = 1000.0000   mtry = 2.0000   min.node.size = 2.0000  max.depth = 5.0000  sample.fraction = 0.5000    Value = -0.04398668
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.116  Round = 2   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.04189024
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.107  Round = 3   num.trees = 500.0000    mtry = 4.0000   min.node.size = 2.0000  max.depth = 5.0000  sample.fraction = 0.8000    Value = -0.04189024
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.169  Round = 4   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.04189024
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.17   Round = 5   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.174  Round = 6   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.18   Round = 7   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.183  Round = 8   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Best Parameters Found:
#> Round = 2    num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.04189024

head(tuner_results_bayesian)
#>    setting_id num.trees  mtry min.node.size max.depth sample.fraction       Value probability metric_optim_mean
#>         <int>     <num> <num>         <num>     <num>           <num>       <num>      <lgcl>             <num>
#> 1:          1      1000     2             2         5       0.5000000 -0.04398668        TRUE        0.04398668
#> 2:          2       500     2             2         9       0.5000000 -0.04189024        TRUE        0.04189024
#> 3:          3       500     4             2         5       0.8000000 -0.04189024        TRUE        0.04189024
#> 4:          4      1000     4             6         9       0.8000000 -0.04189024        TRUE        0.04189024
#> 5:          5       952     7             5        21       0.8216647 -0.04817972        TRUE        0.04817972
#> 6:          6       952     7             5        21       0.8216647 -0.04817972        TRUE        0.04817972

k-Fold Cross Validation

validator <- mlexperiments::MLCrossValidation$new(
  learner = mllrnrs::LearnerRanger$new(),
  fold_list = fold_list,
  ncores = ncores,
  seed = seed
)

validator$learner_args <- tuner$results$best.setting[-1]

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3

head(validator_results)
#>      fold performance  mtry min.node.size max.depth sample.fraction probability
#>    <char>       <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:  Fold1   0.9925861     2             2         9             0.5        TRUE
#> 2:  Fold2   0.9935853     2             2         9             0.5        TRUE
#> 3:  Fold3   0.9888393     2             2         9             0.5        TRUE

Nested Cross Validation

validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "grid",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.

head(validator_results)
#>      fold performance num.trees  mtry min.node.size max.depth sample.fraction probability
#>    <char>       <num>     <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:  Fold1   0.9931156       500     2             6         9             0.8        TRUE
#> 2:  Fold2   0.9935853      1000     2             2         5             0.5        TRUE
#> 3:  Fold3   0.9886676       500     2             6         9             0.5        TRUE

Inner Bayesian Optimization

validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "bayesian",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = 312
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"


validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.086  Round = 1   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.05331217
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.14   Round = 2   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05645683
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 3   num.trees = 500.0000    mtry = 2.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05645683
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.104  Round = 4   num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.05960148
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.116  Round = 5   num.trees = 807.0000    mtry = 9.0000   min.node.size = 2.0000  max.depth = 10.0000 sample.fraction = 0.4920744 Value = -0.05645683
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.105  Round = 6   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.05960148
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.095  Round = 7   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.05960148
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.099  Round = 8   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.05960148
#>
#>  Best Parameters Found:
#> Round = 1    num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.05331217
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.081  Round = 1   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.05031447
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.133  Round = 2   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05031447
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.083  Round = 3   num.trees = 500.0000    mtry = 2.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05345912
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.095  Round = 4   num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04716981
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.116  Round = 5   num.trees = 826.0000    mtry = 9.0000   min.node.size = 15.0000 max.depth = 40.0000 sample.fraction = 1.0000    Value = -0.05660377
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.118  Round = 6   num.trees = 1000.0000   mtry = 9.0000   min.node.size = 2.0000  max.depth = 25.0000 sample.fraction = 0.3404917 Value = -0.05660377
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.102  Round = 7   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.04716981
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 8   num.trees = 913.0000    mtry = 5.0000   min.node.size = 16.0000 max.depth = 1.0000  sample.fraction = 0.3000    Value = -0.04716981
#>
#>  Best Parameters Found:
#> Round = 4    num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04716981
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.091  Round = 1   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.04414495
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.13   Round = 2   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.04731956
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.098  Round = 3   num.trees = 500.0000    mtry = 2.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.04414495
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 4   num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04103025
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 5   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.04731956
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.065  Round = 6   num.trees = 253.0000    mtry = 8.0000   min.node.size = 12.0000 max.depth = 15.0000 sample.fraction = 0.4823285 Value = -0.0441749
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.064  Round = 7   num.trees = 245.0000    mtry = 7.0000   min.node.size = 18.0000 max.depth = 14.0000 sample.fraction = 0.3033942 Value = -0.04103025
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.131  Round = 8   num.trees = 994.0000    mtry = 6.0000   min.node.size = 15.0000 max.depth = 15.0000 sample.fraction = 0.5716413 Value = -0.0441749
#>
#>  Best Parameters Found:
#> Round = 4    num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04103025

head(validator_results)
#>      fold performance num.trees  mtry min.node.size max.depth sample.fraction probability
#>    <char>       <num>     <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:  Fold1   0.9932921       500     2             2         9             0.5        TRUE
#> 2:  Fold2   0.9913315      1000     6             2         1             0.8        TRUE
#> 3:  Fold3   0.9876374      1000     6             2         1             0.8        TRUE

Holdout Test Dataset Performance

Predict Outcome in Holdout Test Dataset

preds_ranger <- mlexperiments::predictions(
  object = validator,
  newdata = test_x
)

Evaluate Performance on Holdout Test Dataset

perf_ranger <- mlexperiments::performance(
  object = validator,
  prediction_results = preds_ranger,
  y_ground_truth = test_y,
  type = "binary"
)
perf_ranger
#>     model performance       AUC      Brier BrierScaled       BAC    TP    TN    FP    FN       TPR       TNR        FPR        FNR       PPV
#>    <char>       <num>     <num>      <num>       <num>     <num> <int> <int> <int> <int>     <num>     <num>      <num>      <num>     <num>
#> 1:  Fold1   0.9888060 0.9888060 0.04307263   0.8105483 0.9508706    66   132     2     6 0.9166667 0.9850746 0.01492537 0.08333333 0.9705882
#> 2:  Fold2   0.9795813 0.9795813 0.07807664   0.6565858 0.9257877    64   129     5     8 0.8888889 0.9626866 0.03731343 0.11111111 0.9275362
#> 3:  Fold3   0.9795813 0.9795813 0.07868587   0.6539061 0.9220564    64   128     6     8 0.8888889 0.9552239 0.04477612 0.11111111 0.9142857
#>          NPV        FDR       MCC        F1     GMEAN       GPR       ACC       MMCE        BER
#>        <num>      <num>     <num>     <num>     <num>     <num>     <num>      <num>      <num>
#> 1: 0.9565217 0.02941176 0.9143377 0.9428571 0.9502553 0.9432422 0.9611650 0.03883495 0.04912935
#> 2: 0.9416058 0.07246377 0.8603139 0.9078014 0.9250521 0.9080070 0.9368932 0.06310680 0.07421227
#> 3: 0.9411765 0.08571429 0.8497685 0.9014085 0.9214597 0.9014979 0.9320388 0.06796117 0.07794362

These binaries (installable software) and packages are in development.
They may not be fully stable and should be used with caution. We make no claims about them.