# nolint start
library(mlexperiments)
library(mllrnrs)The hardware and bandwidth for this mirror is donated by METANET, the Webhosting and Full Service-Cloud Provider.
If you wish to report a bug, or if you are interested in having us mirror your free-software or open-source project, please feel free to contact us at mirror[@]metanet.ch.
# nolint start
library(mlexperiments)
library(mllrnrs)See https://github.com/kapsner/mllrnrs/blob/main/R/learner_xgboost.R for implementation details.
library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
data.table::as.data.table() |>
na.omit()
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
# on cran
ncores <- 2L
} else {
ncores <- ifelse(
test = parallel::detectCores() > 4,
yes = 4L,
no = ifelse(
test = parallel::detectCores() < 2L,
yes = 1L,
no = parallel::detectCores()
)
)
}
options("mlexperiments.bayesian.max_init" = 4L)
options("mlexperiments.optim.xgb.nrounds" = 20L)
options("mlexperiments.optim.xgb.early_stopping_rounds" = 5L)data_split <- splitTools::partition(
y = dataset[, get(target_col)],
p = c(train = 0.7, test = 0.3),
type = "stratified",
seed = seed
)
train_x <- model.matrix(
~ -1 + .,
dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L
test_x <- model.matrix(
~ -1 + .,
dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1Lfold_list <- splitTools::create_folds(
y = train_y,
k = 3,
type = "stratified",
seed = seed
)# required learner arguments, not optimized
learner_args <- list(
objective = "binary:logistic",
eval_metric = "logloss"
)
# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- NULL
performance_metric <- metric("auc")
performance_metric_args <- list(positive = "1", negative = "0")
return_models <- FALSE
# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
subsample = seq(0.6, 1, .2),
colsample_bytree = seq(0.6, 1, .2),
min_child_weight = seq(1, 5, 4),
learning_rate = seq(0.1, 0.2, 0.1),
max_depth = seq(1, 5, 4)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
set.seed(123)
sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}
# required for bayesian optimization
parameter_bounds <- list(
subsample = c(0.2, 1),
colsample_bytree = c(0.2, 1),
min_child_weight = c(1L, 10L),
learning_rate = c(0.1, 0.2),
max_depth = c(1L, 10L)
)
optim_args <- list(
n_iter = ncores,
kappa = 3.5,
acq = "ucb"
)tuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_grid <- tuner$execute(k = 3)
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
head(tuner_results_grid)
#> setting_id metric_optim_mean nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric
#> <int> <num> <int> <num> <num> <num> <num> <num> <char> <char>
#> 1: 1 0.1473724 20 0.6 0.8 5 0.2 1 binary:logistic logloss
#> 2: 2 0.1836365 20 1.0 0.8 5 0.1 5 binary:logistic logloss
#> 3: 3 0.2004256 20 0.8 0.8 5 0.1 1 binary:logistic logloss
#> 4: 4 0.1431941 20 0.6 0.8 5 0.2 5 binary:logistic logloss
#> 5: 5 0.1766953 20 1.0 0.8 1 0.1 5 binary:logistic logloss
#> 6: 6 0.1866896 20 0.8 0.8 5 0.1 5 binary:logistic loglosstuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds
tuner$learner_args <- learner_args
tuner$optim_args <- optim_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.045 Round = 1 subsample = 0.8000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 1.0000 Value = -0.2004256
#> elapsed = 0.037 Round = 2 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 1.0000 learning_rate = 0.2000 max_depth = 1.0000 Value = -0.146329
#> elapsed = 0.033 Round = 3 subsample = 1.0000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 5.0000 Value = -0.1836365
#> elapsed = 0.033 Round = 4 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 5.0000 learning_rate = 0.2000 max_depth = 5.0000 Value = -0.143957
#> elapsed = 0.026 Round = 5 subsample = 0.2000 colsample_bytree = 0.2253119 min_child_weight = 6.0000 learning_rate = 0.1478213 max_depth = 9.0000 Value = -0.2851956
#> elapsed = 0.048 Round = 6 subsample = 1.0000 colsample_bytree = 0.6428195 min_child_weight = 7.0000 learning_rate = 0.2000 max_depth = 10.0000 Value = -0.1407496
#> elapsed = 0.029 Round = 7 subsample = 0.6370911 colsample_bytree = 0.8772461 min_child_weight = 3.0000 learning_rate = 0.1746002 max_depth = 4.0000 Value = -0.1423412
#> elapsed = 0.034 Round = 8 subsample = 1.0000 colsample_bytree = 1.0000 min_child_weight = 1.0000 learning_rate = 0.1753007 max_depth = 10.0000 Value = -0.1424366
#>
#> Best Parameters Found:
#> Round = 6 subsample = 1.0000 colsample_bytree = 0.6428195 min_child_weight = 7.0000 learning_rate = 0.2000 max_depth = 10.0000 Value = -0.1407496
head(tuner_results_bayesian)
#> setting_id subsample colsample_bytree min_child_weight learning_rate max_depth Value objective eval_metric metric_optim_mean
#> <int> <num> <num> <num> <num> <num> <num> <char> <char> <num>
#> 1: 1 0.8 0.8000000 5 0.1000000 1 -0.2004256 binary:logistic logloss 0.2004256
#> 2: 2 0.6 1.0000000 1 0.2000000 1 -0.1463290 binary:logistic logloss 0.1463290
#> 3: 3 1.0 0.8000000 5 0.1000000 5 -0.1836365 binary:logistic logloss 0.1836365
#> 4: 4 0.6 1.0000000 5 0.2000000 5 -0.1439570 binary:logistic logloss 0.1439570
#> 5: 5 0.2 0.2253119 6 0.1478213 9 -0.2851956 binary:logistic logloss 0.2851956
#> 6: 6 1.0 0.6428195 7 0.2000000 10 -0.1407496 binary:logistic logloss 0.1407496validator <- mlexperiments::MLCrossValidation$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
fold_list = fold_list,
ncores = ncores,
seed = seed
)
validator$learner_args <- tuner$results$best.setting[-1]
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
head(validator_results)
#> fold performance colsample_bytree min_child_weight learning_rate max_depth objective eval_metric nrounds
#> <char> <num> <num> <num> <num> <num> <char> <char> <int>
#> 1: Fold1 0.9920565 0.6428195 7 0.2 10 binary:logistic logloss 20
#> 2: Fold2 0.9928918 0.6428195 7 0.2 10 binary:logistic logloss 20
#> 3: Fold3 0.9818853 0.6428195 7 0.2 10 binary:logistic logloss 20validator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
head(validator_results)
#> fold performance nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric
#> <char> <num> <int> <num> <num> <num> <num> <num> <char> <char>
#> 1: Fold1 0.9876434 20 0.6 1 1 0.2 1 binary:logistic logloss
#> 2: Fold2 0.9905513 20 0.6 1 1 0.2 1 binary:logistic logloss
#> 3: Fold3 0.9843750 20 0.6 1 1 0.2 1 binary:logistic loglossvalidator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.045 Round = 1 subsample = 0.8000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 1.0000 Value = -0.2144507
#> elapsed = 0.025 Round = 2 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 1.0000 learning_rate = 0.2000 max_depth = 1.0000 Value = -0.1632438
#> elapsed = 0.028 Round = 3 subsample = 1.0000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 5.0000 Value = -0.1995263
#> elapsed = 0.037 Round = 4 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 5.0000 learning_rate = 0.2000 max_depth = 5.0000 Value = -0.1765465
#> elapsed = 0.052 Round = 5 subsample = 0.6587262 colsample_bytree = 0.2441275 min_child_weight = 4.0000 learning_rate = 0.1583349 max_depth = 8.0000 Value = -0.1751771
#> elapsed = 0.032 Round = 6 subsample = 0.2000 colsample_bytree = 0.7380485 min_child_weight = 1.0000 learning_rate = 0.1221712 max_depth = 6.0000 Value = -0.2001825
#> elapsed = 0.026 Round = 7 subsample = 0.4372448 colsample_bytree = 0.304417 min_child_weight = 1.0000 learning_rate = 0.1949787 max_depth = 8.0000 Value = -0.1653602
#> elapsed = 0.029 Round = 8 subsample = 1.0000 colsample_bytree = 0.3752773 min_child_weight = 1.0000 learning_rate = 0.176386 max_depth = 9.0000 Value = -0.151424
#>
#> Best Parameters Found:
#> Round = 8 subsample = 1.0000 colsample_bytree = 0.3752773 min_child_weight = 1.0000 learning_rate = 0.176386 max_depth = 9.0000 Value = -0.151424
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.024 Round = 1 subsample = 0.8000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 1.0000 Value = -0.2016618
#> elapsed = 0.024 Round = 2 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 1.0000 learning_rate = 0.2000 max_depth = 1.0000 Value = -0.1358879
#> elapsed = 0.036 Round = 3 subsample = 1.0000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 5.0000 Value = -0.1937136
#> elapsed = 0.025 Round = 4 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 5.0000 learning_rate = 0.2000 max_depth = 5.0000 Value = -0.1629105
#> elapsed = 0.026 Round = 5 subsample = 0.9597928 colsample_bytree = 0.2087203 min_child_weight = 1.0000 learning_rate = 0.1807996 max_depth = 3.0000 Value = -0.1409365
#> elapsed = 0.026 Round = 6 subsample = 0.9648661 colsample_bytree = 0.2000 min_child_weight = 10.0000 learning_rate = 0.2000 max_depth = 10.0000 Value = -0.1876724
#> elapsed = 0.027 Round = 7 subsample = 0.2232804 colsample_bytree = 0.89331 min_child_weight = 1.0000 learning_rate = 0.2000 max_depth = 10.0000 Value = -0.147581
#> elapsed = 0.051 Round = 8 subsample = 0.2000 colsample_bytree = 0.8707313 min_child_weight = 1.0000 learning_rate = 0.1000 max_depth = 10.0000 Value = -0.2066387
#>
#> Best Parameters Found:
#> Round = 2 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 1.0000 learning_rate = 0.2000 max_depth = 1.0000 Value = -0.1358879
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.042 Round = 1 subsample = 0.8000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 1.0000 Value = -0.1996737
#> elapsed = 0.023 Round = 2 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 1.0000 learning_rate = 0.2000 max_depth = 1.0000 Value = -0.1550507
#> elapsed = 0.03 Round = 3 subsample = 1.0000 colsample_bytree = 0.8000 min_child_weight = 5.0000 learning_rate = 0.1000 max_depth = 5.0000 Value = -0.1921469
#> elapsed = 0.024 Round = 4 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 5.0000 learning_rate = 0.2000 max_depth = 5.0000 Value = -0.1630041
#> elapsed = 0.028 Round = 5 subsample = 0.8696134 colsample_bytree = 0.5493993 min_child_weight = 2.0000 learning_rate = 0.1541663 max_depth = 4.0000 Value = -0.156581
#> elapsed = 0.012 Round = 6 subsample = 0.2000 colsample_bytree = 0.4853539 min_child_weight = 10.0000 learning_rate = 0.1531602 max_depth = 10.0000 Value = -0.6478907
#> elapsed = 0.05 Round = 7 subsample = 0.2232806 colsample_bytree = 1.0000 min_child_weight = 3.0000 learning_rate = 0.1040753 max_depth = 10.0000 Value = -0.2244361
#> elapsed = 0.024 Round = 8 subsample = 0.5157127 colsample_bytree = 0.9291875 min_child_weight = 7.0000 learning_rate = 0.1998166 max_depth = 3.0000 Value = -0.2151049
#>
#> Best Parameters Found:
#> Round = 2 subsample = 0.6000 colsample_bytree = 1.0000 min_child_weight = 1.0000 learning_rate = 0.2000 max_depth = 1.0000 Value = -0.1550507
head(validator_results)
#> fold performance subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric nrounds
#> <char> <num> <num> <num> <num> <num> <num> <char> <char> <int>
#> 1: Fold1 0.9955869 1.0 0.3752773 1 0.176386 9 binary:logistic logloss 20
#> 2: Fold2 0.9905513 0.6 1.0000000 1 0.200000 1 binary:logistic logloss 20
#> 3: Fold3 0.9843750 0.6 1.0000000 1 0.200000 1 binary:logistic logloss 20preds_xgboost <- mlexperiments::predictions(
object = validator,
newdata = test_x
)perf_xgboost <- mlexperiments::performance(
object = validator,
prediction_results = preds_xgboost,
y_ground_truth = test_y,
type = "binary"
)
perf_xgboost
#> model performance AUC Brier BrierScaled BAC TP TN FP FN TPR TNR FPR FNR PPV
#> <char> <num> <num> <num> <num> <num> <int> <int> <int> <int> <num> <num> <num> <num> <num>
#> 1: Fold1 0.9839863 0.9839863 0.04247381 0.8131821 0.9369818 64 132 2 8 0.8888889 0.9850746 0.014925373 0.1111111 0.9696970
#> 2: Fold2 0.9812915 0.9812915 0.04675659 0.7943447 0.9059909 59 133 1 13 0.8194444 0.9925373 0.007462687 0.1805556 0.9833333
#> 3: Fold3 0.9830535 0.9830535 0.04884222 0.7851712 0.9161484 61 132 2 11 0.8472222 0.9850746 0.014925373 0.1527778 0.9682540
#> NPV FDR MCC F1 GMEAN GPR ACC MMCE BER
#> <num> <num> <num> <num> <num> <num> <num> <num> <num>
#> 1: 0.9428571 0.03030303 0.8930504 0.9275362 0.9357467 0.9284142 0.9514563 0.04854369 0.06301824
#> 2: 0.9109589 0.01666667 0.8521438 0.8939394 0.9018477 0.8976564 0.9320388 0.06796117 0.09400912
#> 3: 0.9230769 0.03174603 0.8613082 0.9037037 0.9135519 0.9057187 0.9368932 0.06310680 0.08385158These binaries (installable software) and packages are in development.
They may not be fully stable and should be used with caution. We make no claims about them.