# nolint start
library(mlexperiments)
library(mllrnrs)The hardware and bandwidth for this mirror is donated by METANET, the Webhosting and Full Service-Cloud Provider.
If you wish to report a bug, or if you are interested in having us mirror your free-software or open-source project, please feel free to contact us at mirror[@]metanet.ch.
# nolint start
library(mlexperiments)
library(mllrnrs)See https://github.com/kapsner/mllrnrs/blob/main/R/learner_xgboost.R for implementation details.
library(mlbench)
data("PimaIndiansDiabetes2")
dataset <- PimaIndiansDiabetes2 |>
data.table::as.data.table() |>
na.omit()
feature_cols <- colnames(dataset)[1:8]
target_col <- "diabetes"seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
# on cran
ncores <- 2L
} else {
ncores <- ifelse(
test = parallel::detectCores() > 4,
yes = 4L,
no = ifelse(
test = parallel::detectCores() < 2L,
yes = 1L,
no = parallel::detectCores()
)
)
}
options("mlexperiments.bayesian.max_init" = 10L)
options("mlexperiments.optim.xgb.nrounds" = 100L)
options("mlexperiments.optim.xgb.early_stopping_rounds" = 10L)data_split <- splitTools::partition(
y = dataset[, get(target_col)],
p = c(train = 0.7, test = 0.3),
type = "stratified",
seed = seed
)
train_x <- model.matrix(
~ -1 + .,
dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L
test_x <- model.matrix(
~ -1 + .,
dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1Lfold_list <- splitTools::create_folds(
y = train_y,
k = 3,
type = "stratified",
seed = seed
)# required learner arguments, not optimized
learner_args <- list(
objective = "binary:logistic",
eval_metric = "logloss"
)
# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- NULL
performance_metric <- metric("auc")
performance_metric_args <- list(positive = "1")
return_models <- FALSE
# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
subsample = seq(0.6, 1, .2),
colsample_bytree = seq(0.6, 1, .2),
min_child_weight = seq(1, 5, 4),
learning_rate = seq(0.1, 0.2, 0.1),
max_depth = seq(1, 5, 4)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
set.seed(123)
sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}
# required for bayesian optimization
parameter_bounds <- list(
subsample = c(0.2, 1),
colsample_bytree = c(0.2, 1),
min_child_weight = c(1L, 10L),
learning_rate = c(0.1, 0.2),
max_depth = c(1L, 10L)
)
optim_args <- list(
iters.n = ncores,
kappa = 3.5,
acq = "ucb"
)tuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_grid <- tuner$execute(k = 3)
#>
#> Parameter settings [==================>-----------------------------------------------------------------------------] 2/10 ( 20%)
#> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%)
#> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%)
#> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%)
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
head(tuner_results_grid)
#> setting_id metric_optim_mean nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective
#> 1: 1 0.4121967 34 0.6 0.8 5 0.2 1 binary:logistic
#> 2: 2 0.3890956 57 1.0 0.8 5 0.1 5 binary:logistic
#> 3: 3 0.3925308 100 0.8 0.8 5 0.1 1 binary:logistic
#> 4: 4 0.4082505 34 0.6 0.8 5 0.2 5 binary:logistic
#> 5: 5 0.3975907 36 1.0 0.8 1 0.1 5 binary:logistic
#> 6: 6 0.3932451 66 0.8 0.8 5 0.1 5 binary:logistic
#> eval_metric
#> 1: logloss
#> 2: logloss
#> 3: logloss
#> 4: logloss
#> 5: logloss
#> 6: loglosstuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds
tuner$learner_args <- learner_args
tuner$optim_args <- optim_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Registering parallel backend using 4 cores.
head(tuner_results_bayesian)
#> Epoch setting_id subsample colsample_bytree min_child_weight learning_rate max_depth gpUtility acqOptimum inBounds Elapsed
#> 1: 0 1 0.6 0.8 5 0.2 1 NA FALSE TRUE 1.695
#> 2: 0 2 1.0 0.8 5 0.1 5 NA FALSE TRUE 1.702
#> 3: 0 3 0.8 0.8 5 0.1 1 NA FALSE TRUE 1.734
#> 4: 0 4 0.6 0.8 5 0.2 5 NA FALSE TRUE 1.724
#> 5: 0 5 1.0 0.8 1 0.1 5 NA FALSE TRUE 0.849
#> 6: 0 6 0.8 0.8 5 0.1 5 NA FALSE TRUE 0.850
#> Score metric_optim_mean nrounds errorMessage objective eval_metric
#> 1: -0.4089735 0.4089735 56 NA binary:logistic logloss
#> 2: -0.3970937 0.3970937 49 NA binary:logistic logloss
#> 3: -0.4013240 0.4013240 100 NA binary:logistic logloss
#> 4: -0.4070968 0.4070968 69 NA binary:logistic logloss
#> 5: -0.3819756 0.3819756 39 NA binary:logistic logloss
#> 6: -0.3987643 0.3987643 99 NA binary:logistic loglossvalidator <- mlexperiments::MLCrossValidation$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
fold_list = fold_list,
ncores = ncores,
seed = seed
)
validator$learner_args <- tuner$results$best.setting[-1]
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
head(validator_results)
#> fold performance subsample colsample_bytree min_child_weight learning_rate max_depth nrounds objective eval_metric
#> 1: Fold1 0.8799577 1 0.8 1 0.1 5 39 binary:logistic logloss
#> 2: Fold2 0.8635643 1 0.8 1 0.1 5 39 binary:logistic logloss
#> 3: Fold3 0.9027699 1 0.8 1 0.1 5 39 binary:logistic loglossvalidator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
#> CV fold: Fold2
#> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%)
#>
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
#> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%)
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
head(validator_results)
#> fold performance nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric
#> 1: Fold1 0.8675304 40 0.6 1 1 0.2 1 binary:logistic logloss
#> 2: Fold2 0.8635643 44 1.0 1 5 0.1 5 binary:logistic logloss
#> 3: Fold3 0.8793103 24 0.6 1 1 0.2 1 binary:logistic loglossvalidator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Registering parallel backend using 4 cores.
#>
#> CV fold: Fold2
#> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%)
#>
#> Registering parallel backend using 4 cores.
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
#> Registering parallel backend using 4 cores.
head(validator_results)
#> fold performance subsample colsample_bytree min_child_weight learning_rate max_depth nrounds objective eval_metric
#> 1: Fold1 0.8662084 0.6 1.0 1 0.2 1 28 binary:logistic logloss
#> 2: Fold2 0.8746695 1.0 0.8 5 0.1 5 44 binary:logistic logloss
#> 3: Fold3 0.8903335 0.6 1.0 1 0.1 5 30 binary:logistic loglosspreds_xgboost <- mlexperiments::predictions(
object = validator,
newdata = test_x
)perf_xgboost <- mlexperiments::performance(
object = validator,
prediction_results = preds_xgboost,
y_ground_truth = test_y,
type = "binary"
)
perf_xgboost
#> model performance auc prauc sensitivity specificity ppv npv tn tp fn fp tnr tpr fnr
#> 1: Fold1 0.7922752 0.7922752 0.6016630 0.5128205 0.8734177 0.6666667 0.7840909 69 20 19 10 0.8734177 0.5128205 0.4871795
#> 2: Fold2 0.7687439 0.7687439 0.5601442 0.3846154 0.8860759 0.6250000 0.7446809 70 15 24 9 0.8860759 0.3846154 0.6153846
#> 3: Fold3 0.7594937 0.7594937 0.6142299 0.4871795 0.8481013 0.6129032 0.7701149 67 19 20 12 0.8481013 0.4871795 0.5128205
#> fpr bbrier acc ce fbeta
#> 1: 0.1265823 0.1726355 0.7542373 0.2457627 0.5797101
#> 2: 0.1139241 0.1885316 0.7203390 0.2796610 0.4761905
#> 3: 0.1518987 0.1854326 0.7288136 0.2711864 0.5428571These binaries (installable software) and packages are in development.
They may not be fully stable and should be used with caution. We make no claims about them.