# nolint start
library(mlexperiments)
library(mllrnrs)The hardware and bandwidth for this mirror is donated by METANET, the Webhosting and Full Service-Cloud Provider.
If you wish to report a bug, or if you are interested in having us mirror your free-software or open-source project, please feel free to contact us at mirror[@]metanet.ch.
# nolint start
library(mlexperiments)
library(mllrnrs)See https://github.com/kapsner/mllrnrs/blob/main/R/learner_lightgbm.R for implementation details.
library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
data.table::as.data.table() |>
na.omit()
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
# on cran
ncores <- 2L
} else {
ncores <- ifelse(
test = parallel::detectCores() > 4,
yes = 4L,
no = ifelse(
test = parallel::detectCores() < 2L,
yes = 1L,
no = parallel::detectCores()
)
)
}
options("mlexperiments.bayesian.max_init" = 4L)
options("mlexperiments.optim.lgb.nrounds" = 20L)
options("mlexperiments.optim.lgb.early_stopping_rounds" = 5L)data_split <- splitTools::partition(
y = dataset[, get(target_col)],
p = c(train = 0.7, test = 0.3),
type = "stratified",
seed = seed
)
train_x <- model.matrix(
~ -1 + .,
dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L
test_x <- model.matrix(
~ -1 + .,
dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1Lfold_list <- splitTools::create_folds(
y = train_y,
k = 3,
type = "stratified",
seed = seed
)# required learner arguments, not optimized
learner_args <- list(
max_depth = -1L,
verbose = -1L,
objective = "binary",
metric = "binary_logloss"
)
# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- NULL
performance_metric <- metric("auc")
performance_metric_args <- list(positive = "1", negative = "0")
return_models <- FALSE
# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
bagging_fraction = seq(0.6, 1, .2),
feature_fraction = seq(0.6, 1, .2),
min_data_in_leaf = seq(2, 10, 2),
learning_rate = seq(0.1, 0.2, 0.1),
num_leaves = seq(2, 20, 4)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
set.seed(123)
sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}
# required for bayesian optimization
parameter_bounds <- list(
bagging_fraction = c(0.2, 1),
feature_fraction = c(0.2, 1),
min_data_in_leaf = c(2L, 12L),
learning_rate = c(0.1, 0.2),
num_leaves = c(2L, 20L)
)
optim_args <- list(
n_iter = ncores,
kappa = 3.5,
acq = "ucb"
)tuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_grid <- tuner$execute(k = 3)
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
head(tuner_results_grid)
#> setting_id metric_optim_mean nrounds bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective
#> <int> <num> <int> <num> <num> <num> <num> <num> <int> <int> <char>
#> 1: 1 0.1413020 18 0.6 0.6 4 0.2 18 -1 -1 binary
#> 2: 2 0.1336577 18 0.8 1.0 10 0.2 6 -1 -1 binary
#> 3: 3 0.1959308 20 0.8 0.8 4 0.1 2 -1 -1 binary
#> 4: 4 0.1665362 20 1.0 0.8 4 0.1 10 -1 -1 binary
#> 5: 5 0.1325429 20 1.0 0.6 6 0.2 18 -1 -1 binary
#> 6: 6 0.1651197 20 1.0 1.0 8 0.1 14 -1 -1 binary
#> metric
#> <char>
#> 1: binary_logloss
#> 2: binary_logloss
#> 3: binary_logloss
#> 4: binary_logloss
#> 5: binary_logloss
#> 6: binary_loglosstuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds
tuner$learner_args <- learner_args
tuner$optim_args <- optim_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.032 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.1959308
#> elapsed = 0.04 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1288185
#> elapsed = 0.034 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1336577
#> elapsed = 0.038 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1622234
#> elapsed = 0.048 Round = 5 bagging_fraction = 0.8514907 feature_fraction = 0.5359993 min_data_in_leaf = 9.0000 learning_rate = 0.1801683 num_leaves = 20.0000 Value = -0.1392806
#> elapsed = 0.043 Round = 6 bagging_fraction = 0.2079895 feature_fraction = 0.385567 min_data_in_leaf = 3.0000 learning_rate = 0.1618143 num_leaves = 11.0000 Value = -0.137801
#> elapsed = 0.036 Round = 7 bagging_fraction = 0.8973112 feature_fraction = 0.2399434 min_data_in_leaf = 12.0000 learning_rate = 0.2000 num_leaves = 15.0000 Value = -0.1243682
#> elapsed = 0.044 Round = 8 bagging_fraction = 0.4285091 feature_fraction = 0.2176898 min_data_in_leaf = 12.0000 learning_rate = 0.1330769 num_leaves = 20.0000 Value = -0.1403338
#>
#> Best Parameters Found:
#> Round = 7 bagging_fraction = 0.8973112 feature_fraction = 0.2399434 min_data_in_leaf = 12.0000 learning_rate = 0.2000 num_leaves = 15.0000 Value = -0.1243682
head(tuner_results_bayesian)
#> setting_id bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves Value max_depth verbose objective metric
#> <int> <num> <num> <num> <num> <num> <num> <int> <int> <char> <char>
#> 1: 1 0.8000000 0.8000000 4 0.1000000 2 -0.1959308 -1 -1 binary binary_logloss
#> 2: 2 0.6000000 0.6000000 6 0.2000000 10 -0.1288185 -1 -1 binary binary_logloss
#> 3: 3 0.8000000 1.0000000 10 0.2000000 6 -0.1336577 -1 -1 binary binary_logloss
#> 4: 4 0.8000000 0.6000000 8 0.1000000 14 -0.1622234 -1 -1 binary binary_logloss
#> 5: 5 0.8514907 0.5359993 9 0.1801683 20 -0.1392806 -1 -1 binary binary_logloss
#> 6: 6 0.2079895 0.3855670 3 0.1618143 11 -0.1378010 -1 -1 binary binary_logloss
#> metric_optim_mean
#> <num>
#> 1: 0.1959308
#> 2: 0.1288185
#> 3: 0.1336577
#> 4: 0.1622234
#> 5: 0.1392806
#> 6: 0.1378010validator <- mlexperiments::MLCrossValidation$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
fold_list = fold_list,
ncores = ncores,
seed = seed
)
validator$learner_args <- tuner$results$best.setting[-1]
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
head(validator_results)
#> fold performance feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective metric nrounds
#> <char> <num> <num> <num> <num> <num> <int> <int> <char> <char> <int>
#> 1: Fold1 0.9890556 0.2399434 12 0.2 15 -1 -1 binary binary_logloss 20
#> 2: Fold2 0.9932386 0.2399434 12 0.2 15 -1 -1 binary binary_logloss 20
#> 3: Fold3 0.9876374 0.2399434 12 0.2 15 -1 -1 binary binary_logloss 20validator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
head(validator_results)
#> fold performance nrounds bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective
#> <char> <num> <int> <num> <num> <num> <num> <num> <int> <int> <char>
#> 1: Fold1 0.9902913 20 0.8 1.0 10 0.2 6 -1 -1 binary
#> 2: Fold2 0.9883842 19 0.8 1.0 10 0.2 6 -1 -1 binary
#> 3: Fold3 0.9862637 18 0.6 0.6 6 0.2 10 -1 -1 binary
#> metric
#> <char>
#> 1: binary_logloss
#> 2: binary_logloss
#> 3: binary_loglossvalidator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.027 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.2081399
#> elapsed = 0.035 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1575606
#> elapsed = 0.03 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1462635
#> elapsed = 0.036 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1733679
#> elapsed = 0.029 Round = 5 bagging_fraction = 0.5596134 feature_fraction = 0.9300915 min_data_in_leaf = 7.0000 learning_rate = 0.1800818 num_leaves = 2.0000 Value = -0.1645675
#> elapsed = 0.032 Round = 6 bagging_fraction = 0.8950514 feature_fraction = 0.2196616 min_data_in_leaf = 12.0000 learning_rate = 0.1620435 num_leaves = 20.0000 Value = -0.149935
#> elapsed = 0.025 Round = 7 bagging_fraction = 0.2000 feature_fraction = 0.991415 min_data_in_leaf = 12.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.210863
#> elapsed = 0.043 Round = 8 bagging_fraction = 0.2000 feature_fraction = 0.8495294 min_data_in_leaf = 2.0000 learning_rate = 0.1000 num_leaves = 20.0000 Value = -0.2048307
#>
#> Best Parameters Found:
#> Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1462635
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.029 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.1968844
#> elapsed = 0.04 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1448359
#> elapsed = 0.032 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1431115
#> elapsed = 0.035 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1744139
#> elapsed = 0.037 Round = 5 bagging_fraction = 0.5157127 feature_fraction = 0.9291875 min_data_in_leaf = 9.0000 learning_rate = 0.1626326 num_leaves = 6.0000 Value = -0.1437751
#> elapsed = 0.042 Round = 6 bagging_fraction = 0.803404 feature_fraction = 0.8105688 min_data_in_leaf = 9.0000 learning_rate = 0.1840982 num_leaves = 6.0000 Value = -0.1449229
#> elapsed = 0.034 Round = 7 bagging_fraction = 0.3970573 feature_fraction = 0.271388 min_data_in_leaf = 9.0000 learning_rate = 0.16237 num_leaves = 10.0000 Value = -0.1432943
#> elapsed = 0.039 Round = 8 bagging_fraction = 0.3970573 feature_fraction = 0.2713869 min_data_in_leaf = 9.0000 learning_rate = 0.16237 num_leaves = 10.0000 Value = -0.1432943
#>
#> Best Parameters Found:
#> Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1431115
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.026 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.1935595
#> elapsed = 0.034 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1456779
#> elapsed = 0.03 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1592296
#> elapsed = 0.039 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1740933
#> elapsed = 0.029 Round = 5 bagging_fraction = 0.5157127 feature_fraction = 0.9291875 min_data_in_leaf = 9.0000 learning_rate = 0.1998166 num_leaves = 6.0000 Value = -0.1436106
#> elapsed = 0.042 Round = 6 bagging_fraction = 0.2000 feature_fraction = 0.4928526 min_data_in_leaf = 6.0000 learning_rate = 0.1736983 num_leaves = 20.0000 Value = -0.1451436
#> elapsed = 0.039 Round = 7 bagging_fraction = 0.8514907 feature_fraction = 0.5359993 min_data_in_leaf = 9.0000 learning_rate = 0.1068914 num_leaves = 20.0000 Value = -0.1693466
#> elapsed = 0.034 Round = 8 bagging_fraction = 0.2000 feature_fraction = 0.2000 min_data_in_leaf = 12.0000 learning_rate = 0.1000 num_leaves = 20.0000 Value = -0.1729361
#>
#> Best Parameters Found:
#> Round = 5 bagging_fraction = 0.5157127 feature_fraction = 0.9291875 min_data_in_leaf = 9.0000 learning_rate = 0.1998166 num_leaves = 6.0000 Value = -0.1436106
head(validator_results)
#> fold performance bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective metric
#> <char> <num> <num> <num> <num> <num> <num> <int> <int> <char> <char>
#> 1: Fold1 0.9902913 0.8000000 1.0000000 10 0.2000000 6 -1 -1 binary binary_logloss
#> 2: Fold2 0.9883842 0.8000000 1.0000000 10 0.2000000 6 -1 -1 binary binary_logloss
#> 3: Fold3 0.9869505 0.5157127 0.9291875 9 0.1998166 6 -1 -1 binary binary_logloss
#> nrounds
#> <int>
#> 1: 20
#> 2: 19
#> 3: 19preds_lightgbm <- mlexperiments::predictions(
object = validator,
newdata = test_x
)perf_lightgbm <- mlexperiments::performance(
object = validator,
prediction_results = preds_lightgbm,
y_ground_truth = test_y,
type = "binary"
)
perf_lightgbm
#> model performance AUC Brier BrierScaled BAC TP TN FP FN TPR TNR FPR FNR PPV
#> <char> <num> <num> <num> <num> <num> <int> <int> <int> <int> <num> <num> <num> <num> <num>
#> 1: Fold1 0.9764718 0.9764718 0.03740030 0.8354976 0.9407131 64 133 1 8 0.8888889 0.9925373 0.007462687 0.11111111 0.9846154
#> 2: Fold2 0.9906198 0.9906198 0.04184296 0.8159569 0.9268242 62 133 1 10 0.8611111 0.9925373 0.007462687 0.13888889 0.9841270
#> 3: Fold3 0.9811360 0.9811360 0.04052871 0.8217375 0.9401949 65 131 3 7 0.9027778 0.9776119 0.022388060 0.09722222 0.9558824
#> NPV FDR MCC F1 GMEAN GPR ACC MMCE BER
#> <num> <num> <num> <num> <num> <num> <num> <num> <num>
#> 1: 0.9432624 0.01538462 0.9043538 0.9343066 0.9392845 0.9355286 0.9563107 0.04368932 0.05928690
#> 2: 0.9300699 0.01587302 0.8834041 0.9185185 0.9244917 0.9205665 0.9466019 0.05339806 0.07317579
#> 3: 0.9492754 0.04411765 0.8926878 0.9285714 0.9394500 0.9289507 0.9514563 0.04854369 0.05980514These binaries (installable software) and packages are in development.
They may not be fully stable and should be used with caution. We make no claims about them.