The hardware and bandwidth for this mirror is donated by METANET, the Webhosting and Full Service-Cloud Provider.
If you wish to report a bug, or if you are interested in having us mirror your free-software or open-source project, please feel free to contact us at mirror[@]metanet.ch.
development <- simulate_gazepoint_governed_data(18L, 6L, 1L, seed = 2901L)
predictors <- c("tracking_ratio", "blink_rate", "gaze_dispersion")
task <- create_gazepoint_synthetic_task(development, "recording_quality", "new_participants")
manifest <- create_gazepoint_synthetic_manifest(task$outcome, predictors)
folds <- create_gazepoint_group_folds(
development, task$outcome, predictors, manifest,
task$generalization_target,
"participant_id", "trial_id", "stimulus_id",
v = 3L, repeats = 1L, seed = 2901L
)
development_evaluation <- evaluate_gazepoint_group_folds(
folds, task, predictors, "glm", seed = 2901L,
assess_calibration = TRUE, calibration_bootstrap = 0L
)
model <- fit_gazepoint_model(
development, task, predictors, "glm", seed = 2901L
)external <- simulate_gazepoint_governed_data(10L, 6L, 1L, seed = 2902L)
external$participant_id <- paste0("E", external$participant_id)
external$trial_id <- paste0("E", external$trial_id)
external$stimulus_id <- paste0("E", external$stimulus_id)
declaration <- declare_gazepoint_external_dataset(
external,
label = "synthetic_external_site",
independent = TRUE,
origin = "Independent deterministic synthetic generation"
)
report <- evaluate_gazepoint_external_transportability(
model,
development_data = development,
external_data = external,
declaration = declaration,
development_evaluation = development_evaluation,
bootstrap = 0L,
seed = 2902L
)
report
#> <gp3ml_transportability_report>
#> Status: externally_validated
#> Reason: The explicitly independent dataset passed schema and identifier-overlap gates.
#> metric development_estimate external_estimate
#> accuracy 0.87037037 0.8833333
#> balanced_accuracy 0.49425287 0.5000000
#> sensitivity 0.00000000 0.0000000
#> specificity 0.98850575 1.0000000
#> precision 0.00000000 NA
#> recall 0.00000000 0.0000000
#> f1 NaN NA
#> mcc -0.08304548 NA
#> roc_auc 0.57340641 0.7277628
#> pr_auc 0.30964591 0.3291055
#> brier 0.11055175 0.0956074
#> log_loss 0.39533485 0.3260348
#> ece 0.11646202 NA
#> calibration_intercept_abs 3.23095109 NA
#> calibration_slope_abs_error 1.47811645 NA
#> difference
#> 0.012962963
#> 0.005747126
#> 0.000000000
#> 0.011494253
#> NA
#> 0.000000000
#> NA
#> NA
#> 0.154356393
#> 0.019459639
#> -0.014944353
#> -0.069300020
#> NA
#> NA
#> NA
report$performance_comparison
#> metric development_estimate external_estimate
#> 1 accuracy 0.87037037 0.8833333
#> 2 balanced_accuracy 0.49425287 0.5000000
#> 3 sensitivity 0.00000000 0.0000000
#> 4 specificity 0.98850575 1.0000000
#> 5 precision 0.00000000 NA
#> 6 recall 0.00000000 0.0000000
#> 7 f1 NaN NA
#> 8 mcc -0.08304548 NA
#> 9 roc_auc 0.57340641 0.7277628
#> 10 pr_auc 0.30964591 0.3291055
#> 11 brier 0.11055175 0.0956074
#> 12 log_loss 0.39533485 0.3260348
#> 13 ece 0.11646202 NA
#> 14 calibration_intercept_abs 3.23095109 NA
#> 15 calibration_slope_abs_error 1.47811645 NA
#> difference
#> 1 0.012962963
#> 2 0.005747126
#> 3 0.000000000
#> 4 0.011494253
#> 5 NA
#> 6 0.000000000
#> 7 NA
#> 8 NA
#> 9 0.154356393
#> 10 0.019459639
#> 11 -0.014944353
#> 12 -0.069300020
#> 13 NA
#> 14 NA
#> 15 NA
report$group_coverage
#> unit identifier development_groups external_groups
#> 1 participant participant_id 18 10
#> 2 stimulus stimulus_id 6 6
#> overlapping_groups external_novel_groups external_coverage_prop status
#> 1 0 10 1 pass
#> 2 0 6 1 passnot_validated <- evaluate_gazepoint_external_transportability(
model,
development_data = development,
external_data = NULL
)
not_validated
#> <gp3ml_transportability_report>
#> Status: not_externally_validated
#> Reason: No independent external dataset was supplied.An internal holdout is never renamed external validation. The external result is specific to the declared dataset and context.
These binaries (installable software) and packages are in development.
They may not be fully stable and should be used with caution. We make no claims about them.