From a45be0776cf886ce758a7781c22abc1faaa59dd6 Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 11:21:48 +0200 Subject: [PATCH 1/8] Add checks on packages being present. --- R/TestDataCreators.R | 27 +++++++++++++++ R/TestFunctions.R | 45 +++++++++++++++++++++++++ tests/testthat/test-collection_labels.R | 2 ++ tests/testthat/test-stratification.R | 2 ++ 4 files changed, 76 insertions(+) diff --git a/R/TestDataCreators.R b/R/TestDataCreators.R index 13f13d19..d5a35583 100644 --- a/R/TestDataCreators.R +++ b/R/TestDataCreators.R @@ -1,3 +1,30 @@ +test_data_package_installed <- function(outcome_type) { + run_test <- TRUE + + data_packages <- list( + "survival" = "survival", + "multinomial" = "datasets", + "binomial" = "MASS", + "continuous" = "Ecdat", + "count" = "MASS" + ) + + if (!is_package_installed(data_packages[[outcome_type]])) run_test <- FALSE + + if (!run_test) { + rlang::inform( + message = paste0( + "Cannot run test because the ", + data_packages[[outcome_type]], + " package is not installed."), + class = "familiar_message_inform_no_test" + ) + } + + return(run_test) +} + + test.create_good_data_set <- function(outcome_type, to_data_object=TRUE){ # Suppress NOTES due to non-standard evaluation in data.table diff --git a/R/TestFunctions.R b/R/TestFunctions.R index 95698d5c..bc1cdd32 100644 --- a/R/TestFunctions.R +++ b/R/TestFunctions.R @@ -1,3 +1,26 @@ +test_object_package_installed <- function(x) { + run_test <- TRUE + if (!is.null(x$error)) { + if (any(grepl("following package has to be installed", x$error, fixed = TRUE))) { + run_test <- FALSE + } else if (any(grepl("following packages have to be installed", x$error, fixed = TRUE))) { + run_test <- FALSE + } else { + stop(x$error) + } + } + + if (!run_test) { + rlang::inform( + message = x$error, + class = "familiar_message_inform_no_test" + ) + } + + return(run_test) +} + + test_all_learners_available <- function(learners){ # Create placeholder flags. @@ -57,6 +80,8 @@ test_all_learners_train_predict_vimp <- function( # Iterate over the outcome type. for(outcome_type in c("count", "continuous", "binomial", "multinomial", "survival")){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) full_one_sample_data <- test.create_one_sample_data_set(outcome_type) @@ -1048,6 +1073,8 @@ test_all_learners_parallel_train_predict_vimp <- function(learners, # Iterate over the outcome type. for(outcome_type in c("count", "continuous", "binomial", "multinomial", "survival")){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) @@ -1211,6 +1238,8 @@ test_all_novelty_detectors <- function(detectors, # Outcome type is not important, but set to get suitable datasets. outcome_type <- "continuous" + if (!test_data_package_installed(outcome_type)) return() + # Obtain data. full_data <- test.create_good_data_set(outcome_type) full_one_sample_data <- test.create_one_sample_data_set(outcome_type) @@ -1418,6 +1447,8 @@ test_all_novelty_detectors_parallel <- function(detectors, # Outcome type is not important, but set to get suitable datasets. outcome_type <- "continuous" + if (!test_data_package_installed(outcome_type)) return() + # Obtain data. full_data <- test.create_good_data_set(outcome_type) @@ -1531,6 +1562,8 @@ test_all_vimp_methods <- function(vimp_methods, # Iterate over the outcome type. for(outcome_type in c("count", "continuous", "binomial", "multinomial", "survival")){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) full_one_sample_data <- test.create_one_sample_data_set(outcome_type) @@ -1858,6 +1891,8 @@ test_all_vimp_methods_parallel <- function(vimp_methods, # Iterate over the outcome type. for(outcome_type in c("count", "continuous", "binomial", "multinomial", "survival")){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) @@ -1984,6 +2019,8 @@ test_all_metrics <- function(metrics, # Iterate over the outcome type. for(outcome_type in c("count", "continuous", "binomial", "multinomial", "survival")){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) identical_sample_data <- test.create_all_identical_data_set(outcome_type) @@ -2911,6 +2948,8 @@ test_hyperparameter_optimisation <- function(vimp_methods=NULL, # Iterate over the outcome type. for(outcome_type in outcome_type_available){ + if (!test_data_package_installed(outcome_type)) next + # Multi-feature data sets. full_data <- test.create_good_data_set(outcome_type) identical_sample_data <- test.create_all_identical_data_set(outcome_type) @@ -3482,6 +3521,8 @@ test_plots <- function(plot_function, # Iterate over the outcome type. for(outcome_type in c("count", "continuous", "survival", "binomial", "multinomial")){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) identical_sample_data <- test.create_all_identical_data_set(outcome_type) @@ -5171,6 +5212,8 @@ test_export_specific <- function(export_function, # Iterate over the outcome type. for(outcome_type in outcome_type_available){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. main_data <- test.create_good_data_set(outcome_type) @@ -5283,6 +5326,8 @@ integrated_test <- function(..., for(outcome_type in outcome_type_available){ + if (!test_data_package_installed(outcome_type)) next + .warning_good <- warning_good if(is.list(warning_good)) .warning_good <- warning_good[[outcome_type]] diff --git a/tests/testthat/test-collection_labels.R b/tests/testthat/test-collection_labels.R index 0158d461..da5ff7c6 100644 --- a/tests/testthat/test-collection_labels.R +++ b/tests/testthat/test-collection_labels.R @@ -2,6 +2,8 @@ outcome_type <- "multinomial" for(outcome_type in c("continuous", "multinomial", "survival")){ + if (!familiar:::test_data_package_installed(outcome_type)) next + # Get data. data <- familiar:::test.create_good_data_set(outcome_type = outcome_type) diff --git a/tests/testthat/test-stratification.R b/tests/testthat/test-stratification.R index 7dfd6858..dd2c088b 100644 --- a/tests/testthat/test-stratification.R +++ b/tests/testthat/test-stratification.R @@ -1,6 +1,8 @@ # Find available stratification methods. stratification_methods <- familiar:::.get_available_stratification_methods() +if (!familiar:::test_data_package_installed("survival")) testthat::skip() + # Test for good dataset. for(stratification_method in stratification_methods){ # Create a dataset using the good dataset. From dfda1798b1e5047a6dea07c4b16d9a930b260d58 Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 11:22:24 +0200 Subject: [PATCH 2/8] Add checks to test-stratification --- tests/testthat/test-stratification.R | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/testthat/test-stratification.R b/tests/testthat/test-stratification.R index dd2c088b..2b0b5d08 100644 --- a/tests/testthat/test-stratification.R +++ b/tests/testthat/test-stratification.R @@ -9,13 +9,19 @@ for(stratification_method in stratification_methods){ data <- familiar:::test.create_good_data_set("survival") # Train a simple linear GLM using the good dataset. - fam_model <- familiar:::test_train(data=data, - cluster_method="none", - imputation_method="simple", - hyperparameter_list=list("sign_size"=familiar:::get_n_features(data)), - learner="cox", - stratification_method=stratification_method, - create_novelty_detector=FALSE) + fam_model <- familiar:::do.call_with_handlers( + familiar:::test_train, + args = list(data=data, + cluster_method="none", + imputation_method="simple", + hyperparameter_list=list("sign_size"=familiar:::get_n_features(data)), + learner="cox", + stratification_method=stratification_method, + create_novelty_detector=FALSE) + ) + + if (!test_object_package_installed(fam_model)) testthat::skip() + fam_model <- fam_model$value # Risk stratification. predictions_risk <- familiar::predict(object=fam_model, From 23fb5de651ab80aa3dfdab20e619fdaedf4a31c4 Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 11:23:07 +0200 Subject: [PATCH 3/8] Add checks to test-vimp_table --- tests/testthat/test-vimp_table.R | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/testthat/test-vimp_table.R b/tests/testthat/test-vimp_table.R index d7cced92..01f3096b 100644 --- a/tests/testthat/test-vimp_table.R +++ b/tests/testthat/test-vimp_table.R @@ -4,17 +4,22 @@ data <- familiar:::test_create_synthetic_correlated_data(outcome_type="continuou n_numeric=2, cluster_size=c(1, 1, 2, 3)) -vimp_object <- familiar:::prepare_vimp_object(data=data, - vimp_method="pearson", - outcome_type="continuous", - transformation_method="none", - normalisation_method="none", - cluster_method="hclust", - cluster_cut_method="fixed_cut", - cluster_similarity_metric="mcfadden_r2", - cluster_similarity_threshold=0.99, - imputation_method="simple") - +vimp_object <- familiar:::do.call_with_handlers( + familiar:::prepare_vimp_object, + args = list(data=data, + vimp_method="pearson", + outcome_type="continuous", + transformation_method="none", + normalisation_method="none", + cluster_method="hclust", + cluster_cut_method="fixed_cut", + cluster_similarity_metric="mcfadden_r2", + cluster_similarity_threshold=0.99, + imputation_method="simple") +) +if (!test_object_package_installed(vimp_object)) testthat::skip() +vimp_object <- vimp_object$value + vimp_table_object <- suppressWarnings(familiar:::.vimp(vimp_object, data)) From a9f41969c18553c2ca7bb22fce8119bc7384e963 Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 11:23:23 +0200 Subject: [PATCH 4/8] Add checks to test-collection_labels --- tests/testthat/test-collection_labels.R | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/testthat/test-collection_labels.R b/tests/testthat/test-collection_labels.R index da5ff7c6..50cc6d07 100644 --- a/tests/testthat/test-collection_labels.R +++ b/tests/testthat/test-collection_labels.R @@ -19,15 +19,20 @@ for(outcome_type in c("continuous", "multinomial", "survival")){ skip_data_elements <- c("ice_data", "permutation_vimp") # Create experiment data. - experiment_data <- familiar::summon_familiar( - data=data, - experimental_design="fs+mb", - fs_method="mim", - imputation_method="simple", - learner=learner, - estimation_type="point", - skip_evaluation_elements=skip_data_elements, - parallel=FALSE) + experiment_data <- familiar:::do.call_with_handlers( + familiar::summon_familiar, + args = list(data=data, + experimental_design="fs+mb", + fs_method="mim", + imputation_method="simple", + learner=learner, + estimation_type="point", + skip_evaluation_elements=skip_data_elements, + parallel=FALSE) + ) + + if (!test_object_package_installed(experiment_data)) next + experiment_data <- experiment_data$value # Test both familiarCollection and familiarModel objects. familiar_collection_list <- list( From 9bcba508f3627599d017d84933473cc434dbeb09 Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 11:36:51 +0200 Subject: [PATCH 5/8] Check for packages being installed. --- R/TestFunctions.R | 154 +++++++++++++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 56 deletions(-) diff --git a/R/TestFunctions.R b/R/TestFunctions.R index bc1cdd32..3f962187 100644 --- a/R/TestFunctions.R +++ b/R/TestFunctions.R @@ -127,13 +127,18 @@ test_all_learners_train_predict_vimp <- function( #### Full dataset -------------------------------------------------------- # Train the model. - model <- suppressWarnings(test_train(data=full_data, - cluster_method="none", - imputation_method="simple", - hyperparameter_list=hyperparameters, - learner=learner, - time_max=1832, - trim_model=FALSE)) + model <- do.call_with_handlers( + test_train, + args = list(data=full_data, + cluster_method="none", + imputation_method="simple", + hyperparameter_list=hyperparameters, + learner=learner, + time_max=1832, + trim_model=FALSE) + ) + if (!test_object_package_installed(model)) next + model <- model$value # Create a trimmed model -- this is the only instance were we do that # without setting the time-out to infinite to test whether the timeout @@ -1266,11 +1271,15 @@ test_all_novelty_detectors <- function(detectors, #####Full dataset######################################################### # Train the novelty detector. - model <- suppressWarnings(test_train_novelty_detector(data=full_data, - cluster_method="none", - imputation_method="simple", - hyperparameter_list=hyperparameter_list, - detector=detector)) + model <- do.call_with_handlers( + test_train_novelty_detector, + args = list(data=full_data, + cluster_method="none", + imputation_method="simple", + hyperparameter_list=hyperparameter_list, + detector=detector)) + if (!test_object_package_installed(model)) next + model <- model$value # Create a trimmed detector. trimmed_model <- trim_model(model, timeout=Inf) @@ -2080,12 +2089,17 @@ test_all_metrics <- function(metrics, #####Full dataset######################################################### # Train the model. - model <- suppressWarnings(test_train(data=full_data, - cluster_method="none", - imputation_method="simple", - hyperparameter_list=hyperparameters, - learner="glm", - time_max=1832)) + model <- do.call_with_handlers( + test_train, + args = list(data=full_data, + cluster_method="none", + imputation_method="simple", + hyperparameter_list=hyperparameters, + learner="glm", + time_max=1832)) + + if (!test_object_package_installed(model)) next + model <- model$value # Create metric object metric_object <- as_metric(metric=metric, @@ -3578,15 +3592,20 @@ test_plots <- function(plot_function, #####Full data set######################################################## # Train the model. - model_full_1 <- suppressWarnings(test_train(cl=cl, - data=full_data, - cluster_method="none", - imputation_method="simple", - fs_method="mim", - hyperparameter_list=hyperparameters, - learner="lasso", - time_max=1832, - create_novelty_detector=create_novelty_detector)) + model_full_1 <- do.call_with_handlers( + test_train, + args = list(cl=cl, + data=full_data, + cluster_method="none", + imputation_method="simple", + fs_method="mim", + hyperparameter_list=hyperparameters, + learner="lasso", + time_max=1832, + create_novelty_detector=create_novelty_detector)) + + if (!test_object_package_installed(model_full_1)) next + model_full_1 <- model_full_1$value model_full_2 <- model_full_1 model_full_2@fs_method <- "mifs" @@ -4254,6 +4273,8 @@ test_plot_ordering <- function( # Iterate over the outcome type. for(outcome_type in outcome_type_available){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) empty_data <- test.create_empty_data_set(outcome_type) @@ -4271,8 +4292,9 @@ test_plot_ordering <- function( "survival"="cox")) # Train the lasso model. - model_full_lasso_1 <- suppressWarnings( - do.call( + + + model_full_lasso_1 <- do.call_with_handlers( test_train, args=c( list( @@ -4280,7 +4302,10 @@ test_plot_ordering <- function( "hyperparameter_list"=hyperparameters_lasso, "learner"="lasso", "create_novelty_detector"=create_novelty_detector), - experiment_args))) + experiment_args)) + + if (!test_object_package_installed(model_full_lasso_1)) next + model_full_lasso_1 <- model_full_lasso_1$value model_full_lasso_2 <- model_full_lasso_1 model_full_lasso_2@fs_method <- "mifs" @@ -4445,6 +4470,8 @@ test_export <- function(export_function, # Iterate over the outcome type. for(outcome_type in c("count", "continuous", "binomial", "multinomial", "survival")){ + if (!test_data_package_installed(outcome_type)) next + # Obtain data. full_data <- test.create_good_data_set(outcome_type) identical_sample_data <- test.create_all_identical_data_set(outcome_type) @@ -4499,15 +4526,20 @@ test_export <- function(export_function, if(n_models == 1){ # Train the model. - model_full_1 <- suppressWarnings(test_train(cl=cl, - data=full_data, - cluster_method="none", - imputation_method="simple", - fs_method="mim", - hyperparameter_list=hyperparameters, - learner="lasso", - time_max=1832, - create_novelty_detector=create_novelty_detector)) + model_full_1 <- do.call_with_handlers( + test_train, + args = list(cl=cl, + data=full_data, + cluster_method="none", + imputation_method="simple", + fs_method="mim", + hyperparameter_list=hyperparameters, + learner="lasso", + time_max=1832, + create_novelty_detector=create_novelty_detector)) + + if (!test_object_package_installed(model_full_1)) next + model_full_1 <- model_full_1$value model_full_2 <- model_full_1 model_full_2@fs_method <- "mifs" @@ -5233,14 +5265,19 @@ test_export_specific <- function(export_function, if(n_models == 1){ # Train the model. - model_full_1 <- suppressWarnings(test_train(data=main_data, - cluster_method="none", - imputation_method="simple", - fs_method="mim", - hyperparameter_list=hyperparameters, - learner="lasso", - time_max=1832, - create_novelty_detector=create_novelty_detector)) + model_full_1 <- do.call_with_handlers( + test_train, + args = list(data=main_data, + cluster_method="none", + imputation_method="simple", + fs_method="mim", + hyperparameter_list=hyperparameters, + learner="lasso", + time_max=1832, + create_novelty_detector=create_novelty_detector)) + + if (!test_object_package_installed(model_full_1)) next + model_full_1 <- model_full_1$value model_full_2 <- model_full_1 model_full_2@fs_method <- "mifs" @@ -5251,15 +5288,20 @@ test_export_specific <- function(export_function, model_full_2 <- list() for(ii in seq_len(n_models)){ - temp_model_1 <- suppressWarnings(test_train(data=main_data, - cluster_method="none", - imputation_method="simple", - fs_method="mim", - hyperparameter_list=hyperparameters, - learner="lasso", - time_max=1832, - create_bootstrap=TRUE, - create_novelty_detector=create_novelty_detector)) + temp_model_1 <- do.call_with_handlers( + test_train, + args = list(data=main_data, + cluster_method="none", + imputation_method="simple", + fs_method="mim", + hyperparameter_list=hyperparameters, + learner="lasso", + time_max=1832, + create_bootstrap=TRUE, + create_novelty_detector=create_novelty_detector)) + + if (!test_object_package_installed(temp_model_1)) next + temp_model_1 <- temp_model_1$value temp_model_2 <- temp_model_1 temp_model_2@fs_method <- "mifs" From 355e3f1b430b3192dc9e75cf1bb12e17baa405ca Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 11:44:13 +0200 Subject: [PATCH 6/8] Update news and description. --- DESCRIPTION | 2 +- NEWS.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 88d82ddd..16bf4b75 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: familiar Title: End-to-End Automated Machine Learning and Model Evaluation -Version: 1.4.7 +Version: 1.4.8 Authors@R: c( person("Alex", "Zwanenburg", email = "alexander.zwanenburg@nct-dresden.de", diff --git a/NEWS.md b/NEWS.md index f4fb9ac2..37a6d50d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +# Version 1.4.8 (Valorous Viper) + +## Bug fixes + +- Adapted tests to work when packages are missing. + # Version 1.4.7 (Uncertain Unicorn) ## Bug fixes From f650fd983cded57626fa78c54b20a0a0d4f0eb65 Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 11:49:36 +0200 Subject: [PATCH 7/8] Update auto-test-no-suggests-pull.yml --- .github/workflows/auto-test-no-suggests-pull.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/auto-test-no-suggests-pull.yml b/.github/workflows/auto-test-no-suggests-pull.yml index 2878e773..a54cac78 100644 --- a/.github/workflows/auto-test-no-suggests-pull.yml +++ b/.github/workflows/auto-test-no-suggests-pull.yml @@ -6,7 +6,7 @@ # See https://github.com/r-lib/actions/tree/master/examples#readme for # additional example workflows available for the R community. -name: Run R CMD check for no +name: Run R CMD check for strong dependencies only (nosuggests) on: [pull_request, workflow_dispatch] @@ -39,6 +39,7 @@ jobs: any::knitr any::rmarkdown any::ggplot2 + any::roxygen2 needs: check - name: Create documentation From 709ebc3fafc2256b6eb304642a65b92c7d260261 Mon Sep 17 00:00:00 2001 From: Alex Zwanenburg Date: Wed, 15 May 2024 16:11:07 +0200 Subject: [PATCH 8/8] Update cran-comments.md --- cran-comments.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cran-comments.md b/cran-comments.md index f8749c1b..cfa3307c 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,4 +1,4 @@ -Update familiar to version 1.4.7 +Update familiar to version 1.4.8 Vignettes are pre-compiled to avoid long compilation times on build (several minutes). @@ -8,7 +8,7 @@ Longer tests and tests involving parallel processing are not performed on CRAN, ## R CMD check results -R CMD check was run on GitHub against R-release using r-lib/actions/check-r-package: +R CMD check was run on GitHub using https://github.com/alexzwanenburg/familiar/actions/workflows/auto-test-package_pull.yml ---------------------------------- window-latest: @@ -28,7 +28,20 @@ macos-latest ubuntu-latest: 0 errors | 0 warnings | 0 notes +## R CMD check for strong dependencies only +R CMD check was run on GitHub using https://github.com/alexzwanenburg/familiar/actions/workflows/auto-test-no-suggests-pull.yml + +---------------------------------- +ubuntu-latest: +0 errors | 0 warnings | 1 note + +* checking package dependencies ... NOTE +Packages suggested but not available for checking: + 'BART', 'CORElearn', 'coro', 'dynamicTreeCut', 'e1071', 'Ecdat', + 'fastcluster', 'fastglm', 'glmnet', 'harmonicmeanp', 'isotree', + 'laGP', 'maxstat', 'mboost', 'microbenchmark', 'partykit', 'praznik', + 'proxy', 'qvalue', 'randomForestSRC', 'ranger', 'VGAM', 'xgboost' ## Downstream dependencies