diff --git a/data-raw/benchmark.R b/data-raw/benchmark.R index e5d107ac..6548d27a 100644 --- a/data-raw/benchmark.R +++ b/data-raw/benchmark.R @@ -38,6 +38,9 @@ for (version in c("CRAN", "main", "branch")) { missing <- jsonlite::fromJSON("SCDB.lock")$packages$ref %>% purrr::discard(rlang::is_installed) if (length(missing) > 0) pak::pkg_install(missing, lib = lib_path) + + # Explicitly install the packages + pak::pkg_install(source, lib = lib_path, dependencies = FALSE) } @@ -66,8 +69,12 @@ if (identical(Sys.getenv("CI"), "true") && identical(Sys.getenv("BACKEND"), "")) version == "branch" ~ glue::glue("ssi-dk-SCDB-{sha}") ) - .libPaths(c(here::here("installations", lib_dir), lib_paths_default)) - library("SCDB") + library("SCDB", lib.loc = here::here("installations", lib_dir)) + + # Add proper version labels to the benchmarks + if (version == "CRAN") { + version <- paste0("SCDB v", packageVersion("SCDB")) + } # Open connection to the database conns <- get_test_conns() @@ -137,8 +144,11 @@ if (identical(Sys.getenv("CI"), "true") && identical(Sys.getenv("BACKEND"), "")) "n" = n ) - dir.create("data", showWarnings = FALSE) - saveRDS(update_snapshot_benchmark, glue::glue("data/benchmark-update_snapshot_{names(conns)[[1]]}_{version}.rds")) + dir.create("inst/extdata", showWarnings = FALSE, recursive = TRUE) + saveRDS( + update_snapshot_benchmark, + glue::glue("inst/extdata/benchmark-update_snapshot_{names(conns)[[1]]}_{version}.rds") + ) }) # Benchmark 2, update_snapshot() with increasing data size @@ -164,10 +174,10 @@ if (identical(Sys.getenv("CI"), "true") && identical(Sys.getenv("BACKEND"), "")) "n" = n ) - dir.create("data", showWarnings = FALSE) + dir.create("inst/extdata", showWarnings = FALSE, recursive = TRUE) saveRDS( update_snapshot_benchmark, - glue::glue("data/benchmark-update_snapshot_complexity_{n}_{names(conns)[[1]]}_{version}.rds") + glue::glue("inst/extdata/benchmark-update_snapshot_complexity_{n}_{names(conns)[[1]]}_{version}.rds") ) } diff --git a/inst/extdata/benchmarks.rds b/inst/extdata/benchmarks.rds index 47076795..53a635a1 100644 Binary files a/inst/extdata/benchmarks.rds and b/inst/extdata/benchmarks.rds differ diff --git a/pak.lock b/pak.lock index 6703d773..98095105 100644 --- a/pak.lock +++ b/pak.lock @@ -131,7 +131,7 @@ { "ref": "bit64", "package": "bit64", - "version": "4.5.2", + "version": "4.6.0-1", "type": "standard", "direct": false, "binary": true, @@ -143,10 +143,10 @@ "RemoteRef": "bit64", "RemoteRepos": "https://packagemanager.posit.co/cran/__linux__/noble/latest", "RemotePkgPlatform": "x86_64-pc-linux-gnu-ubuntu-24.04", - "RemoteSha": "4.5.2" + "RemoteSha": "4.6.0-1" }, - "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/bit64_4.5.2.tar.gz", - "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/bit64_4.5.2.tar.gz", + "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/bit64_4.6.0-1.tar.gz", + "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/bit64_4.6.0-1.tar.gz", "platform": "x86_64-pc-linux-gnu-ubuntu-24.04", "rversion": "4.4", "directpkg": false, @@ -2887,7 +2887,7 @@ { "ref": "pak", "package": "pak", - "version": "0.8.0", + "version": "0.8.0.1", "type": "standard", "direct": false, "binary": true, @@ -2899,10 +2899,10 @@ "RemoteRef": "pak", "RemoteRepos": "https://packagemanager.posit.co/cran/__linux__/noble/latest", "RemotePkgPlatform": "x86_64-pc-linux-gnu-ubuntu-24.04", - "RemoteSha": "0.8.0" + "RemoteSha": "0.8.0.1" }, - "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/pak_0.8.0.tar.gz", - "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/pak_0.8.0.tar.gz", + "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/pak_0.8.0.1.tar.gz", + "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/pak_0.8.0.1.tar.gz", "platform": "x86_64-pc-linux-gnu-ubuntu-24.04", "rversion": "4.4", "directpkg": false, @@ -2980,7 +2980,7 @@ { "ref": "pkgbuild", "package": "pkgbuild", - "version": "1.4.5", + "version": "1.4.6", "type": "standard", "direct": false, "binary": true, @@ -2992,10 +2992,10 @@ "RemoteRef": "pkgbuild", "RemoteRepos": "https://packagemanager.posit.co/cran/__linux__/noble/latest", "RemotePkgPlatform": "x86_64-pc-linux-gnu-ubuntu-24.04", - "RemoteSha": "1.4.5" + "RemoteSha": "1.4.6" }, - "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/pkgbuild_1.4.5.tar.gz", - "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/pkgbuild_1.4.5.tar.gz", + "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/pkgbuild_1.4.6.tar.gz", + "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/pkgbuild_1.4.6.tar.gz", "platform": "x86_64-pc-linux-gnu-ubuntu-24.04", "rversion": "4.4", "directpkg": false, @@ -3508,7 +3508,7 @@ { "ref": "Rcpp", "package": "Rcpp", - "version": "1.0.13-1", + "version": "1.0.14", "type": "standard", "direct": false, "binary": true, @@ -3520,10 +3520,10 @@ "RemoteRef": "Rcpp", "RemoteRepos": "https://packagemanager.posit.co/cran/__linux__/noble/latest", "RemotePkgPlatform": "x86_64-pc-linux-gnu-ubuntu-24.04", - "RemoteSha": "1.0.13-1" + "RemoteSha": "1.0.14" }, - "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/Rcpp_1.0.13-1.tar.gz", - "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/Rcpp_1.0.13-1.tar.gz", + "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/Rcpp_1.0.14.tar.gz", + "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/Rcpp_1.0.14.tar.gz", "platform": "x86_64-pc-linux-gnu-ubuntu-24.04", "rversion": "4.4", "directpkg": false, @@ -4387,11 +4387,11 @@ { "ref": "systemfonts", "package": "systemfonts", - "version": "1.1.0", + "version": "1.2.0", "type": "standard", "direct": false, "binary": true, - "dependencies": "lifecycle", + "dependencies": ["jsonlite", "lifecycle"], "vignettes": false, "metadata": { "RemotePkgRef": "systemfonts", @@ -4399,10 +4399,10 @@ "RemoteRef": "systemfonts", "RemoteRepos": "https://packagemanager.posit.co/cran/__linux__/noble/latest", "RemotePkgPlatform": "x86_64-pc-linux-gnu-ubuntu-24.04", - "RemoteSha": "1.1.0" + "RemoteSha": "1.2.0" }, - "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/systemfonts_1.1.0.tar.gz", - "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/systemfonts_1.1.0.tar.gz", + "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/systemfonts_1.2.0.tar.gz", + "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/systemfonts_1.2.0.tar.gz", "platform": "x86_64-pc-linux-gnu-ubuntu-24.04", "rversion": "4.4", "directpkg": false, @@ -4431,7 +4431,7 @@ { "ref": "testthat", "package": "testthat", - "version": "3.2.2", + "version": "3.2.3", "type": "standard", "direct": false, "binary": true, @@ -4443,10 +4443,10 @@ "RemoteRef": "testthat", "RemoteRepos": "https://packagemanager.posit.co/cran/__linux__/noble/latest", "RemotePkgPlatform": "x86_64-pc-linux-gnu-ubuntu-24.04", - "RemoteSha": "3.2.2" + "RemoteSha": "3.2.3" }, - "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/testthat_3.2.2.tar.gz", - "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/testthat_3.2.2.tar.gz", + "sources": "https://packagemanager.posit.co/cran/__linux__/noble/latest/src/contrib/testthat_3.2.3.tar.gz", + "target": "src/contrib/x86_64-pc-linux-gnu-ubuntu-24.04/4.4/testthat_3.2.3.tar.gz", "platform": "x86_64-pc-linux-gnu-ubuntu-24.04", "rversion": "4.4", "directpkg": false, diff --git a/vignettes/benchmarks.Rmd b/vignettes/benchmarks.Rmd index 45fd1970..9ffdb8c5 100644 --- a/vignettes/benchmarks.Rmd +++ b/vignettes/benchmarks.Rmd @@ -14,6 +14,10 @@ knitr::opts_chunk$set( comment = "#>" ) +# Set a flag to determine if the vignette should be run in simple context (vignettes) +# or in an extended context (pkgdown) +simple_rendering <- !pkgdown::in_pkgdown() + # NOTE: # To re-run the benchmarks, run the "benchmark" workflow on GitHub ``` @@ -41,8 +45,21 @@ then update it to the second and third snapshot. Finally, the table is deleted. The performance of this benchmark function is timed with the `{microbenchmark}` package using 10 replicates. All benchmarks are run on the same machine. -The results of the benchmark are shown graphically below (mean and standard deviation), where we compare the current -development version of `SCDB` with the current CRAN version. +```{r benchmark context, results = "asis", include = FALSE} +if (simple_rendering) { + cat( + "The results of the benchmark are shown graphically below", + "(mean and standard deviation), where measure the", + "performance of `SCDB`." + ) +} else { + cat( + "The results of the benchmark are shown graphically below", + "(mean and standard deviation), where we compare the", + "current development version of `SCDB` with the current CRAN version." + ) +} +``` ```{r benchmark_preprocessing, echo = FALSE, eval = requireNamespace("here")} benchmark_location <- c( @@ -52,12 +69,13 @@ benchmark_location <- c( purrr::discard(~ identical(., "")) %>% purrr::pluck(1) -benchmarks <- readRDS(benchmark_location) +benchmarks <- readRDS(benchmark_location) %>% + dplyr::mutate("version" = as.character(.data$version)) # Determine if the SHA is on main sha <- benchmarks %>% - dplyr::distinct(version) %>% - dplyr::filter(!(version %in% c("CRAN", "main", "branch"))) %>% + dplyr::distinct(.data$version) %>% + dplyr::filter(!startsWith(.data$version, "SCDB"), .data$version != "main") %>% dplyr::pull("version") # Check local git history @@ -71,13 +89,19 @@ on_main <- tryCatch({ return(identical(Sys.getenv("CI"), "true")) }) -# If the SHA has been merged, use as the "main" version and remove the other, -# older, main version -if (on_main) { +# In the simple context we use the newest benchmark (version = sha) +# This benchmark is then labelled with the newest version number of SCDB +if (simple_rendering) { + benchmarks <- benchmarks %>% + dplyr::filter(.data$version == !!sha) %>% + dplyr::mutate("version" = paste0("SCDB v", packageVersion("SCDB"))) +} else if (on_main) { + # If the SHA has been merged, use as the "main" version and remove the other, + # older, main version benchmarks <- benchmarks %>% dplyr::filter(.data$version != "main") %>% dplyr::mutate( - "version" = dplyr::if_else(.data$version == "CRAN", "CRAN", "development") + "version" = dplyr::if_else(.data$version == sha, "development", .data$version) ) } @@ -96,45 +120,47 @@ benchmark_1 <- benchmarks %>% !stringr::str_ends(.data$benchmark_function, stringr::fixed("complexity")) ) -# Add note slow backends -slow_backends <- benchmark_1 %>% - dplyr::distinct(.data$database, .data$n) %>% - dplyr::filter(.data$n < max(.data$n)) %>% - dplyr::pull("database") - -benchmark_1 <- benchmark_1 %>% - dplyr::mutate( - "database" = paste0(database, ifelse(database %in% slow_backends, "*", "")) - ) - # Insert newline into database name to improve rendering of figures labeller <- ggplot2::as_labeller( function(l) stringr::str_replace_all(l, stringr::fixed(" v"), "\nv") ) - +# Apply "dodging" to sub-groups to show graphically +dodge <- ggplot2::position_dodge(width = 0.6) g <- ggplot2::ggplot( benchmark_1, - ggplot2::aes(x = version, y = time / 1e9) + ggplot2::aes(x = version, y = time / 1e9, color = database) ) + ggplot2::stat_summary( fun.data = mean_sd, geom = "pointrange", size = 0.5, - linewidth = 1 - ) + - ggplot2::facet_grid( - rows = ggplot2::vars(benchmark_function), - cols = ggplot2::vars(database), - labeller = labeller + linewidth = 1, + position = dodge ) + - ggplot2::labs(x = "Codebase version", y = "Time (s)") + ggplot2::scale_x_discrete(guide = ggplot2::guide_axis(n.dodge = 2)) + + ggplot2::labs(x = "Codebase version", y = "Time (s)") + + ggplot2::theme(legend.position = "bottom") -if (length(slow_backends) > 1) { - g <- g + ggplot2::labs( - caption = "* IMPORTANT: Benchmark data halved for this backend!" + +if (simple_rendering) { + # Reduce font size for simple version + g <- g + ggplot2::theme(text = ggplot2::element_text(size = 8)) + + # Make the legend two rows + g <- g + ggplot2::guides( + color = ggplot2::guide_legend(title = "", nrow = 2, byrow = TRUE) ) + +} else { + # Add facets to extended rendering + g <- g + + ggplot2::facet_grid( + rows = ggplot2::vars(benchmark_function), + cols = ggplot2::vars(database), + labeller = labeller + ) } g @@ -171,26 +197,54 @@ benchmark_2 <- benchmarks %>% ) ) -ggplot2::ggplot( + +# Apply "dodging" to sub-groups to show graphically +dodge <- ggplot2::position_dodge(width = 0.6) + +# Set aesthetics for simple and extended versions +if (simple_rendering) { + aes <- ggplot2::aes(x = n * nrow(iris) / 1e3, y = time / 1e9, color = database) +} else { + aes <- ggplot2::aes(x = n * nrow(iris) / 1e3, y = time / 1e9, color = version) +} + +g <- ggplot2::ggplot( benchmark_2, - ggplot2::aes(x = n * nrow(iris) / 1e3, y = time / 1e9, color = version) + aes ) + ggplot2::stat_summary( fun.data = mean_sd, geom = "pointrange", - size = 0.5, - linewidth = 1 + size = 0.5, linewidth = 1, + position = dodge ) + ggplot2::geom_smooth(method = "lm", formula = y ~ x, se = FALSE, linetype = 3) + - ggplot2::facet_grid( - rows = ggplot2::vars(benchmark_function), - cols = ggplot2::vars(database), - labeller = labeller - ) + ggplot2::labs( x = "Data size (1,000 rows)", y = "Time (s)", color = "Codebase version" ) + ggplot2::theme(panel.spacing = grid::unit(1, "lines"), legend.position = "bottom") + + +if (simple_rendering) { + # Reduce font size for simple version + g <- g + ggplot2::theme(text = ggplot2::element_text(size = 8)) + + # Make the legend two rows + g <- g + ggplot2::guides( + color = ggplot2::guide_legend(title = "", nrow = 2, byrow = TRUE) + ) + +} else { + # Add facets to extended rendering + g <- g + + ggplot2::facet_grid( + rows = ggplot2::vars(benchmark_function), + cols = ggplot2::vars(database), + labeller = labeller + ) +} + +g ```