diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml new file mode 100644 index 00000000..3b158351 --- /dev/null +++ b/.github/workflows/test-coverage.yaml @@ -0,0 +1,32 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master, development] + pull_request: + branches: [main, master] + +name: test-coverage + +jobs: + test-coverage: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::covr + needs: coverage + + - name: Test coverage + run: covr::codecov(token = Sys.getenv("CODECOV_TOKEN"), quiet = FALSE) + shell: Rscript {0} diff --git a/DESCRIPTION b/DESCRIPTION index 1cd5f458..a5d33d8d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,10 +1,11 @@ Type: Package Package: pxweb Title: R Interface to PXWEB APIs -Version: 0.13.1 -Date: 2022-03-10 +Version: 0.15.0 +Date: 2022-07-01 Authors@R: c( - person("Mans", "Magnusson", , "mons.magnusson@gmail.com", role = c("aut", "cre")), + person("Mans", "Magnusson", , "mons.magnusson@gmail.com", role = c("aut", "cre"), + comment = c(ORCID = "0000-0002-0296-2719")), person("Markus", "Kainu", role = "aut"), person("Janne", "Huovari", role = "aut"), person("Leo", "Lahti", role = "aut", @@ -38,6 +39,7 @@ Imports: jsonlite Suggests: knitr, + digest, remotes, rmarkdown, testthat (>= 0.11), @@ -45,6 +47,6 @@ Suggests: VignetteBuilder: knitr Encoding: UTF-8 -RoxygenNote: 7.1.1 +RoxygenNote: 7.2.0 X-schema.org-isPartOf: http://ropengov.org/ X-schema.org-keywords: ropengov diff --git a/NAMESPACE b/NAMESPACE index 05114222..677e9e18 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -20,6 +20,7 @@ S3method(pxweb_query,json) S3method(pxweb_query,list) S3method(pxweb_query,pxweb_explorer) S3method(pxweb_query,pxweb_query) +S3method(pxweb_query,response) export(api_catalogue) export(api_parameters) export(base_url) @@ -42,6 +43,7 @@ export(pxweb_get_data) export(pxweb_interactive) export(pxweb_query) export(pxweb_query_as_json) +export(pxweb_query_as_rcode) export(pxweb_test_api) export(pxweb_validate_query_with_metadata) export(update_pxweb_apis) diff --git a/NEWS.md b/NEWS.md index 26ae829b..bc4af417 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,14 @@ +# Version 0.15.0 + +- Added possibility to download px and sdmx response formats as files +- Added StatSI API +- Added codecov code coverage stats +- Fixed some minor API configs + +# Version 0.14.0 + +- Added feature to print pxweb_query objects to R code + # Version 0.10.4 - Bug fixes diff --git a/R/pxweb_c.R b/R/pxweb_c.R index 1a149a56..118051a8 100644 --- a/R/pxweb_c.R +++ b/R/pxweb_c.R @@ -17,6 +17,17 @@ pxweb_c <- function(x){ return(x) } + if(inherits(x[[1]], "character")) { + fp <- unlist(x) + fe <- file.exists(fp) + if(all(fe)){ + message("PXWEB API did not return JSON. Files has been stored locally (tempdir) and paths has been returned.") + return(x) + } else { + stop("Files doesn't exist:\n", paste(fp[!fe], collapse = "\n"), call. = FALSE) + } + } + stop("pxweb_c() not implemented for class '", class(x[[1]])[1], "'.", call. = FALSE) } diff --git a/R/pxweb_interactive.R b/R/pxweb_interactive.R index 9da439b8..595ea573 100644 --- a/R/pxweb_interactive.R +++ b/R/pxweb_interactive.R @@ -985,7 +985,7 @@ pxe_print_download_code <- function(pxe, as){ if(as == "r"){ cat("# PXWEB query \n") q_path <- "pxweb_query_list" - cat(pxweb_query_as_rcode(q), sep ="\n") + pxweb_query_as_rcode(q) cat("\n") } cat("# Download data \n", diff --git a/R/pxweb_parse_response.R b/R/pxweb_parse_response.R index 892c8b70..08a39b69 100644 --- a/R/pxweb_parse_response.R +++ b/R/pxweb_parse_response.R @@ -6,7 +6,15 @@ pxweb_parse_response <- function(x){ checkmate::assert_class(x, "response") - obj <- suppressWarnings(httr::content(x, as = "parsed")) + pxq <- pxweb_query(x) + if(is.null(pxq) || pxq$response %in% c("json", "json-stat")){ + obj <- suppressWarnings(httr::content(x, as = "parsed")) + } else if (pxq$response %in% pxweb_file_response_formats()){ + obj <- suppressWarnings(httr::content(x, as = "raw")) + obj_path <- file.path(tempdir(), paste0(digest::sha1(obj), ".", pxq$response)) + writeBin(con = obj_path, object = obj) + return(obj_path) + } try_obj <- try(pxweb_database_list(obj), silent = TRUE) if(!inherits(try_obj, "try-error")) { diff --git a/R/pxweb_query.R b/R/pxweb_query.R index 60760d39..ab90e035 100644 --- a/R/pxweb_query.R +++ b/R/pxweb_query.R @@ -6,16 +6,20 @@ #' #' @param x an object to cast as a pxweb_query object. #' +#' @seealso \code{\link{pxweb_query_as_json}}, \code{\link{pxweb_query_as_rcode}} +#' #' @examples #' dims <- list(Alue = c("*"), #' "Asuntokunnan koko" = c("*"), #' Talotyyppi = c("S"), #' Vuosi = c("*")) #' pxq1 <- pxweb_query(dims) +#' #' json_query <- file.path(system.file(package = "pxweb"), #' "extdata", "examples", "json_query_example.json") #' pxq2 <- pxweb_query(json_query) #' +#' #' @export pxweb_query <- function(x){ UseMethod("pxweb_query") @@ -31,15 +35,22 @@ pxweb_query.character <- function(x){ } class(obj) <- c("pxweb_query", "list") assert_pxweb_query(obj, check_response_format = FALSE) - if(tolower(obj$response$format) %in% c("json-stat", "jsonstat")){ - obj$response$format <- "json-stat" + if(tolower(obj$response$format) == "json"){ + obj$response$format <- "json" + } else if (tolower(obj$response$format) %in% c("json-stat", "jsonstat")){ + obj$response$format <- "json-stat" + } else if (tolower(obj$response$format) %in% pxweb_file_response_formats()){ + } else { - obj$response$format <- "json" + warning(paste0("'", obj$response$format, "' is not a valid query response format, set to 'json'.")) + obj$response$format <- "json" } assert_pxweb_query(obj) obj } +pxweb_file_response_formats <- function() c("px", "sdmx") + #' @rdname pxweb_query #' @keywords internal #' @export @@ -74,6 +85,15 @@ pxweb_query.list <- function(x){ obj } +#' @rdname pxweb_query +#' @keywords internal +#' @export +pxweb_query.response <- function(x){ + if(is.null(x$request$options$postfields)) return(NULL) + pxweb_query(x = readBin(x$request$options$postfields, what = "character")) +} + + #' @rdname pxweb_query #' @keywords internal #' @export @@ -106,9 +126,9 @@ assert_pxweb_query <- function(x, check_response_format = TRUE){ checkmate::assert_names(names(x), must.include = c("query", "response"), .var.name = "names(pxweb_query)") checkmate::assert_names(names(x$response), must.include = c("format")) if(check_response_format){ - checkmate::assert_choice(x$response$format, c("json", "json-stat")) + checkmate::assert_choice(x$response$format, c("json", "json-stat", pxweb_file_response_formats())) } - + checkmate::assert_named(x$query, "unnamed") for(i in seq_along(x$query)){ @@ -318,6 +338,8 @@ pxweb_query_filter <- function(pxq){ #' @param pxq a \code{pxweb_query} object. #' @param ... further argument to \code{jsonlite::toJSON()}. #' +#' @seealso \code{\link{pxweb_query}}, \code{\link{pxweb_query_as_rcode}} +#' #' @examples #' json_query <- file.path(system.file(package = "pxweb"), #' "extdata", "examples", "json_query_example.json") @@ -335,10 +357,13 @@ pxweb_query_as_json <- function(pxq, ...){ jsonlite::toJSON(pxq, ...) } -#' Convert a \code{pxweb_query} object to R code -#' @details One element per row is returned. +#' Print a \code{pxweb_query} object as R code +#' #' @param pxq a \code{pxweb_query} object. -#' @keywords internal +#' +#' @seealso \code{\link{pxweb_query_as_json}}, \code{\link{pxweb_query}} +#' +#' @export pxweb_query_as_rcode <- function(pxq){ checkmate::assert_class(pxq, "pxweb_query") @@ -356,6 +381,6 @@ pxweb_query_as_rcode <- function(pxq){ res[-length(res)] <- paste0(res[-length(res)], ",") } res <- c("pxweb_query_list <- ", res) - - res + cat(res, sep ="\n") + invisible(res) } diff --git a/README.Rmd b/README.Rmd index ebcbdcc7..6d710467 100644 --- a/README.Rmd +++ b/README.Rmd @@ -15,7 +15,7 @@ knitr::opts_chunk$set( [![rOG-badge](https://ropengov.github.io/rogtemplate/reference/figures/ropengov-badge.svg)](http://ropengov.org/) [![R build status](https://github.com/rOpenGov/pxweb/workflows/R-CMD-check/badge.svg)](https://github.com/rOpenGov/pxweb/actions) -[![Coverage Status](https://coveralls.io/repos/github/rOpenGov/pxweb/badge.svg?branch=master)](https://coveralls.io/github/rOpenGov/pxweb?branch=master) +[![codecov](https://codecov.io/gh/rOpenGov/pxweb/branch/master/graph/badge.svg?token=zYtxsus27g)](https://codecov.io/gh/rOpenGov/pxweb) [![Downloads](http://cranlogs.r-pkg.org/badges/grand-total/pxweb)](https://cran.r-project.org/package=pxweb) [![Downloads](http://cranlogs.r-pkg.org/badges/pxweb)](https://cran.r-project.org/package=pxweb) [![Gitter](https://badges.gitter.im/rOpenGov/pxweb.svg)](https://gitter.im/rOpenGov/pxweb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/README.md b/README.md index 35ed2bd6..26ae2946 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,7 @@ [![rOG-badge](https://ropengov.github.io/rogtemplate/reference/figures/ropengov-badge.svg)](http://ropengov.org/) [![R build status](https://github.com/rOpenGov/pxweb/workflows/R-CMD-check/badge.svg)](https://github.com/rOpenGov/pxweb/actions) -[![Coverage -Status](https://coveralls.io/repos/github/rOpenGov/pxweb/badge.svg?branch=master)](https://coveralls.io/github/rOpenGov/pxweb?branch=master) +[![codecov](https://codecov.io/gh/rOpenGov/pxweb/branch/master/graph/badge.svg?token=zYtxsus27g)](https://codecov.io/gh/rOpenGov/pxweb) [![Downloads](http://cranlogs.r-pkg.org/badges/grand-total/pxweb)](https://cran.r-project.org/package=pxweb) [![Downloads](http://cranlogs.r-pkg.org/badges/pxweb)](https://cran.r-project.org/package=pxweb) [![Gitter](https://badges.gitter.im/rOpenGov/pxweb.svg)](https://gitter.im/rOpenGov/pxweb?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/inst/extdata/api.json b/inst/extdata/api.json index 6c6def0e..f90455a9 100644 --- a/inst/extdata/api.json +++ b/inst/extdata/api.json @@ -26,7 +26,7 @@ "lang": ["en","fi","sv"], "calls_per_period": 20, "period_in_seconds": 10, - "max_values_to_download" : 100000 + "max_values_to_download" : 120000 }, "pxwebapi2.stat.fi": { @@ -400,6 +400,20 @@ "lang": ["en"], "calls_per_period": 100, "period_in_seconds": 10, + "max_values_to_download" : 1000 + }, + + "pxweb.stat.si": { + "description" : "SiStat Database", + "citation" : { + "organization" : "Statistical Office of the Republic of Slovenia", + "address" : "Ljubljana, Slovenia" + }, + "url" : "https://pxweb.stat.si/SiStatData/api/[version]/[lang]/Data", + "version": ["v1"], + "lang": ["sl"], + "calls_per_period": 10, + "period_in_seconds": 10, "max_values_to_download" : 1000000 } }, diff --git a/man/pxweb_query.Rd b/man/pxweb_query.Rd index 4ea5d3ca..45255b3d 100644 --- a/man/pxweb_query.Rd +++ b/man/pxweb_query.Rd @@ -6,6 +6,7 @@ \alias{pxweb_query.json} \alias{pxweb_query.pxweb_query} \alias{pxweb_query.list} +\alias{pxweb_query.response} \alias{pxweb_query.pxweb_explorer} \title{Create a PXWEB query} \usage{ @@ -19,6 +20,8 @@ pxweb_query(x) \method{pxweb_query}{list}(x) +\method{pxweb_query}{response}(x) + \method{pxweb_query}{pxweb_explorer}(x) } \arguments{ @@ -34,9 +37,14 @@ dims <- list(Alue = c("*"), Talotyyppi = c("S"), Vuosi = c("*")) pxq1 <- pxweb_query(dims) + json_query <- file.path(system.file(package = "pxweb"), "extdata", "examples", "json_query_example.json") pxq2 <- pxweb_query(json_query) + +} +\seealso{ +\code{\link{pxweb_query_as_json}}, \code{\link{pxweb_query_as_rcode}} } \keyword{internal} diff --git a/man/pxweb_query_as_json.Rd b/man/pxweb_query_as_json.Rd index 4fea89ec..55409e43 100644 --- a/man/pxweb_query_as_json.Rd +++ b/man/pxweb_query_as_json.Rd @@ -21,3 +21,6 @@ pxq <- pxweb_query(json_query) json <- pxweb_query_as_json(pxq, pretty = TRUE) } +\seealso{ +\code{\link{pxweb_query}}, \code{\link{pxweb_query_as_rcode}} +} diff --git a/man/pxweb_query_as_rcode.Rd b/man/pxweb_query_as_rcode.Rd index be9aa703..43963579 100644 --- a/man/pxweb_query_as_rcode.Rd +++ b/man/pxweb_query_as_rcode.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/pxweb_query.R \name{pxweb_query_as_rcode} \alias{pxweb_query_as_rcode} -\title{Convert a \code{pxweb_query} object to R code} +\title{Print a \code{pxweb_query} object as R code} \usage{ pxweb_query_as_rcode(pxq) } @@ -10,9 +10,8 @@ pxweb_query_as_rcode(pxq) \item{pxq}{a \code{pxweb_query} object.} } \description{ -Convert a \code{pxweb_query} object to R code +Print a \code{pxweb_query} object as R code } -\details{ -One element per row is returned. +\seealso{ +\code{\link{pxweb_query_as_json}}, \code{\link{pxweb_query}} } -\keyword{internal} diff --git a/tests/testthat/test-pxweb_get.R b/tests/testthat/test-pxweb_get.R index 365d99b7..d6f07bd5 100644 --- a/tests/testthat/test-pxweb_get.R +++ b/tests/testthat/test-pxweb_get.R @@ -2,6 +2,35 @@ context("pxweb_get") +test_that(desc="Test to download px and sdmx",{ + # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN. + skip_on_cran() + json_px_query <- readLines(test_path("test_data/test_query_px.json")) + + expect_silent(px_file_path1 <- + pxweb_get(url = "http://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy", + query = json_px_query) + ) + checkmate::expect_file_exists(px_file_path1) + + # data <- pxR::read.px(px_file_path1) + + json_sdmx_query <- readLines(test_path("test_data/test_query_sdmx.json")) + expect_silent(px_file_path2 <- + pxweb_get(url = "http://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy", + query = json_sdmx_query) + ) + checkmate::expect_file_exists(px_file_path2) + expect_true(px_file_path1 != px_file_path2) + + pxq <- pxweb_query(json_px_query) + pxq$response$format <- "sdmx" + pxfp <- pxweb_get(url = "http://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy", + pxq) + expect_true(px_file_path2 == pxfp) + +}) + test_that(desc="Constructor works as it should with Statistics Sweden",{ # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN. skip_on_cran() @@ -244,3 +273,6 @@ test_that(desc="Query with non-ascii characters work as well",{ ) }) + + + diff --git a/tests/testthat/test_data/test_query_px.json b/tests/testthat/test_data/test_query_px.json new file mode 100644 index 00000000..1d2d50e7 --- /dev/null +++ b/tests/testthat/test_data/test_query_px.json @@ -0,0 +1,42 @@ +{ + "query": [ + { + "code": "Region", + "selection": { + "filter": "item", + "values": ["00"] + } + }, + { + "code": "Alder", + "selection": { + "filter": "item", + "values": ["tot"] + } + }, + { + "code": "Kon", + "selection": { + "filter": "item", + "values": ["1", "2"] + } + }, + { + "code": "ContentsCode", + "selection": { + "filter": "item", + "values": ["BE0101N1", "BE0101N2"] + } + }, + { + "code": "Tid", + "selection": { + "filter": "item", + "values": ["2019", "2020", "2021"] + } + } + ], + "response": { + "format": "px" + } +} diff --git a/tests/testthat/test_data/test_query_sdmx.json b/tests/testthat/test_data/test_query_sdmx.json new file mode 100644 index 00000000..6631ec3f --- /dev/null +++ b/tests/testthat/test_data/test_query_sdmx.json @@ -0,0 +1,42 @@ +{ + "query": [ + { + "code": "Region", + "selection": { + "filter": "item", + "values": ["00"] + } + }, + { + "code": "Alder", + "selection": { + "filter": "item", + "values": ["tot"] + } + }, + { + "code": "Kon", + "selection": { + "filter": "item", + "values": ["1", "2"] + } + }, + { + "code": "ContentsCode", + "selection": { + "filter": "item", + "values": ["BE0101N1", "BE0101N2"] + } + }, + { + "code": "Tid", + "selection": { + "filter": "item", + "values": ["2019", "2020", "2021"] + } + } + ], + "response": { + "format": "sdmx" + } +} diff --git a/vignettes/pxweb.Rmd b/vignettes/pxweb.Rmd index c7190e7b..510c0f72 100644 --- a/vignettes/pxweb.Rmd +++ b/vignettes/pxweb.Rmd @@ -18,13 +18,13 @@ API](https://pxnet2.stat.fi/API-description_SCB.pdf). Your welcome! -More information on the PX-Web/PC-Axis API can be found [here](https://www.scb.se/en/services/statistical-programs-for-px-files/px-web/). +We can find more information on the PX-Web/PC-Axis API [here](https://www.scb.se/en/services/statistical-programs-for-px-files/px-web/). ## Introduction -PXWEB is an API structure developed by Statistics Sweden together with other national statistical institutions (NSI) to disseminate public statistics in a structured way. This enables downloading and usage of data from statistical agencies without using a web browser direct over HTTP/HTTPS. +PXWEB is an API structure developed by Statistics Sweden and other national statistical institutions (NSI) to disseminate public statistics in a structured way. This API enables downloading and using data from statistical agencies without using a web browser direct over HTTP/HTTPS. -The `pxweb` R package connects any PXWEB API to R and hence facilitate the access, use and referencing of data from PXWEB APIs. +The `pxweb` R package connects any PXWEB API to R and facilitates the access, use and referencing of data from PXWEB APIs. ### Available data sources and tools @@ -36,12 +36,9 @@ The `pxweb` R package connects any PXWEB API to R and hence facilitate the acces ### About PXWEB APIs -The data in PXWEB APIs consists of a metadata part and a data -part. Metadata is structured in a hierarchical node tree, where each -node contains information about subnodes that are below it in the tree -or, if the nodes are at the bottom of the tree structure, the data -referenced by the node as well as what dimensions are available for -the data at that subnode. +The data in PXWEB APIs consists of metadata and data +parts. Metadata is structured in a hierarchical node tree, where each node contains information about subnodes. The leaf nodes have information on which the dimensions are available for +the data at that leaf node. ## Installation @@ -73,7 +70,7 @@ vignette(topic="pxweb") ### Installation issues -We also recommend setting the UTF-8 encoding since each individual API may have local specificl letters: +We also recommend setting the UTF-8 encoding since each API may have local specific letters: ```{r locale, eval=FALSE} Sys.setlocale(locale="UTF-8") @@ -82,11 +79,11 @@ Sys.setlocale(locale="UTF-8") ## Accessing PXWEB from R -There are two ways of using the `pxweb` R package to access data, either interactively of using the core functions. To access data, two parts are needed, an URL to the data table in the API and a query specifying what data is of interest. +There are two ways of using the `pxweb` R package to access data, either interactively or using the core functions. To access data, two parts are needed, an URL to the data table in the API and a query specifying what data is of interest. ## Interactive use -The simplest way of using `pxweb` is to use it interactively and navigate the API to the data of interest and then set up the data query of interest. +The simplest way of using `pxweb` is to use it interactively, navigate the API to the data of interest, and then set up the query of interest. ```{r standardquery, message=FALSE, eval=FALSE} # Navigate through all pxweb api:s in the R package API catalogue @@ -105,26 +102,26 @@ d <- pxweb_interactive("data.ssb.no") pxweb_apis <- pxweb_api_catalogue() ``` -In the example above we use the interactive functionality from the PXWEB API root, but we could use any path to the API. +In the example above, we use the interactive functionality from the PXWEB API root, but we could use any path to the API. ```{r inapi, message=FALSE, eval=FALSE} # Start with a specific path. d <- pxweb_interactive("http://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A") ``` -This also means that we can navigate any PXWEB API, irrespectively of if they are a part of the R package API catalog or not. Just supply an URL to somewhere in the API and then navigate the API from there. +This functionality also means that we can navigate any PXWEB API, irrespectively of if they are a part of the R package API catalogue or not. Just supply an URL to somewhere in the API and then navigate the API from there. -Due to new CRAN policies, it is not possible to use an R function to edit the api catalogue of the R package, but editing the can be done easily from R using `file.edit()`. +Due to new CRAN policies, it is not possible to use an R function to edit the API catalogue of the R package, but editing them can be done quickly from R using `file.edit()`. ```{r, message=FALSE, eval=FALSE} file.edit(pxweb_api_catalogue_path()) ``` -Although, if the `pxweb` is installed again, it will overwrite the old api catalogue. So the easiest way is to do add a PXWEB API to the global catalogue. To do this, just do a pull request at the pxweb GitHub page [here](https://github.com/rOpenGov/pxweb). +Although, if the `pxweb` is installed again, it will overwrite the old API catalogue. So the easiest way is to add a PXWEB API to the global catalogue. To do this, do a pull request at the pxweb GitHub page [here](https://github.com/rOpenGov/pxweb). ## Direct use -Under the hood, the pxweb package uses the `pxweb_get()` function to access data from the PXWEB API. It also keeps track of the time limits of the API and split up to big queries into optimal downloadable chunks. If we use `pxweb_get()` without a query, the function either returns a PXWEB LEVELS object or a PXWEB METADATA object, depending if the URL points to a table in the API or not. Here is an example of a PXWEB LEVELS object. +Under the hood, the pxweb package uses the `pxweb_get()` function to access data from the PXWEB API. It also keeps track of the API's time limits and splits big queries into optimal downloadable chunks. If we use `pxweb_get()` without a query, the function either returns a PXWEB LEVELS object or a PXWEB METADATA object. What is returned depends on if the URL points to a table in the API or not. Here is an example of a PXWEB LEVELS object. ```{r levels, message=FALSE, eval=TRUE} # Get PXWEB levels @@ -142,13 +139,13 @@ px_meta ### Creating data queries -To download data we need both the URL to the table and a query specifying what parts of the table are of interest. An URL to a table is an URL that will return a metadata object if not a query is supplied. Creating a query can be done in three main ways. The first and simplest approach is to use `pxweb_interactive()` to explore the table URL and create a query interactively. +To download data, we need both the URL to the table and a query specifying what parts of the table are of interest. An URL to a table is an URL that will return a metadata object if not a query is supplied. Creating a query can be done in three main ways. The first and most straightforward approach is to use `pxweb_interactive()` to explore the table URL and create a query interactively. ```{r, message=FALSE, eval=FALSE} d <- pxweb_interactive("http://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A/BefolkningNy") ``` -The interactive function will return the query and the url, even if the data is not downloaded. +The interactive function will return the query and the URL, even if the data is not downloaded. ```{r , message=FALSE, eval=TRUE, echo=FALSE} # save(d, file = "d_example.rda") @@ -160,14 +157,14 @@ d$url d$query ``` -We can also turn the query to a json query that can be used outside R. +We can also turn the query into a JSON query that we can use outside R. ```{r interactive_query, message=FALSE, eval=TRUE} pxweb_query_as_json(d$query, pretty = TRUE) ``` -The second approach is to specify the query either as an R list or a JSON object. Some Statistical Agencies, such as Statistics Sweden, supply queries directly as a JSON object on their web pages. These queries can be used directly. Below is another example of a JSON query for the table above. For details on how to set up a JSON query, see the PXWEB API documentation. +The second approach is to specify the query either as an R list or a JSON object. Some Statistical Agencies, such as Statistics Sweden, supply queries directly as a JSON object on their web pages. We can use these queries directly. Below is another example of a JSON query for the table above. For details on setting up a JSON query, see the PXWEB API documentation. ``` { @@ -207,7 +204,7 @@ The second approach is to specify the query either as an R list or a JSON object } ``` -To use this JSON query we just store the JSON query as a file and supply the path to the file to the ```pxweb_query()``` function. +To use this JSON query, we store the JSON query as a file and supply the path to the file to the "`pxweb_query()` "function. ```{r pxq, message=FALSE, eval=FALSE} pxq <- pxweb_query("path/to/the/json/query.json") @@ -225,7 +222,7 @@ pxq <- pxweb_query(pxweb_query_list) pxq ``` -The query can be validated against the metadata object to asses that the query can be used. This is done automatically when the data is fetched with ```pxweb_get()```, but can also be done manually. +We can validate the query against the metadata object to asses that we can use the query. This validation is done automatically when the data is fetched with `pxweb_get()` but can also be done manually. ```{r validate_query, message=FALSE, eval=TRUE} pxweb_validate_query_with_metadata(pxq, px_meta) @@ -233,7 +230,7 @@ pxweb_validate_query_with_metadata(pxq, px_meta) ### Downloading data -When we have the URL to a data table and a query we can simply download the data with ```pxweb_get()```. The function returns a `pxweb_data` object that contains the downloaded data. +When we have the URL to a data table and a query, we can download the data with "`pxweb_get()` ". The function returns a `pxweb_data` object that contains the downloaded data. ```{r, message=FALSE, eval=TRUE} pxd <- pxweb_get("http://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A/BefolkningNy", @@ -241,7 +238,7 @@ pxd <- pxweb_get("http://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A/Bef pxd ``` -If we instead want a JSON-stat object, we just change the response format to JSON-stat and we will get a JSON-stat object returned. Only JSON and JSON-stat formats are implemented in the PXWEB API. +If we instead want a JSON-stat object, we change the response format to JSON-stat, and we will get a JSON-stat object returned. ```{r, message=FALSE, eval=TRUE} pxq$response$format <- "json-stat" @@ -250,11 +247,20 @@ pxjstat <- pxweb_get("http://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A pxjstat ``` -If the queries are large (contain more values than the PXWEB API maximum allowed values), the query is chunked into optimal chunks and is then downloaded sequentially. PXWEB data objects are then combined to one large PXWEB data object, while JSON-stat objects are returned as a list of JSON-stat objects. +Some return formats return files. Then, these responses are stored in the R `tempdir()` folded, and the file paths are returned by `pxweb_get()`. Currently, `px` and `sdmx` formats can be downloaded as files, but file an issue if you need other response formats. -For more advanced connections to the API, the `pxweb_advanced_get()` gives the flexibility to access the underlying HTTP calls using `httr` as well as logging the HTTP calls for debugging. +```{r, message=FALSE, eval=TRUE} +pxq$response$format <- "px" +pxfp <- pxweb_get("http://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A/BefolkningNy", + pxq) +pxfp +``` -The downloaded PXWEB data objects can then be converted to either `data.frame`s or to a character matrix. The character matrix contains the "raw" data while the data.frame returns a data.frame for analysis in a tidy format. This means that missing values (such as ".." are converted to `NA`) in a data.frame. Using the arguments `variable.value.type` and `column.name.type` we can also choose if we want the code or the text column names and value types. +If the queries are large (contain more values than the PXWEB API maximum allowed values), the query is chunked into optimal chunks and is then downloaded sequentially. PXWEB data objects are then combined into one large PXWEB data object, while JSON-stat objects are returned as a list of JSON-stat objects, and other files are stored in `tempdir()` as separate files. + +For more advanced connections to the API, the `pxweb_advanced_get()` gives the flexibility to access the underlying HTTP calls using `httr` and log the HTTP calls for debugging. + +We can then convert the downloaded PXWEB data objects to a `data. frame` or to a character matrix. The character matrix contains the "raw" data while `data. frame` returns an R `data.frame` in a tidy format. This conversion means missing values (such as ".." are converted to `NA`) in a `data. frame`. Using the arguments `variable.value.type` and `column.name.type`, we can choose if we want the code or the text column names and value types. ```{r, message=FALSE, eval=TRUE} pxdf <- as.data.frame(pxd, column.name.type = "text", variable.value.type = "text") @@ -267,7 +273,7 @@ pxdf <- as.data.frame(pxd, column.name.type = "code", variable.value.type = "cod head(pxdf) ``` -In a similar way, we can access the raw data as a character matrix with `as.matrix`. +Similarly, we can access the raw data as a character matrix with `as.matrix`. ```{r, message=FALSE, eval=TRUE} pxmat <- as.matrix(pxd, column.name.type = "code", variable.value.type = "code") @@ -283,7 +289,7 @@ pxdc <- pxweb_data_comments(pxd) pxdc ``` -In this case, we did not have any comments. If we have comments we can turn the comments into a data.frame with one comment per row. +In this case, we did not have any comments. If we have comments, we can turn the comments into a `data. frame` with one comment per row. ```{r, message=FALSE, eval=FALSE} as.data.frame(pxdc) @@ -310,8 +316,9 @@ This work can be freely used, modified and distributed under the open license sp ## Session info -This vignette was created with +We created this vignette with ```{r sessioninfo, message=FALSE, warning=FALSE} sessionInfo() ``` + diff --git a/vignettes/pxweb.html b/vignettes/pxweb.html index 00cae767..852fd6c3 100644 --- a/vignettes/pxweb.html +++ b/vignettes/pxweb.html @@ -12,43 +12,57 @@ - + PX-WEB API Interface for R - + + + - +