Fix issue #251, partly address issue #250

DESCRIPTION

@@ -1,7 +1,7 @@
 Type: Package
 Package: eurostat
 Title: Tools for Eurostat Open Data
-Version: 3.7.12
+Version: 3.7.13
 Date: 2022-06-28
 Authors@R: c(
     person(given = "Leo", 
@@ -98,6 +98,6 @@ MailingList: rOpenGov <ropengov-forum@googlegroups.com>
 NeedsCompilation: no
 Repository: CRAN
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.2.0
+RoxygenNote: 7.2.3
 X-schema.org-isPartOf: http://ropengov.org/
 X-schema.org-keywords: ropengov

NAMESPACE

@@ -26,7 +26,6 @@ export(search_eurostat)
 export(set_eurostat_cache_dir)
 export(validate_geo_code)
 export(validate_nuts_regions)
-import(httr)
 importFrom(RefManageR,BibEntry)
 importFrom(RefManageR,toBiblatex)
 importFrom(broom,tidy)
@@ -38,8 +37,12 @@ importFrom(dplyr,coalesce)
 importFrom(dplyr,filter)
 importFrom(dplyr,left_join)
 importFrom(dplyr,mutate)
+importFrom(httr,RETRY)
+importFrom(httr,build_url)
 importFrom(httr,content)
 importFrom(httr,http_error)
+importFrom(httr,parse_url)
+importFrom(httr,status_code)
 importFrom(jsonlite,fromJSON)
 importFrom(lubridate,day)
 importFrom(lubridate,dmy)
@@ -61,4 +64,5 @@ importFrom(tidyr,gather)
 importFrom(tidyr,separate)
 importFrom(utils,data)
 importFrom(utils,download.file)
+importFrom(utils,hasName)
 importFrom(utils,toBibtex)

NEWS.md

@@ -1,3 +1,8 @@
+# eurostat 3.7.13 (2023-02-01)
+
+* Updated `get_eurostat_json` to migrate from JSON web service to API Statistics (addressed in issues #250, #251). Removed parameters *lang* and *type*; *lang* is now part of the **filters** parameter and . See Eurostat - Data browser online help website for more information: https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+migrating+from+JSON+web+service+to+API+Statistics
+* Minor fixes in `get_bibentry` and `get_eurostat_geospatial`
+
 # eurostat 3.7.12 (2022-06-28)
 
 * Updated included dataset `eurostat_geodata_60_2016` to fix the issue of old-style crs object (#237)

R/get_bibentry.R

@@ -33,7 +33,7 @@ get_bibentry <- function(code,
   if (!any(class(code) %in% c("character", "factor"))) {
     stop("The code(s) must be added as character vector")
   }
-  if (!is.null(keywords) & !class(keywords) == "list") {
+  if (!is.null(keywords) & !inherits(keywords, "list")) {
     stop("If keyword(s) are added, they must be added as a list.")
   }
 

R/get_eurostat_geospatial.R

@@ -184,7 +184,7 @@
 #'
 #' @importFrom utils data
 #' @importFrom broom tidy
-#' @importFrom httr http_error content
+#' @importFrom httr http_error content RETRY
 #' @importFrom methods as
 #' @importFrom dplyr left_join
 #'
@@ -337,7 +337,7 @@ get_eurostat_geospatial <- function(output_class = "sf",
 
         if (nuts_level %in% c("0", "all")) {
           url <- paste0(burl, resolution, "M_", year, "_", crs, "_LEVL_0.geojson")
-          resp <- RETRY("GET", url, terminate_on = c(404))
+          resp <- httr::RETRY("GET", url, terminate_on = c(404))
           if (httr::http_error(resp)) {
             stop(paste("The requested url cannot be found within
           the get_eurostat_geospatial function:", url))
@@ -349,7 +349,7 @@ get_eurostat_geospatial <- function(output_class = "sf",
         }
         if (nuts_level %in% c("1", "all")) {
           url <- paste0(burl, resolution, "M_", year, "_", crs, "_LEVL_1.geojson")
-          resp <- RETRY("GET", url, terminate_on = c(404))
+          resp <- httr::RETRY("GET", url, terminate_on = c(404))
           if (httr::http_error(resp)) {
             stop(paste("The requested url cannot be found within the get_eurostat_geospatial function:", url))
           } else {
@@ -359,7 +359,7 @@ get_eurostat_geospatial <- function(output_class = "sf",
           }
         }
         if (nuts_level %in% c("2", "all")) {
-          resp <- RETRY("GET", paste0(burl, resolution, "M_", year, "_", crs, "_LEVL_2.geojson"), terminate_on = c(404))
+          resp <- httr::RETRY("GET", paste0(burl, resolution, "M_", year, "_", crs, "_LEVL_2.geojson"), terminate_on = c(404))
           if (httr::http_error(resp)) {
             stop(paste("The requested url cannot be found within the get_eurostat_geospatial function:", url))
           } else {
@@ -369,7 +369,7 @@ get_eurostat_geospatial <- function(output_class = "sf",
           }
         }
         if (nuts_level %in% c("3", "all")) {
-          resp <- RETRY("GET", paste0(burl, resolution, "M_", year, "_", crs, "_LEVL_3.geojson"), terminate_on = c(404))
+          resp <- httr::RETRY("GET", paste0(burl, resolution, "M_", year, "_", crs, "_LEVL_3.geojson"), terminate_on = c(404))
           if (httr::http_error(resp)) {
             stop(paste("The requested url cannot be found within the get_eurostat_geospatial function:", url))
           } else {

R/get_eurostat_json.R

@@ -2,8 +2,8 @@
 #' @description Retrieve data from Eurostat API in JSON format.
 #' @details
 #'   Data to retrieve from
-#'   [The
-#'   Eurostat Web Services](https://ec.europa.eu/eurostat/web/json-and-unicode-web-services) can be specified with filters. Normally, it is
+#'   [The Eurostat Web Services](https://ec.europa.eu/eurostat/web/json-and-unicode-web-services) 
+#'   can be specified with filters. Normally, it is
 #'   better to use JSON query through [get_eurostat()], than to use
 #'   [get_eurostat_json()] directly.
 #'
@@ -23,7 +23,8 @@
 #' @param filters A named list of filters. Names of list objects are Eurostat
 #'   variable codes and values are vectors of observation codes. If `NULL`
 #'   (default) the whole dataset is returned. See details for more on filters
-#'   and limitations per query.
+#'   and limitations per query. **Please note that before eurostat 3.7.13 lang
+#'   was a separate parameter, but from 3.7.13 onwards it is one of the filters.
 #' @param lang A language used for metadata (en/fr/de).
 #' @param type A type of variables, "code" (default), "label" or "both". The
 #'   "both" will return a data_frame with named vectors, labels as values and
@@ -37,7 +38,7 @@
 #' @inheritDotParams httr::GET
 #' @return A dataset as a data_frame.
 #' @export
-#' @author Przemyslaw Biecek, Leo Lahti, Janne Huovari and Markus Kainu
+#' @author Przemyslaw Biecek, Leo Lahti, Janne Huovari Markus Kainu and Pyry Kantanen
 #' @references
 #' See `citation("eurostat")`:
 #'
@@ -55,14 +56,15 @@
 #'   indic_na = "B1GM"
 #' ))
 #' }
-#' @import httr
+#' @importFrom httr http_error status_code
 #' @importFrom jsonlite fromJSON
 #' @importFrom tibble as_tibble
 #' @seealso [httr::GET()]
 #' @keywords utilities database
-get_eurostat_json <- function(id, filters = NULL,
-                              type = c("code", "label", "both"),
-                              lang = c("en", "fr", "de"),
+get_eurostat_json <- function(id, 
+                              filters = NULL,
+                              type = "code",
+                              lang = "EN",
                               stringsAsFactors = FALSE,
                               ...) {
 
@@ -130,11 +132,11 @@ get_eurostat_json <- function(id, filters = NULL,
 
     dims_list <- lapply(dims[rev(ids)], function(x) {
       y <- x$category$label
-      if (type[1] == "label") {
+      if (type == "label") {
         y <- unlist(y, use.names = FALSE)
-      } else if (type[1] == "code") {
+      } else if (type == "code") {
         y <- names(unlist(y))
-      } else if (type[1] == "both") {
+      } else if (type == "both") {
         y <- unlist(y)
       } else {
         stop("Invalid type ", type)
@@ -164,37 +166,59 @@ get_eurostat_json <- function(id, filters = NULL,
 
 
 
-# Internal function to build json url
-eurostat_json_url <- function(id, filters, lang) {
+#' @title Internal function to build json url
+#' @description Internal function to build json url
+#' @importFrom utils hasName
+#' @importFrom httr parse_url build_url
+#' @noRd
+#' @keywords internal utilities
+eurostat_json_url <- function(id, filters = NULL, lang = NULL) {
 
   # prepare filters for query
   filters2 <- as.list(unlist(filters))
+  # Give items names without numbers
   names(filters2) <- rep(names(filters), lapply(filters, length))
+
+  # The “format” parameter’s only possible value is "JSON".
+  if (!hasName(filters2, "format")) {
+    filters2$format <- "JSON"
+  }
+
+  if (!is.null(lang)) {
+    # The Language parameter (“lang”) can have only three values: 
+    # "EN" (English), "FR" (French), and "DE" (German).
+    if (toupper(lang) %in% c("EN", "FR", "DE")) {
+      filters2$lang <- toupper(lang)
+    } else {
+      message("Unsupported language code used. Using the default language: \"EN\"")
+      filters2$lang <- "EN"
+    }
+  } else {
+    # In case the parameter isn’t specified, the default value "EN" is taken.
+    message("Using the default language: \"EN\"")
+    filters2$lang <- "EN"
+  }
 
-  # prepare url
-  url_list <- list(
-    scheme = "http",
-    hostname = "ec.europa.eu",
-    path = file.path(
-      "eurostat/wdds/rest/data/v2.1/json",
-      lang[1], id
-    ),
-    query = filters2
-  )
-
-  # Data pulled from API v1.1 has different structure than v.2.1
-  # names(jdat2)
-  # [1] "version"   "label"     "href"      "source"    "updated"   "status"
-  # [7] "extension" "class"     "value"     "dimension" "id"        "size"
-  # names(jdat1[[1]])
-  # [1] "wsVersion"      "code"           "language"       "title"
-  # [5] "subTitle"       "description"    "lastUpdateDate" "status"
-  # [9] "value"          "dimension"
+  host_url <- "https://ec.europa.eu/eurostat/api/dissemination/"
+  service <- "statistics/"
+  version <- "1.0/"
+  response_type <- "data/"
+  datasetCode <- id
+
+  # Parse host_url and add relevant information in the standard list
+  url_list <- httr::parse_url(host_url)
+  # Paste "statistics/1.0/data/{id}" at the end of the fixed part of the URL
+  url_list$path <- paste0(url_list$path, 
+                          service, 
+                          version, 
+                          response_type, 
+                          datasetCode)
+  url_list$query <- filters2
 
-  class(url_list) <- "url"
   url <- httr::build_url(url_list)
   url
 }
+
 # Internal function to give warning if symmetric input-output tables need to download into strict matirx formats.
 special_id_values <- function(id) {
   siot_id_codes <- c(

tests/testthat/test_05_json.R

@@ -6,7 +6,7 @@ test_that("Get json data", {
     na_item = "B1GQ",
     unit = "CLV_I10"
   )),
-  c("geo", "unit", "na_item", "time", "values"),
+  c("geo", "unit", "na_item", "time", "values", "freq"),
   ignore.order = TRUE
   )
 })