rOpenGov · pitkant · Jan 18, 2024 · Jan 19, 2024 · Jan 19, 2024 · Jan 19, 2024
diff --git a/.github/workflows/check-full.yaml b/.github/workflows/check-full.yaml
@@ -6,6 +6,9 @@ on:
     - cron:  '0 0 * * 0'
 
 name: R-CMD-check (full)
+
+permissions: read-all
+
 jobs:
   R-CMD-check:
     runs-on: ${{ matrix.config.os }}
@@ -27,15 +30,14 @@ jobs:
       R_KEEP_PKG_SOURCE: yes
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - uses: r-lib/actions/setup-pandoc@v2
 
       - uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
           http-user-agent: ${{ matrix.config.http-user-agent }}
-          use-public-rspm: true
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
@@ -44,4 +46,5 @@ jobs:
 
       - uses: r-lib/actions/check-r-package@v2
         with:
-          upload-snapshots: true
+          upload-snapshots: true
+          build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")'
diff --git a/.github/workflows/check-standard.yaml b/.github/workflows/check-standard.yaml
@@ -8,22 +8,25 @@ on:
 
 name: R-CMD-check (standard)
 
+permissions: read-all
+
 jobs:
   R-CMD-check:
     runs-on: ubuntu-latest
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       R_KEEP_PKG_SOURCE: yes
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - uses: r-lib/actions/setup-r@v2
-        with:
-          use-public-rspm: true
 
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
           extra-packages: any::rcmdcheck
           needs: check
 
-      - uses: r-lib/actions/check-r-package@v2
+      - uses: r-lib/actions/check-r-package@v2
+        with:
+          upload-snapshots: true
+          build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")'
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,15 +1,17 @@
 Type: Package
 Package: eurostat
 Title: Tools for Eurostat Open Data
-Version: 4.0.0
-Date: 2023-12-19
+Version: 4.1.0.9007
+Date: 2026-04-29
 Authors@R: c(
     person("Leo", "Lahti", , "leo.lahti@iki.fi", role = c("aut", "cre"),
            comment = c(ORCID = "0000-0001-5537-637X")),
     person("Janne", "Huovari", role = "aut"),
     person("Markus", "Kainu", role = "aut"),
     person("Przemyslaw", "Biecek", role = "aut"),
     person("Daniel", "Antal", role = "ctb"),
+    person("Daniel", "Loos", role = "ctb",
+           comment = c(ORCID = "0000-0002-4024-4443")),
     person("Diego", "Hernangomez", role = "ctb",
            comment = c(ORCID = "0000-0001-8457-4658")),
     person("Joona", "Lehtomaki", role = "ctb"),
@@ -18,8 +20,9 @@ Authors@R: c(
     person("Paul", "Rougieux", role = "ctb"),
     person("Anna", "Vasylytsya", role = "ctb"),
     person("Oliver", "Reiter", role = "ctb"),
-    person("Pyry", "Kantanen", role = "ctb",
+    person("Pyry", "Kantanen", role = "aut",
            comment = c(ORCID = "0000-0003-2853-2765")),
+    person("Akewak", "Jeba", role = "ctb"),
     person("Enrico", "Spinielli", , "enrico.spinielli@gmail.com", role = "ctb",
            comment = c(ORCID = "0000-0001-8584-9131"))
   )
@@ -38,27 +41,29 @@ Imports:
     curl,
     digest,
     dplyr,
-    httr2 (>= 0.2.3),
+    httr2 (>= 1.1.2),
+    purrr,
     ISOweek,
     jsonlite,
     lubridate,
+    R.utils,
     rappdirs,
     readr,
-    RefManageR,
     regions,
     rlang,
     stringi,
     stringr,
     tibble,
     tidyr (>= 1.0.0),
     xml2,
-    data.table (>= 1.14.8)
+    data.table (>= 1.18.2.1)
 Suggests:
-    giscoR,
+    giscoR (>= 1.1.0),
     knitr,
     rmarkdown,
     sf,
-    testthat (>= 3.0.0)
+    testthat (>= 3.0.0),
+    lifecycle
 VignetteBuilder: 
     knitr
 Config/Needs/website: ggplot2, tmap, styler, sessioninfo,
@@ -71,6 +76,6 @@ MailingList: rOpenGov <ropengov-forum@googlegroups.com>
 NeedsCompilation: no
 Repository: CRAN
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.3
 X-schema.org-isPartOf: http://ropengov.org/
 X-schema.org-keywords: ropengov
diff --git a/LICENSE b/LICENSE
@@ -1,2 +1,2 @@
-YEAR: 2014-2022
+YEAR: 2014-2024
 COPYRIGHT HOLDER: Leo Lahti, Janne Huovari, Markus Kainu, Przemyslaw Biecek
diff --git a/NAMESPACE b/NAMESPACE
@@ -14,6 +14,9 @@ export(get_eurostat_folder)
 export(get_eurostat_geospatial)
 export(get_eurostat_interactive)
 export(get_eurostat_json)
+export(get_eurostat_link)
+export(get_eurostat_local)
+export(get_eurostat_sdmx)
 export(get_eurostat_toc)
 export(grepEurostatTOC)
 export(harmonize_country_code)
@@ -30,8 +33,7 @@ export(set_eurostat_cache_dir)
 export(validate_geo_code)
 export(validate_nuts_regions)
 importFrom(ISOweek,ISOweek2date)
-importFrom(RefManageR,BibEntry)
-importFrom(RefManageR,toBiblatex)
+importFrom(R.utils,isUrl)
 importFrom(classInt,classIntervals)
 importFrom(countrycode,countrycode)
 importFrom(curl,curl_download)
@@ -40,6 +42,7 @@ importFrom(data.table,.SD)
 importFrom(data.table,fread)
 importFrom(data.table,melt)
 importFrom(data.table,setDT)
+importFrom(data.table,setcolorder)
 importFrom(digest,digest)
 importFrom(dplyr,"%>%")
 importFrom(dplyr,case_when)
@@ -52,9 +55,13 @@ importFrom(httr2,req_error)
 importFrom(httr2,req_perform)
 importFrom(httr2,req_proxy)
 importFrom(httr2,req_retry)
+importFrom(httr2,req_url_path_append)
+importFrom(httr2,req_url_query)
 importFrom(httr2,req_user_agent)
 importFrom(httr2,request)
 importFrom(httr2,resp_body_json)
+importFrom(httr2,resp_body_raw)
+importFrom(httr2,resp_check_status)
 importFrom(httr2,resp_content_type)
 importFrom(httr2,resp_is_error)
 importFrom(httr2,url_build)
@@ -66,15 +73,19 @@ importFrom(lubridate,dmy)
 importFrom(lubridate,month)
 importFrom(lubridate,year)
 importFrom(lubridate,ymd)
+importFrom(purrr,map_dfr)
 importFrom(readr,col_character)
 importFrom(readr,cols)
+importFrom(readr,read_csv)
 importFrom(readr,read_tsv)
 importFrom(regions,recode_nuts)
 importFrom(regions,validate_geo_code)
 importFrom(regions,validate_nuts_regions)
 importFrom(rlang,"!!")
+importFrom(rlang,.data)
 importFrom(rlang,sym)
 importFrom(stats,na.omit)
+importFrom(stats,setNames)
 importFrom(stringi,stri_extract_first_regex)
 importFrom(stringi,stri_replace_all_fixed)
 importFrom(stringi,stri_replace_all_regex)
@@ -89,8 +100,14 @@ importFrom(utils,capture.output)
 importFrom(utils,download.file)
 importFrom(utils,hasName)
 importFrom(utils,menu)
+importFrom(utils,modifyList)
 importFrom(utils,person)
+importFrom(utils,read.csv)
+importFrom(utils,read.delim)
 importFrom(utils,toBibtex)
 importFrom(xml2,read_xml)
+importFrom(xml2,xml_attr)
 importFrom(xml2,xml_find_all)
+importFrom(xml2,xml_find_first)
+importFrom(xml2,xml_ns)
 importFrom(xml2,xml_text)
diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,33 @@
+# eurostat 4.1.0.9006
+
+## Minor updates
+
+* Removed experimental functions from the codebase, fixed tests that were pointing at datasets renamed by Eurostat
+
+# eurostat 4.1.0.9005
+
+## Deprecated and defunct
+
+* Removed RefManageR dependency from the package due to it being unmaintained
+
+# eurostat 4.1.0.9004
+
+## Major updates
+
+* Add new function `get_eurostat_sdmx()` for implementing functionalities that are specifically targeted for the new SDMX API. It is accompanied by other helper functions that are related to fetching metadata from the SDMX API: `build_api_base_uri()`, `build_agencyID()`, `data_filtering_on_dimension()`, `get_codelist_id()` and `label_eurostat_sdmx()`
+* Add new function `get_eurostat_local()` for reading local SDMX-CSV (.csv) and compressed SDMX-CSV (.csv.gz) files downloaded manually from the Eurostat website. This function relies only on `data.table` functions as it is expected that locally stored files may be larger and require more efficient handling with regards to RAM and processing time constrains. `tidy_eurostat_sdmx()` and `parse_filename()` are helper functions related to this function.
+* New `legacy_data_format()` function and `legacy.data.output` attribute in `get_eurostat()` and `get_eurostat_local()` transforms modern API naming conventions into the format that was used in the old API and eurostat package versions 3.8.* and prior. It is a helper function to other functions.
+* Add internal functions for retrieving SDMX metadata
+
+## Minor updates
+
+* `get_eurostat_geospatial()` documentation mentions that data from 2024 is now available (fixes issue #327 by @yhsong1013)
+* Fix mentions to Eurostat base URLs in `check_access_to_data()`, `get_eurostat()`, `get_eurostat_json()`, `get_eurostat_sdmx()`, `label_eurostat()` and `search_eurostat()` (fixes issue #326 by @olivierchantrel)
+
+## Deprecated and defunct
+
+* Remove deprecated parameter `make_valid` from `get_eurostat_geospatial()`
+
 # eurostat 4.0.0
 
 ## Major updates
@@ -70,7 +100,7 @@ downloading geospatial data (PR #264, thanks to @dieghernan):
 
 ## Major updates
 
-* Updated `get_eurostat()` and its assorted functions to download data from the new dissemination API (related to issues #251, #243). See Eurostat web page Transition - from Eurostat Bulk Download to API for a list of differences between old and new data sources: https://wikis.ec.europa.eu/display/EUROSTATHELP/Transition+-+from+Eurostat+Bulk+Download+to+API
+* Updated `get_eurostat()` and its assorted functions to download data from the new dissemination API (related to issues #251, #243). See Eurostat web page Transition - from Eurostat Bulk Download to API for a list of differences between old and new data sources: https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/bulkdownload
 * Added new temporary functions for downloading and handling data from the new dissemination API: `get_eurostat_raw2`, `tidy_eurostat2`, `convert_time_col2`, `eurotime2date2`, `eurotime2num2` and `label_eurostat2`. When the old bulk download facilities are decommissioned, these functions will replace the old functions with old naming schemes (without the 2s at the end).
 * `tidy_eurostat2` function is now able to handle multiple time frequencies in one call: For example, you can download annual, quarterly, and monthly data simply by using a vector c("A", "Q", "M") in select_time instead of using these singular frequencies in separate calls. The function will also return multiple time series in one dataset if select_time is NULL (as it is by default). If the dataset contains multiple time series and these are explicitly downloaded / no select_time parameter is given, a message will be printed.
 * `eurotime2num` can now handle monthly and weekly data as well.
@@ -82,7 +112,7 @@ downloading geospatial data (PR #264, thanks to @dieghernan):
 
 # eurostat 3.7.13 (2023-02-01)
 
-* Updated `get_eurostat_json()` to migrate from JSON web service to API Statistics (addressed in issues #243, #251). Please note that the output from JSON API is now slightly different than before: the datasets now contain a freq column to indicate the frequency with which data has been collected, for example annually "A", monthly "M" or quarterly "Q". See Eurostat - Data browser online help website for more information: https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+migrating+from+JSON+web+service+to+API+Statistics
+* Updated `get_eurostat_json()` to migrate from JSON web service to API Statistics (addressed in issues #243, #251). Please note that the output from JSON API is now slightly different than before: the datasets now contain a freq column to indicate the frequency with which data has been collected, for example annually "A", monthly "M" or quarterly "Q". See Eurostat - Data browser online help website for more information: https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/api-migrating/json
 * Minor fixes in `get_bibentry()` and `get_eurostat_geospatial()`
 
 # eurostat 3.7.12 (2022-06-28)

diff --git a/R/check_access_to_data.R b/R/check_access_to_data.R
@@ -1,4 +1,4 @@
-#' @title Check access to ec.europe.eu
+#' @title Check access to ec.europa.eu
 #' @description Check if R has access to resources at http://ec.europa.eu
 #' @author Markus Kainu <markus.kainu@@kapsi.fi>
 #' @return a logical.
@@ -9,12 +9,14 @@
 #'
 #' @importFrom curl curl_download
 #' @importFrom utils download.file
+#' @importFrom R.utils isUrl
 #'
 #' @export
 
 check_access_to_data <- function() {
   temp <- tempfile()
   http_url <- "https://ec.europa.eu/eurostat/cache/GISCO/distribution/v2/nuts/geojson/NUTS_RG_60M_2006_4326_LEVL_0.geojson"
+  stopifnot(R.utils::isUrl(http_url))
   # If unix use curl::curl_download to test connection
   # If windows use download.file with default method 'wininet'
   if (.Platform$OS.type == "unix") {

diff --git a/R/data_countries.R b/R/data_countries.R
@@ -2,9 +2,16 @@
 #' @description Countries and country codes in EU,
 #' Euro area, EFTA and EU candidate countries.
 #' @format A data_frame:
-#'   * **code**: Country code in the Eurostat database.
+#'   * **code**: Country code in the Eurostat database (two-letter ISO code (ISO 3166 alpha-2) except in the case of Greece where EL is used).
 #'   * **name**: Country name in English.
-#'   * **label**: Country name in the Eurostat database.
+#'   * **label**: Country name in the Eurostat database
+#'   * **name_fr**: Country name in French
+#'   * **name_de**: Country name in German
+#'   * **country_language**: Country name in national language(s).
+#'
+#' @details
+#' Country codes are two-letter ISO codes (ISO 3166 alpha-2) except in the case
+#' of Greece where EL is used instead of the standard ISO code.
 #'
 #' @family datasets
 #' @source <https://ec.europa.eu/eurostat/statistics-explained/index.php/Tutorial:Country_codes_and_protocol_order>,