diff --git a/.buildlibrary b/.buildlibrary index 2272c7f5..e02b1e87 100644 --- a/.buildlibrary +++ b/.buildlibrary @@ -1,4 +1,4 @@ -ValidationKey: '54982535' +ValidationKey: '55437291' AutocreateReadme: yes AcceptedWarnings: - 'Warning: package .* was built under R version' diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 28fd3939..a685741f 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -23,7 +23,7 @@ jobs: options(repos = c(pikpiam = 'https://pik-piam.r-universe.dev', CRAN = Sys.getenv('RSPM'))) pak::pak() - + - name: Run pre-commit checks shell: bash run: | diff --git a/CITATION.cff b/CITATION.cff index 6a8c97ea..9135f1f3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: If you use this software, please cite it using the metadata from this file. type: software title: 'mrvalidation: madrat data preparation for validation purposes' -version: 2.67.1 -date-released: '2026-05-12' +version: 2.69.1 +date-released: '2026-05-28' abstract: Package contains routines to prepare data for validation exercises. authors: - family-names: Bodirsky diff --git a/DESCRIPTION b/DESCRIPTION index 0c81f925..a2596314 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Type: Package Package: mrvalidation Title: madrat data preparation for validation purposes -Version: 2.67.1 -Date: 2026-05-12 +Version: 2.69.1 +Date: 2026-05-28 Authors@R: c( person("Benjamin Leon", "Bodirsky", , "bodirsky@pik-potsdam.de", role = c("aut", "cre")), person("Stephen", "Wirth", role = "aut"), @@ -53,6 +53,7 @@ Imports: reshape2, rlang, stringr, + terra, tidyr, utils, withr, diff --git a/NAMESPACE b/NAMESPACE index ab93e8f8..b17ed5e1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -104,6 +104,10 @@ importFrom(rlang,.data) importFrom(stringr,str_extract) importFrom(stringr,str_match) importFrom(stringr,str_split) +importFrom(terra,aggregate) +importFrom(terra,extract) +importFrom(terra,project) +importFrom(terra,rast) importFrom(tidyr,drop_na) importFrom(tidyr,expand_grid) importFrom(tidyr,nest) diff --git a/R/calcValidBII.R b/R/calcValidBII.R index 0709dc30..2dca935b 100644 --- a/R/calcValidBII.R +++ b/R/calcValidBII.R @@ -1,28 +1,84 @@ #' @title calcValidBII -#' @description validation for the Biodiversity Intactness Index (BII) -#' @author Michael Crawford +#' @description Validation for the Biodiversity Intactness Index (BII) from one +#' NHM source, selected via \code{datasource}: +#' * "Phillips" -- Phillips et al. 2021 (BII-BTE, DOI 10.5519/he1eqmg1), +#' country-level 1970-2014. +#' * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 (DOI 10.5519/k33reyb6), +#' cellular 5-yearly 2000-2020, aggregated to ISO here. #' -#' @return Country-level MAgPIE object +#' De Palma et al. 2024 differs from Phillips et al. 2021 in three steps: +#' stricter primary-vegetation baseline (minimally/lightly-used primary only, +#' vs Phillips's all-primary including intensely-used sites), logit (not log) +#' transform, and balanced Bray-Curtis (abundance-weighted) compositional +#' similarity in place of Phillips's asymmetric Jaccard (species-identity only). +#' De Palma's absolute BII is markedly lower, with a non-uniform gap that +#' depends on land-use intensity, so the two series are NOT splice-compatible +#' and are kept as separate model labels. +#' +#' MAgPIE's own BII output (modules/44_biodiversity, coefficients from +#' Leclère et al. 2020 "Bending the Curve" via PREDICTS) is on the Phillips +#' scale, so it is close to Phillips et al. 2021 (~0.03 above) but sits markedly +#' above De Palma et al. 2024. For absolute-level validation use Phillips; for +#' harmonized (change-from-baseline) plots either series works, but mixing +#' MAgPIE projections with De Palma historical on a non-harmonized axis is +#' misleading. Two things drive the MAgPIE-vs-De Palma gap: the PREDICTS +#' coefficient version (Phillips vs De Palma), and land-use allocation -- MAgPIE +#' land use is LUH3-initialised with FAO/FRA forest correction +#' (mrlandcore::calcForestArea), vs the LUH2 underlying De Palma's v2.1.1. A 2010 +#' decomposition shows the coefficient change dominates the gap (~0.17 of the +#' ~0.20 total globally, and larger than the land-use term in every H12 region); +#' the land-use/forest difference is real but secondary (~0.03 globally). +#' @author Michael Crawford, Florian Humpenoeder +#' @seealso \code{\link{readBII}}, \code{\link{downloadBII}} +#' +#' @param datasource "Phillips" (default) or "DePalma". +#' +#' @return Country-level MAgPIE object for the selected source under variable +#' "Biodiversity|BII". #' #' @examples #' #' \dontrun{ -#' calcOutput("ValidBII") +#' calcOutput("ValidBII", datasource = "Phillips") +#' calcOutput("ValidBII", datasource = "DePalma") #' } #' +#' @importFrom madrat toolAggregate toolCountryFill toolNAreplace -calcValidBII <- function() { +calcValidBII <- function(datasource = "Phillips") { - bii <- readSource("BII", subtype = "historical", subset = "bii") - bii <- add_dimension(bii, dim = 3.2, add = "model", nm = "Phillips et al") - getNames(bii, dim = "variable") <- "Biodiversity|BII" + if (identical(datasource, "Phillips")) { + # Phillips et al. 2021: already country-level. + bii <- readSource("BII", subtype = "Phillips", subset = "bii") + bii <- add_dimension(bii, dim = 3.2, add = "model", nm = "Phillips et al") + descr <- "Historical BII, Phillips et al. 2021 (DOI 10.5519/he1eqmg1, 1970-2014)." - land <- calcOutput("FAOLand", aggregate = FALSE) - landArea <- land[, "y2015", "6601|Land area"] # total land is stable through time, so I arbitrarily use y2015 + } else if (identical(datasource, "DePalma")) { + # De Palma et al. 2024 NHM v2.1.1: cellular -> ISO. toolNAreplace masks both + # value and weight to 0 where either is NA, so toolAggregate's weighted mean + # ignores water/missing cells instead of zeroing whole countries. + biiCell <- readSource("BII", subtype = "DePalma") + cellArea <- calcOutput("LandArea", cells = "lpjcell", aggregate = FALSE) + naFix <- toolNAreplace(x = biiCell, weight = cellArea) + bii <- toolAggregate(naFix$x, weight = naFix$weight, to = "iso", dim = 1, + zeroWeight = "setNA") + bii <- toolCountryFill(bii, fill = NA, verbosity = 2) + bii <- add_dimension(bii, dim = 3.1, add = "scenario", nm = "historical") + bii <- add_dimension(bii, dim = 3.2, add = "model", nm = "De Palma et al 2024") + descr <- paste("Historical BII, De Palma et al. 2024 v2.1.1", + "(DOI 10.5519/k33reyb6, 2000-2020), cellular aggregated to ISO.") + + } else { + stop("Unknown datasource '", datasource, "'. Use \"Phillips\" or \"DePalma\".") + } + + getNames(bii, dim = "variable") <- "Biodiversity|BII" - return(list(x = bii, - weight = landArea, - unit = "unitless", - description = "Historical BII for 1970-2014")) + land <- calcOutput("FAOLand", aggregate = FALSE) + landArea <- land[, "y2015", "6601|Land area"] # land area stable across time + return(list(x = bii, + weight = landArea, + unit = "unitless", + description = descr)) } diff --git a/R/downloadBII.R b/R/downloadBII.R index da8d975c..1e9ed393 100644 --- a/R/downloadBII.R +++ b/R/downloadBII.R @@ -1,56 +1,129 @@ #' @title downloadBII -#' @description Download historical and projected BII dataset from Phillips et al. 2021 -#' (https://doi.org/10.5519/he1eqmg1). -#' @seealso https://data.nhm.ac.uk/dataset/bii-bte -#' @author Michael Crawford +#' @description Download a Biodiversity Intactness Index dataset from the +#' Natural History Museum. Two subtypes: +#' * "Phillips" (default) -- Phillips et al. 2021 BII-BTE +#' (\doi{10.5519/he1eqmg1}); country/region tables, 1970-2050 incl. SSPs. +#' * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 (\doi{10.5519/k33reyb6}); +#' global 5 arc-minute GeoTIFFs for 2000, 2005, 2010, 2015, 2020. +#' @param subtype "Phillips" (default) or "DePalma". +#' @seealso https://data.nhm.ac.uk/dataset/bii-bte, \code{\link{readBII}} +#' @author Michael Crawford, Florian Humpenoeder #' -#' @return Metadata on downloaded BII data +#' @return Metadata on the downloaded BII data. +#' +#' @details +#' Both NHM resource URLs are behind Cloudflare and return HTTP 403 to +#' non-browser clients, so both downloads use the portal's content-addressable +#' \code{/downloads/direct/.zip} endpoint, which bypasses the challenge. +#' Phillips uses the CSV resource (the .rds resource has no working direct URL); +#' De Palma uses the v2.1.1 raster ZIP. If a URL becomes unreachable, download +#' via a browser and place the file in this source folder. #' #' @examples #' #' \dontrun{ #' downloadSource("BII") +#' downloadSource("BII", subtype = "DePalma") #' } #' -#' @importFrom utils download.file person +#' @importFrom utils download.file person unzip -downloadBII <- function() { +downloadBII <- function(subtype = "Phillips") { - # nolint start + # nolint start - rdsURL <- "https://data.nhm.ac.uk/dataset/07f701d4-9a26-403c-a9d6-248c895fa412/resource/94be0af6-ec90-4b83-8f02-64a4983e1ca1/download/long_data.rds" - download.file(rdsURL, destfile = "BII_historical_and_SSPs.rds") + if (identical(subtype, "DePalma")) { + url <- "https://data.nhm.ac.uk/downloads/direct/8161226fda7a36ca0b94f2becee9afe1b318d0a8.zip" + zipFile <- "bii-v2-1-1-nhm-data-portal.zip" + if (!file.exists(zipFile)) { + tryCatch( + utils::download.file(url, destfile = zipFile, mode = "wb"), + error = function(e) { + stop("Automated NHM download failed (Cloudflare likely). Manually download from ", + "https://data.nhm.ac.uk/dataset/bii-developed-by-nhm-v2-1-1-limited-release ", + "and place '", zipFile, "' in this source folder. Original error: ", conditionMessage(e)) + } + ) + } + # Cloudflare challenge stub is ~6 KB; real ZIP is ~45 MB. + if (file.size(zipFile) < 1e7) { + stop("'", zipFile, "' is too small (", file.size(zipFile), + " bytes) -- likely a Cloudflare challenge page. Manually download via browser.") + } + # Outer ZIP holds an inner ZIP (resource-ID-named) + manifest; inner ZIP has the 5 GeoTIFFs. + utils::unzip(zipFile, exdir = ".") + innerZip <- "c4c281c4-befa-4e1b-a162-ba2f25e5ae82.zip" + if (!file.exists(innerZip)) { + stop("Inner archive '", innerZip, "' missing -- NHM ZIP layout may have changed.") + } + utils::unzip(innerZip, exdir = ".") + return(list(url = "https://data.nhm.ac.uk/dataset/bii-developed-by-nhm-v2-1-1-limited-release", + doi = "https://doi.org/10.5519/k33reyb6", + title = "The Biodiversity Intactness Index developed by The Natural History Museum, London, v2.1.1 (Open Access, Limited Release)", + unit = "Percentage (0-100)", + author = list(person("Adriana", "De Palma"), person("Sara", "Contu"), + person("Gareth E", "Thomas"), person("Connor", "Duffin"), + person("Sabine", "Nix"), person("Andy", "Purvis")), + version = "v2.1.1", + release_date = "2024-10-09", + description = "Global gridded BII at 5 arc-minute resolution for 2000, 2005, 2010, 2015, 2020. Values 0-100 (100 = fully intact).", + license = "CC-BY-NC-SA 4.0", + reference = "Adriana De Palma; Sara Contu; Gareth E Thomas; Connor Duffin; Sabine Nix; Andy Purvis (2024). The Biodiversity Intactness Index developed by The Natural History Museum, London, v2.1.1 (Open Access, Limited Release) [Data set]. Natural History Museum. https://doi.org/10.5519/k33reyb6") + ) + } - return(list(url = "https://data.nhm.ac.uk/dataset/bii-bte", - doi = "https://doi.org/10.5519/he1eqmg1", - title = "The Biodiversity Intactness Index - country, region and global-level summaries for the year - 1970 to 2050 under various scenarios", - unit = "Unitless", - author = list(person("Helen", "Phillips"), - person("Adriana", "De Palma"), - person("Ricardo E", "Gonzalez"), - person("Sara", "Contu"), - person("Samantha L L", "Hill"), - person("Andres", "Baselga"), - person("Luca", "Borger"), - person("Andy", "Purvis")), - version = NULL, - release_date = "October 4, 2021", - description = "Using the PREDICTS database of local biodiversity measures at thousands of sites around the world, - we statistically modelled how total abundance of organisms and compositional similarity responded - to land use and related pressures. We combined these models with spatio-temporal projections of - explanatory variables (at 0.25 degrees spatial resolution) from the year 1970 to 2050 under five - Shared Socioeconomic Pathways (SSPs) to project the Biodiversity Intactness Index (BII). Mean BII - (weighted by cell area) was calculated at the country, subregion, interregion and global level. We - used cross-validation (leaving one biome out in turn) to produce decadal upper and lower uncertainty - margins for 1970-2050. These summary data were uploaded to the Natural History Museum's Biodiversity - Trends Explorer on 2021-10-27. We have also provided mean values of some of the pressures, as changes - in these contribute to changes in BII.", - license = "Creative Commons Non-Commercial", - reference = "Helen Phillips; Adriana De Palma; Ricardo E Gonzalez; Sara Contu et al. (2021). The Biodiversity - Intactness Index - country, region and global-level summaries for the year 1970 to 2050 under - various scenarios [Data set]. Natural History Museum. ") + # Phillips et al. 2021 (default). CKAN resource URL is Cloudflare-blocked; + # use the /downloads/direct/.zip endpoint (CSV resource -- the .rds has + # no working direct URL). The ZIP holds resource.csv + manifest.json. + url <- "https://data.nhm.ac.uk/downloads/direct/bcaa1a8726032605ed11b1db0cb503d6e00ad584.zip" + zipFile <- "bii-bte-nhm-data-portal.zip" + if (!file.exists(zipFile)) { + tryCatch( + utils::download.file(url, destfile = zipFile, mode = "wb"), + error = function(e) { + stop("Automated NHM download failed (Cloudflare likely). Manually download from ", + "https://data.nhm.ac.uk/dataset/bii-bte and place '", zipFile, + "' in this source folder. Original error: ", conditionMessage(e)) + } ) + } + if (file.size(zipFile) < 1e6) { + stop("'", zipFile, "' is too small (", file.size(zipFile), + " bytes) -- likely a Cloudflare challenge page. Manually download via browser.") + } + utils::unzip(zipFile, exdir = ".") + if (file.exists("resource.csv")) file.rename("resource.csv", "BII_historical_and_SSPs.csv") + + return(list(url = "https://data.nhm.ac.uk/dataset/bii-bte", + doi = "https://doi.org/10.5519/he1eqmg1", + title = "The Biodiversity Intactness Index - country, region and global-level summaries for the year + 1970 to 2050 under various scenarios", + unit = "Unitless", + author = list(person("Helen", "Phillips"), + person("Adriana", "De Palma"), + person("Ricardo E", "Gonzalez"), + person("Sara", "Contu"), + person("Samantha L L", "Hill"), + person("Andres", "Baselga"), + person("Luca", "Borger"), + person("Andy", "Purvis")), + version = NULL, + release_date = "October 4, 2021", + description = "Using the PREDICTS database of local biodiversity measures at thousands of sites around the world, + we statistically modelled how total abundance of organisms and compositional similarity responded + to land use and related pressures. We combined these models with spatio-temporal projections of + explanatory variables (at 0.25 degrees spatial resolution) from the year 1970 to 2050 under five + Shared Socioeconomic Pathways (SSPs) to project the Biodiversity Intactness Index (BII). Mean BII + (weighted by cell area) was calculated at the country, subregion, interregion and global level. We + used cross-validation (leaving one biome out in turn) to produce decadal upper and lower uncertainty + margins for 1970-2050. These summary data were uploaded to the Natural History Museum's Biodiversity + Trends Explorer on 2021-10-27. We have also provided mean values of some of the pressures, as changes + in these contribute to changes in BII.", + license = "Creative Commons Non-Commercial", + reference = "Helen Phillips; Adriana De Palma; Ricardo E Gonzalez; Sara Contu et al. (2021). The Biodiversity + Intactness Index - country, region and global-level summaries for the year 1970 to 2050 under + various scenarios [Data set]. Natural History Museum. ") + ) - # nolint end + # nolint end } diff --git a/R/fullVALIDATION.R b/R/fullVALIDATION.R index 94db65ba..709df1fc 100644 --- a/R/fullVALIDATION.R +++ b/R/fullVALIDATION.R @@ -389,7 +389,9 @@ fullVALIDATION <- function(rev = 0.1, aggregate = "region+global") { file = valfile, append = TRUE, try = TRUE) # Diversity indices - calcOutput(type = "ValidBII", aggregate = aggregate, + calcOutput(type = "ValidBII", datasource = "Phillips", aggregate = aggregate, + file = valfile, append = TRUE, try = TRUE) + calcOutput(type = "ValidBII", datasource = "DePalma", aggregate = aggregate, file = valfile, append = TRUE, try = TRUE) calcOutput(type = "ValidCropareaDiversity", index = "shannon", aggregate = aggregate, file = valfile, append = TRUE, try = TRUE) diff --git a/R/readBII.R b/R/readBII.R index f0adc27e..367de4f6 100644 --- a/R/readBII.R +++ b/R/readBII.R @@ -1,53 +1,103 @@ #' @title readBII -#' @description Read and select scenarios and variables from the "The Biodiversity Intactness Index - -#' country, region and global-level summaries for the year 1970 to 2050 under various scenarios" -#' dataset. -#' Default behavior is to return BII for the historical period. -#' @seealso https://data.nhm.ac.uk/dataset/bii-bte -#' @author Michael Crawford +#' @description Read the Biodiversity Intactness Index from the Natural History +#' Museum. Two dataset subtypes: +#' * "Phillips" (default) -- Phillips et al. 2021 BII-BTE, historical scenario, +#' country-level, returned as ISO x year x scenario.variable. +#' * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 -- cellular x.y.iso on the +#' 67420-cell lpjcell grid, fraction (0-1). Country aggregation is left to +#' the consumer (calcValidBII). +#' @seealso https://data.nhm.ac.uk/dataset/bii-bte, +#' https://data.nhm.ac.uk/dataset/bii-developed-by-nhm-v2-1-1-limited-release, +#' \code{\link{downloadBII}}, \code{\link{calcValidBII}} +#' @author Michael Crawford, Florian Humpenoeder #' -#' @param subtype scenario selection, with potential values: "historical", "ssp1rcp2p6image", -#' "ssp2rcp4p5messageglobiom", "ssp3rcp7p0aim", "ssp4rcp6p0gcam", "ssp5rcp8p5remindmagpie". -#' Lists are accepted. -#' @param subset variable selection, with potential values: "bii", "crops", "highintensityag", -#' "hpd", "pastureland", "qualitynatural", "urbanextent". Lists are accepted. +#' @param subtype Dataset: "Phillips" (default, Phillips et al. 2021 historical +#' BII) or "DePalma" (De Palma et al. 2024 v2.1.1 rasters). +#' @param subset Phillips variable selection ("bii", "crops", "highintensityag", +#' "hpd", "pastureland", "qualitynatural", "urbanextent"; lists accepted). +#' Ignored for subtype = "DePalma". #' -#' @return magclass object containing the desired subtypes and subsets +#' @return magclass object. Phillips: ISO x year x scenario.variable. DePalma: +#' x.y.iso x year x "bii" (fraction 0-1). +#' +#' @details +#' The De Palma rasters are 5 arc-minute WGS84 GeoTIFFs. terra::aggregate +#' (fact = 6, fun = "mean") brings them to 30 arc-minute (unweighted mean is +#' fine within a 6x6 block); terra::project pins to the canonical 0.5-deg grid; +#' terra::extract samples the 67420 lpjcell coordinates. Source is percent +#' (0-100), divided by 100 for unit-parity with Phillips. The data sub-dim is +#' named "variable" so calcValidBII can use getNames(x, dim = "variable") <- ... +#' on both subtypes. #' #' @examples #' #' \dontrun{ -#' readSource("BII") +#' readSource("BII") # Phillips historical +#' readSource("BII", subtype = "DePalma") # De Palma 2024 v2.1.1 #' } #' -#' @importFrom dplyr %>% mutate select +#' @importFrom dplyr filter mutate select #' @importFrom tidyr drop_na #' @importFrom rlang .data #' @importFrom stringr str_match -#' @importFrom magclass as.magpie +#' @importFrom magclass as.magpie mbind getCells getNames getSets getYears +#' @importFrom terra rast aggregate project extract +#' @importFrom mstools toolGetMappingCoord2Country +#' @importFrom utils read.csv -readBII <- function(subtype = "historical", subset = "bii") { +readBII <- function(subtype = "Phillips", subset = "bii") { - bii <- readRDS("BII_historical_and_SSPs.rds") + if (identical(subtype, "DePalma")) { + years <- c(2000, 2005, 2010, 2015, 2020) + files <- setNames(sprintf("bii-%d_v2-1-1.tif", years), as.character(years)) + missingFiles <- files[!file.exists(files)] + if (length(missingFiles)) { + stop("Missing BII v2.1.1 GeoTIFF(s): ", paste(missingFiles, collapse = ", "), + ". Run downloadSource(\"BII\", subtype = \"DePalma\") first.") + } + map <- toolGetMappingCoord2Country(pretty = TRUE) + resolution <- terra::rast(resolution = 0.5) + .read <- function(year, file) { + r <- terra::rast(file) + r <- terra::aggregate(r, fact = 6, fun = "mean", na.rm = TRUE) + r <- terra::project(r, resolution) + vals <- terra::extract(r, map[c("lon", "lat")])[, 2] # col 1 = ID + m <- as.magpie(vals, spatial = 1) + getYears(m) <- year + getNames(m) <- "bii" + m + } + out <- do.call(mbind, Map(.read, years, files)) + getCells(out) <- paste(map$coords, map$iso, sep = ".") + getSets(out) <- c("x.y.iso", "t", "variable") + return(out / 100) # percent -> fraction (parity with Phillips) + } - bii <- bii %>% - filter(.data$scenario %in% subtype & .data$variable %in% subset) + # Phillips et al. 2021 (default): historical scenario only, subset = variable. + # downloadBII ships the CSV resource; older source folders have the .rds. + bii <- if (file.exists("BII_historical_and_SSPs.csv")) { + utils::read.csv("BII_historical_and_SSPs.csv") + } else { + readRDS("BII_historical_and_SSPs.rds") + } - bii <- bii %>% - mutate(ISO = str_match(string = .data$area_code, pattern = "[A-Z]{3}")) %>% # identify the country aggregation - drop_na("ISO") %>% # remove all non-country aggregations (e.g. global) - select("ISO", "scenario", "variable", "year", "value") + bii <- bii |> + filter(.data$scenario == "historical" & .data$variable %in% subset) - biiMag <- as.magpie(bii) + bii <- bii |> + mutate(ISO = str_match(string = .data$area_code, pattern = "[A-Z]{3}")) |> # identify the country aggregation + drop_na("ISO") |> # remove all non-country aggregations (e.g. global) + select("ISO", "scenario", "variable", "year", "value") - biiMag <- toolCountryFill(biiMag) # Taiwan and many islands are missing from this dataset - # For each NA, set BII to this year's mean value - for (y in getYears(biiMag)) { - biiMag[, y, ][which(is.na(biiMag[, y, ]))] <- mean(biiMag[, y, ], na.rm = TRUE) - } + biiMag <- as.magpie(bii) - biiMag <- magpiesort(biiMag) + biiMag <- toolCountryFill(biiMag) # Taiwan and many islands are missing from this dataset + # For each NA, set BII to this year's mean value + for (y in getYears(biiMag)) { + biiMag[, y, ][which(is.na(biiMag[, y, ]))] <- mean(biiMag[, y, ], na.rm = TRUE) + } - return(biiMag) + biiMag <- magpiesort(biiMag) + return(biiMag) } diff --git a/README.md b/README.md index b9655419..9b638c7e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # madrat data preparation for validation purposes -R package **mrvalidation**, version **2.67.1** +R package **mrvalidation**, version **2.69.1** [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4317826.svg)](https://doi.org/10.5281/zenodo.4317826) [![R build status](https://github.com/pik-piam/mrvalidation/workflows/check/badge.svg)](https://github.com/pik-piam/mrvalidation/actions) [![codecov](https://codecov.io/gh/pik-piam/mrvalidation/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/mrvalidation) [![r-universe](https://pik-piam.r-universe.dev/badges/mrvalidation)](https://pik-piam.r-universe.dev/builds) @@ -21,13 +21,13 @@ The additional repository can be made available permanently by adding the line a After that the most recent version of the package can be installed using `install.packages`: -```r +```r install.packages("mrvalidation") ``` Package updates can be installed using `update.packages` (make sure that the additional repository has been added before running that command): -```r +```r update.packages() ``` @@ -39,7 +39,7 @@ In case of questions / problems please contact Benjamin Leon Bodirsky , Version: 2.67.1, . +Bodirsky B, Wirth S, Karstens K, Humpenoeder F, Stevanovic M, Mishra A, Biewald A, Weindl I, Beier F, Chen D, Crawford M, Leip D, Molina Bacca E, Kreidenweis U, W. Yalew A, von Jeetze P, Wang X, Dietrich J, Alves M (2026). "mrvalidation: madrat data preparation for validation purposes." doi:10.5281/zenodo.4317826 , Version: 2.69.1, . A BibTeX entry for LaTeX users is @@ -48,9 +48,9 @@ A BibTeX entry for LaTeX users is title = {mrvalidation: madrat data preparation for validation purposes}, author = {Benjamin Leon Bodirsky and Stephen Wirth and Kristine Karstens and Florian Humpenoeder and Mishko Stevanovic and Abhijeet Mishra and Anne Biewald and Isabelle Weindl and Felicitas Beier and David Chen and Michael Crawford and Debbora Leip and Edna {Molina Bacca} and Ulrich Kreidenweis and Amsalu {W. Yalew} and Patrick {von Jeetze} and Xiaoxi Wang and Jan Philipp Dietrich and Marcos Alves}, doi = {10.5281/zenodo.4317826}, - date = {2026-05-12}, + date = {2026-05-28}, year = {2026}, url = {https://github.com/pik-piam/mrvalidation}, - note = {Version: 2.67.1}, + note = {Version: 2.69.1}, } ``` diff --git a/man/calcValidBII.Rd b/man/calcValidBII.Rd index 5470c7e8..a5936d68 100644 --- a/man/calcValidBII.Rd +++ b/man/calcValidBII.Rd @@ -4,21 +4,57 @@ \alias{calcValidBII} \title{calcValidBII} \usage{ -calcValidBII() +calcValidBII(datasource = "Phillips") +} +\arguments{ +\item{datasource}{"Phillips" (default) or "DePalma".} } \value{ -Country-level MAgPIE object +Country-level MAgPIE object for the selected source under variable + "Biodiversity|BII". } \description{ -validation for the Biodiversity Intactness Index (BII) +Validation for the Biodiversity Intactness Index (BII) from one +NHM source, selected via \code{datasource}: + * "Phillips" -- Phillips et al. 2021 (BII-BTE, DOI 10.5519/he1eqmg1), + country-level 1970-2014. + * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 (DOI 10.5519/k33reyb6), + cellular 5-yearly 2000-2020, aggregated to ISO here. + +De Palma et al. 2024 differs from Phillips et al. 2021 in three steps: +stricter primary-vegetation baseline (minimally/lightly-used primary only, +vs Phillips's all-primary including intensely-used sites), logit (not log) +transform, and balanced Bray-Curtis (abundance-weighted) compositional +similarity in place of Phillips's asymmetric Jaccard (species-identity only). +De Palma's absolute BII is markedly lower, with a non-uniform gap that +depends on land-use intensity, so the two series are NOT splice-compatible +and are kept as separate model labels. + +MAgPIE's own BII output (modules/44_biodiversity, coefficients from +Leclère et al. 2020 "Bending the Curve" via PREDICTS) is on the Phillips +scale, so it is close to Phillips et al. 2021 (~0.03 above) but sits markedly +above De Palma et al. 2024. For absolute-level validation use Phillips; for +harmonized (change-from-baseline) plots either series works, but mixing +MAgPIE projections with De Palma historical on a non-harmonized axis is +misleading. Two things drive the MAgPIE-vs-De Palma gap: the PREDICTS +coefficient version (Phillips vs De Palma), and land-use allocation -- MAgPIE +land use is LUH3-initialised with FAO/FRA forest correction +(mrlandcore::calcForestArea), vs the LUH2 underlying De Palma's v2.1.1. A 2010 +decomposition shows the coefficient change dominates the gap (~0.17 of the +~0.20 total globally, and larger than the land-use term in every H12 region); +the land-use/forest difference is real but secondary (~0.03 globally). } \examples{ \dontrun{ - calcOutput("ValidBII") + calcOutput("ValidBII", datasource = "Phillips") + calcOutput("ValidBII", datasource = "DePalma") } +} +\seealso{ +\code{\link{readBII}}, \code{\link{downloadBII}} } \author{ -Michael Crawford +Michael Crawford, Florian Humpenoeder } diff --git a/man/downloadBII.Rd b/man/downloadBII.Rd index a65b36dc..dc773c2c 100644 --- a/man/downloadBII.Rd +++ b/man/downloadBII.Rd @@ -4,25 +4,41 @@ \alias{downloadBII} \title{downloadBII} \usage{ -downloadBII() +downloadBII(subtype = "Phillips") +} +\arguments{ +\item{subtype}{"Phillips" (default) or "DePalma".} } \value{ -Metadata on downloaded BII data +Metadata on the downloaded BII data. } \description{ -Download historical and projected BII dataset from Phillips et al. 2021 -(https://doi.org/10.5519/he1eqmg1). +Download a Biodiversity Intactness Index dataset from the +Natural History Museum. Two subtypes: + * "Phillips" (default) -- Phillips et al. 2021 BII-BTE + (\doi{10.5519/he1eqmg1}); country/region tables, 1970-2050 incl. SSPs. + * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 (\doi{10.5519/k33reyb6}); + global 5 arc-minute GeoTIFFs for 2000, 2005, 2010, 2015, 2020. +} +\details{ +Both NHM resource URLs are behind Cloudflare and return HTTP 403 to +non-browser clients, so both downloads use the portal's content-addressable +\code{/downloads/direct/.zip} endpoint, which bypasses the challenge. +Phillips uses the CSV resource (the .rds resource has no working direct URL); +De Palma uses the v2.1.1 raster ZIP. If a URL becomes unreachable, download +via a browser and place the file in this source folder. } \examples{ \dontrun{ downloadSource("BII") + downloadSource("BII", subtype = "DePalma") } } \seealso{ -https://data.nhm.ac.uk/dataset/bii-bte +https://data.nhm.ac.uk/dataset/bii-bte, \code{\link{readBII}} } \author{ -Michael Crawford +Michael Crawford, Florian Humpenoeder } diff --git a/man/readBII.Rd b/man/readBII.Rd index 40492886..7b271029 100644 --- a/man/readBII.Rd +++ b/man/readBII.Rd @@ -4,35 +4,51 @@ \alias{readBII} \title{readBII} \usage{ -readBII(subtype = "historical", subset = "bii") +readBII(subtype = "Phillips", subset = "bii") } \arguments{ -\item{subtype}{scenario selection, with potential values: "historical", "ssp1rcp2p6image", -"ssp2rcp4p5messageglobiom", "ssp3rcp7p0aim", "ssp4rcp6p0gcam", "ssp5rcp8p5remindmagpie". -Lists are accepted.} +\item{subtype}{Dataset: "Phillips" (default, Phillips et al. 2021 historical +BII) or "DePalma" (De Palma et al. 2024 v2.1.1 rasters).} -\item{subset}{variable selection, with potential values: "bii", "crops", "highintensityag", -"hpd", "pastureland", "qualitynatural", "urbanextent". Lists are accepted.} +\item{subset}{Phillips variable selection ("bii", "crops", "highintensityag", +"hpd", "pastureland", "qualitynatural", "urbanextent"; lists accepted). +Ignored for subtype = "DePalma".} } \value{ -magclass object containing the desired subtypes and subsets +magclass object. Phillips: ISO x year x scenario.variable. DePalma: + x.y.iso x year x "bii" (fraction 0-1). } \description{ -Read and select scenarios and variables from the "The Biodiversity Intactness Index - -country, region and global-level summaries for the year 1970 to 2050 under various scenarios" -dataset. -Default behavior is to return BII for the historical period. +Read the Biodiversity Intactness Index from the Natural History +Museum. Two dataset subtypes: + * "Phillips" (default) -- Phillips et al. 2021 BII-BTE, historical scenario, + country-level, returned as ISO x year x scenario.variable. + * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 -- cellular x.y.iso on the + 67420-cell lpjcell grid, fraction (0-1). Country aggregation is left to + the consumer (calcValidBII). +} +\details{ +The De Palma rasters are 5 arc-minute WGS84 GeoTIFFs. terra::aggregate +(fact = 6, fun = "mean") brings them to 30 arc-minute (unweighted mean is +fine within a 6x6 block); terra::project pins to the canonical 0.5-deg grid; +terra::extract samples the 67420 lpjcell coordinates. Source is percent +(0-100), divided by 100 for unit-parity with Phillips. The data sub-dim is +named "variable" so calcValidBII can use getNames(x, dim = "variable") <- ... +on both subtypes. } \examples{ \dontrun{ - readSource("BII") + readSource("BII") # Phillips historical + readSource("BII", subtype = "DePalma") # De Palma 2024 v2.1.1 } } \seealso{ -https://data.nhm.ac.uk/dataset/bii-bte +https://data.nhm.ac.uk/dataset/bii-bte, + https://data.nhm.ac.uk/dataset/bii-developed-by-nhm-v2-1-1-limited-release, + \code{\link{downloadBII}}, \code{\link{calcValidBII}} } \author{ -Michael Crawford +Michael Crawford, Florian Humpenoeder }