Add BII v2.1.1 (De Palma et al. 2024) as second model in calcValidBII

.buildlibrary

@@ -1,4 +1,4 @@
-ValidationKey: '54982535'
+ValidationKey: '55437291'
 AutocreateReadme: yes
 AcceptedWarnings:
 - 'Warning: package .* was built under R version'

.github/workflows/check.yaml

@@ -23,7 +23,7 @@ jobs:
           options(repos = c(pikpiam = 'https://pik-piam.r-universe.dev',
                             CRAN = Sys.getenv('RSPM')))
           pak::pak()
-          
+
       - name: Run pre-commit checks
         shell: bash
         run: |

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'mrvalidation: madrat data preparation for validation purposes'
-version: 2.67.1
-date-released: '2026-05-12'
+version: 2.69.1
+date-released: '2026-05-28'
 abstract: Package contains routines to prepare data for validation exercises.
 authors:
 - family-names: Bodirsky

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: mrvalidation
 Title: madrat data preparation for validation purposes
-Version: 2.67.1
-Date: 2026-05-12
+Version: 2.69.1
+Date: 2026-05-28
 Authors@R: c(
     person("Benjamin Leon", "Bodirsky", , "bodirsky@pik-potsdam.de", role = c("aut", "cre")),
     person("Stephen", "Wirth", role = "aut"),
@@ -53,6 +53,7 @@ Imports:
     reshape2,
     rlang,
     stringr,
+    terra,
     tidyr,
     utils,
     withr,

NAMESPACE

@@ -104,6 +104,10 @@ importFrom(rlang,.data)
 importFrom(stringr,str_extract)
 importFrom(stringr,str_match)
 importFrom(stringr,str_split)
+importFrom(terra,aggregate)
+importFrom(terra,extract)
+importFrom(terra,project)
+importFrom(terra,rast)
 importFrom(tidyr,drop_na)
 importFrom(tidyr,expand_grid)
 importFrom(tidyr,nest)

R/calcValidBII.R

@@ -1,28 +1,84 @@
 #' @title calcValidBII
-#' @description validation for the Biodiversity Intactness Index (BII)
-#' @author Michael Crawford
+#' @description Validation for the Biodiversity Intactness Index (BII) from one
+#' NHM source, selected via \code{datasource}:
+#'   * "Phillips" -- Phillips et al. 2021 (BII-BTE, DOI 10.5519/he1eqmg1),
+#'     country-level 1970-2014.
+#'   * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 (DOI 10.5519/k33reyb6),
+#'     cellular 5-yearly 2000-2020, aggregated to ISO here.
 #'
-#' @return Country-level MAgPIE object
+#' De Palma et al. 2024 differs from Phillips et al. 2021 in three steps:
+#' stricter primary-vegetation baseline (minimally/lightly-used primary only,
+#' vs Phillips's all-primary including intensely-used sites), logit (not log)
+#' transform, and balanced Bray-Curtis (abundance-weighted) compositional
+#' similarity in place of Phillips's asymmetric Jaccard (species-identity only).
+#' De Palma's absolute BII is markedly lower, with a non-uniform gap that
+#' depends on land-use intensity, so the two series are NOT splice-compatible
+#' and are kept as separate model labels.
+#'
+#' MAgPIE's own BII output (modules/44_biodiversity, coefficients from
+#' Leclère et al. 2020 "Bending the Curve" via PREDICTS) is on the Phillips
+#' scale, so it is close to Phillips et al. 2021 (~0.03 above) but sits markedly
+#' above De Palma et al. 2024. For absolute-level validation use Phillips; for
+#' harmonized (change-from-baseline) plots either series works, but mixing
+#' MAgPIE projections with De Palma historical on a non-harmonized axis is
+#' misleading. Two things drive the MAgPIE-vs-De Palma gap: the PREDICTS
+#' coefficient version (Phillips vs De Palma), and land-use allocation -- MAgPIE
+#' land use is LUH3-initialised with FAO/FRA forest correction
+#' (mrlandcore::calcForestArea), vs the LUH2 underlying De Palma's v2.1.1. A 2010
+#' decomposition shows the coefficient change dominates the gap (~0.17 of the
+#' ~0.20 total globally, and larger than the land-use term in every H12 region);
+#' the land-use/forest difference is real but secondary (~0.03 globally).
+#' @author Michael Crawford, Florian Humpenoeder
+#' @seealso \code{\link{readBII}}, \code{\link{downloadBII}}
+#'
+#' @param datasource "Phillips" (default) or "DePalma".
+#'
+#' @return Country-level MAgPIE object for the selected source under variable
+#'   "Biodiversity|BII".
 #'
 #' @examples
 #'
 #' \dontrun{
-#'    calcOutput("ValidBII")
+#'    calcOutput("ValidBII", datasource = "Phillips")
+#'    calcOutput("ValidBII", datasource = "DePalma")
 #' }
 #'
+#' @importFrom madrat toolAggregate toolCountryFill toolNAreplace
 
-calcValidBII <- function() {
+calcValidBII <- function(datasource = "Phillips") {
 
-  bii <- readSource("BII", subtype = "historical", subset = "bii")
-  bii <- add_dimension(bii, dim = 3.2, add = "model", nm = "Phillips et al")
-  getNames(bii, dim = "variable") <- "Biodiversity|BII"
+  if (identical(datasource, "Phillips")) {
+    # Phillips et al. 2021: already country-level.
+    bii   <- readSource("BII", subtype = "Phillips", subset = "bii")
+    bii   <- add_dimension(bii, dim = 3.2, add = "model", nm = "Phillips et al")
+    descr <- "Historical BII, Phillips et al. 2021 (DOI 10.5519/he1eqmg1, 1970-2014)."
 
-  land <- calcOutput("FAOLand", aggregate = FALSE)
-  landArea <- land[, "y2015", "6601|Land area"] # total land is stable through time, so I arbitrarily use y2015
+  } else if (identical(datasource, "DePalma")) {
+    # De Palma et al. 2024 NHM v2.1.1: cellular -> ISO. toolNAreplace masks both
+    # value and weight to 0 where either is NA, so toolAggregate's weighted mean
+    # ignores water/missing cells instead of zeroing whole countries.
+    biiCell  <- readSource("BII", subtype = "DePalma")
+    cellArea <- calcOutput("LandArea", cells = "lpjcell", aggregate = FALSE)
+    naFix    <- toolNAreplace(x = biiCell, weight = cellArea)
+    bii      <- toolAggregate(naFix$x, weight = naFix$weight, to = "iso", dim = 1,
+                              zeroWeight = "setNA")
+    bii      <- toolCountryFill(bii, fill = NA, verbosity = 2)
+    bii      <- add_dimension(bii, dim = 3.1, add = "scenario", nm = "historical")
+    bii      <- add_dimension(bii, dim = 3.2, add = "model",    nm = "De Palma et al 2024")
+    descr    <- paste("Historical BII, De Palma et al. 2024 v2.1.1",
+                      "(DOI 10.5519/k33reyb6, 2000-2020), cellular aggregated to ISO.")
+
+  } else {
+    stop("Unknown datasource '", datasource, "'. Use \"Phillips\" or \"DePalma\".")
+  }
+
+  getNames(bii, dim = "variable") <- "Biodiversity|BII"
 
-  return(list(x = bii,
-              weight = landArea,
-              unit = "unitless",
-              description = "Historical BII for 1970-2014"))
+  land     <- calcOutput("FAOLand", aggregate = FALSE)
+  landArea <- land[, "y2015", "6601|Land area"]   # land area stable across time
 
+  return(list(x           = bii,
+              weight      = landArea,
+              unit        = "unitless",
+              description = descr))
 }

R/downloadBII.R

@@ -1,56 +1,129 @@
 #' @title downloadBII
-#' @description Download historical and projected BII dataset from Phillips et al. 2021
-#' (https://doi.org/10.5519/he1eqmg1).
-#' @seealso https://data.nhm.ac.uk/dataset/bii-bte
-#' @author Michael Crawford
+#' @description Download a Biodiversity Intactness Index dataset from the
+#' Natural History Museum. Two subtypes:
+#'   * "Phillips" (default) -- Phillips et al. 2021 BII-BTE
+#'     (\doi{10.5519/he1eqmg1}); country/region tables, 1970-2050 incl. SSPs.
+#'   * "DePalma" -- De Palma et al. 2024 NHM v2.1.1 (\doi{10.5519/k33reyb6});
+#'     global 5 arc-minute GeoTIFFs for 2000, 2005, 2010, 2015, 2020.
+#' @param subtype "Phillips" (default) or "DePalma".
+#' @seealso https://data.nhm.ac.uk/dataset/bii-bte, \code{\link{readBII}}
+#' @author Michael Crawford, Florian Humpenoeder
 #'
-#' @return Metadata on downloaded BII data
+#' @return Metadata on the downloaded BII data.
+#'
+#' @details
+#' Both NHM resource URLs are behind Cloudflare and return HTTP 403 to
+#' non-browser clients, so both downloads use the portal's content-addressable
+#' \code{/downloads/direct/<sha>.zip} endpoint, which bypasses the challenge.
+#' Phillips uses the CSV resource (the .rds resource has no working direct URL);
+#' De Palma uses the v2.1.1 raster ZIP. If a URL becomes unreachable, download
+#' via a browser and place the file in this source folder.
 #'
 #' @examples
 #'
 #' \dontrun{
 #'   downloadSource("BII")
+#'   downloadSource("BII", subtype = "DePalma")
 #' }
 #'
-#' @importFrom utils download.file person
+#' @importFrom utils download.file person unzip
 
-downloadBII <- function() {
+downloadBII <- function(subtype = "Phillips") {
 
-    # nolint start
+  # nolint start
 
-    rdsURL  <- "https://data.nhm.ac.uk/dataset/07f701d4-9a26-403c-a9d6-248c895fa412/resource/94be0af6-ec90-4b83-8f02-64a4983e1ca1/download/long_data.rds"
-    download.file(rdsURL, destfile = "BII_historical_and_SSPs.rds")
+  if (identical(subtype, "DePalma")) {
+    url     <- "https://data.nhm.ac.uk/downloads/direct/8161226fda7a36ca0b94f2becee9afe1b318d0a8.zip"
+    zipFile <- "bii-v2-1-1-nhm-data-portal.zip"
+    if (!file.exists(zipFile)) {
+      tryCatch(
+        utils::download.file(url, destfile = zipFile, mode = "wb"),
+        error = function(e) {
+          stop("Automated NHM download failed (Cloudflare likely). Manually download from ",
+               "https://data.nhm.ac.uk/dataset/bii-developed-by-nhm-v2-1-1-limited-release ",
+               "and place '", zipFile, "' in this source folder. Original error: ", conditionMessage(e))
+        }
+      )
+    }
+    # Cloudflare challenge stub is ~6 KB; real ZIP is ~45 MB.
+    if (file.size(zipFile) < 1e7) {
+      stop("'", zipFile, "' is too small (", file.size(zipFile),
+           " bytes) -- likely a Cloudflare challenge page. Manually download via browser.")
+    }
+    # Outer ZIP holds an inner ZIP (resource-ID-named) + manifest; inner ZIP has the 5 GeoTIFFs.
+    utils::unzip(zipFile, exdir = ".")
+    innerZip <- "c4c281c4-befa-4e1b-a162-ba2f25e5ae82.zip"
+    if (!file.exists(innerZip)) {
+      stop("Inner archive '", innerZip, "' missing -- NHM ZIP layout may have changed.")
+    }
+    utils::unzip(innerZip, exdir = ".")
+    return(list(url          = "https://data.nhm.ac.uk/dataset/bii-developed-by-nhm-v2-1-1-limited-release",
+                doi          = "https://doi.org/10.5519/k33reyb6",
+                title        = "The Biodiversity Intactness Index developed by The Natural History Museum, London, v2.1.1 (Open Access, Limited Release)",
+                unit         = "Percentage (0-100)",
+                author       = list(person("Adriana", "De Palma"), person("Sara", "Contu"),
+                                    person("Gareth E", "Thomas"), person("Connor", "Duffin"),
+                                    person("Sabine", "Nix"), person("Andy", "Purvis")),
+                version      = "v2.1.1",
+                release_date = "2024-10-09",
+                description  = "Global gridded BII at 5 arc-minute resolution for 2000, 2005, 2010, 2015, 2020. Values 0-100 (100 = fully intact).",
+                license      = "CC-BY-NC-SA 4.0",
+                reference    = "Adriana De Palma; Sara Contu; Gareth E Thomas; Connor Duffin; Sabine Nix; Andy Purvis (2024). The Biodiversity Intactness Index developed by The Natural History Museum, London, v2.1.1 (Open Access, Limited Release) [Data set]. Natural History Museum. https://doi.org/10.5519/k33reyb6")
+    )
+  }
 
-    return(list(url           = "https://data.nhm.ac.uk/dataset/bii-bte",
-                doi           = "https://doi.org/10.5519/he1eqmg1",
-                title         = "The Biodiversity Intactness Index - country, region and global-level summaries for the year
-                                    1970 to 2050 under various scenarios",
-                unit          = "Unitless",
-                author        = list(person("Helen", "Phillips"),
-                                     person("Adriana", "De Palma"),
-                                     person("Ricardo E", "Gonzalez"),
-                                     person("Sara", "Contu"),
-                                     person("Samantha L L", "Hill"),
-                                     person("Andres", "Baselga"),
-                                     person("Luca", "Borger"),
-                                     person("Andy", "Purvis")),
-                version       = NULL,
-                release_date  = "October 4, 2021",
-                description   = "Using the PREDICTS database of local biodiversity measures at thousands of sites around the world,
-                                    we statistically modelled how total abundance of organisms and compositional similarity responded
-                                    to land use and related pressures. We combined these models with spatio-temporal projections of
-                                    explanatory variables (at 0.25 degrees spatial resolution) from the year 1970 to 2050 under five
-                                    Shared Socioeconomic Pathways (SSPs) to project the Biodiversity Intactness Index (BII). Mean BII
-                                    (weighted by cell area) was calculated at the country, subregion, interregion and global level. We
-                                    used cross-validation (leaving one biome out in turn) to produce decadal upper and lower uncertainty
-                                    margins for 1970-2050. These summary data were uploaded to the Natural History Museum's Biodiversity
-                                    Trends Explorer on 2021-10-27. We have also provided mean values of some of the pressures, as changes
-                                    in these contribute to changes in BII.",
-                license       = "Creative Commons Non-Commercial",
-                reference     = "Helen Phillips; Adriana De Palma; Ricardo E Gonzalez; Sara Contu et al. (2021). The Biodiversity
-                                    Intactness Index - country, region and global-level summaries for the year 1970 to 2050 under
-                                    various scenarios [Data set]. Natural History Museum. ")
+  # Phillips et al. 2021 (default). CKAN resource URL is Cloudflare-blocked;
+  # use the /downloads/direct/<sha>.zip endpoint (CSV resource -- the .rds has
+  # no working direct URL). The ZIP holds resource.csv + manifest.json.
+  url     <- "https://data.nhm.ac.uk/downloads/direct/bcaa1a8726032605ed11b1db0cb503d6e00ad584.zip"
+  zipFile <- "bii-bte-nhm-data-portal.zip"
+  if (!file.exists(zipFile)) {
+    tryCatch(
+      utils::download.file(url, destfile = zipFile, mode = "wb"),
+      error = function(e) {
+        stop("Automated NHM download failed (Cloudflare likely). Manually download from ",
+             "https://data.nhm.ac.uk/dataset/bii-bte and place '", zipFile,
+             "' in this source folder. Original error: ", conditionMessage(e))
+      }
     )
+  }
+  if (file.size(zipFile) < 1e6) {
+    stop("'", zipFile, "' is too small (", file.size(zipFile),
+         " bytes) -- likely a Cloudflare challenge page. Manually download via browser.")
+  }
+  utils::unzip(zipFile, exdir = ".")
+  if (file.exists("resource.csv")) file.rename("resource.csv", "BII_historical_and_SSPs.csv")
+
+  return(list(url           = "https://data.nhm.ac.uk/dataset/bii-bte",
+              doi           = "https://doi.org/10.5519/he1eqmg1",
+              title         = "The Biodiversity Intactness Index - country, region and global-level summaries for the year
+                                  1970 to 2050 under various scenarios",
+              unit          = "Unitless",
+              author        = list(person("Helen", "Phillips"),
+                                   person("Adriana", "De Palma"),
+                                   person("Ricardo E", "Gonzalez"),
+                                   person("Sara", "Contu"),
+                                   person("Samantha L L", "Hill"),
+                                   person("Andres", "Baselga"),
+                                   person("Luca", "Borger"),
+                                   person("Andy", "Purvis")),
+              version       = NULL,
+              release_date  = "October 4, 2021",
+              description   = "Using the PREDICTS database of local biodiversity measures at thousands of sites around the world,
+                                  we statistically modelled how total abundance of organisms and compositional similarity responded
+                                  to land use and related pressures. We combined these models with spatio-temporal projections of
+                                  explanatory variables (at 0.25 degrees spatial resolution) from the year 1970 to 2050 under five
+                                  Shared Socioeconomic Pathways (SSPs) to project the Biodiversity Intactness Index (BII). Mean BII
+                                  (weighted by cell area) was calculated at the country, subregion, interregion and global level. We
+                                  used cross-validation (leaving one biome out in turn) to produce decadal upper and lower uncertainty
+                                  margins for 1970-2050. These summary data were uploaded to the Natural History Museum's Biodiversity
+                                  Trends Explorer on 2021-10-27. We have also provided mean values of some of the pressures, as changes
+                                  in these contribute to changes in BII.",
+              license       = "Creative Commons Non-Commercial",
+              reference     = "Helen Phillips; Adriana De Palma; Ricardo E Gonzalez; Sara Contu et al. (2021). The Biodiversity
+                                  Intactness Index - country, region and global-level summaries for the year 1970 to 2050 under
+                                  various scenarios [Data set]. Natural History Museum. ")
+  )
 
-    # nolint end
+  # nolint end
 }

R/fullVALIDATION.R

@@ -389,7 +389,9 @@ fullVALIDATION <- function(rev = 0.1, aggregate = "region+global") {
              file = valfile, append = TRUE, try = TRUE)
 
   # Diversity indices
-  calcOutput(type = "ValidBII", aggregate = aggregate,
+  calcOutput(type = "ValidBII", datasource = "Phillips", aggregate = aggregate,
+             file = valfile, append = TRUE, try = TRUE)
+  calcOutput(type = "ValidBII", datasource = "DePalma", aggregate = aggregate,
              file = valfile, append = TRUE, try = TRUE)
   calcOutput(type = "ValidCropareaDiversity", index = "shannon", aggregate = aggregate,
              file = valfile, append = TRUE, try = TRUE)