From 2ce6726c444b8b43ffcfc2d17cef52ea5083487c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:34:06 +0000 Subject: [PATCH 1/4] ci: Restore database backend runs via the generic matrix "env" field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync .github/workflows/R-CMD-check.yaml and .github/workflows/versions-matrix/action.R from cynkratemplate (cynkra/cynkratemplate#95). The rcc-full job now applies the generic "env" field of test matrix entries to $GITHUB_ENV itself, before the custom actions run, because composite actions cannot read the matrix context. This fixes a silent breakage: the backend matrix entries carried a bare "test_src" field, and the custom before-install action tried to read it via `${{ matrix.config.test-src }}` — which resolves to an empty string inside a composite action, as visible in the job logs (`DM_TEST_SRC:` empty). As a result, DM_TEST_SRC was never set, tests/testthat/helper-src.R fell back to the "df" source, and the six "... with covr" jobs tested plain data frames instead of their advertised backends, skipping the server provisioning in the custom after-install action. The backend entries now pass DM_TEST_SRC through the "env" field, so the after-install action provisions the matching server again and the tests really run against SQL Server, Postgres, MariaDB, MySQL, DuckDB, and SQLite. The entries also set SKIP_UPDATE_SNAPSHOTS=true, keeping the snapshot updater from running the test suite a second time against the backend and accepting backend-specific snapshots; the instrumented-validation entry sets the flag directly, too. The custom before-install action loses the "env" input, the apply boilerplate, the broken DM_TEST_SRC step, and the DM_VALIDATE-triggered snapshot opt-out, all of which are now covered by the workflow and the matrix. --- .github/versions-matrix.R | 35 +++++++++++------- .github/workflows/R-CMD-check.yaml | 19 +++++++--- .../custom/before-install/action.yml | 36 +++---------------- .github/workflows/versions-matrix/action.R | 11 +++++- 4 files changed, 53 insertions(+), 48 deletions(-) diff --git a/.github/versions-matrix.R b/.github/versions-matrix.R index 33a7b24c0..1bc7dd517 100644 --- a/.github/versions-matrix.R +++ b/.github/versions-matrix.R @@ -1,14 +1,25 @@ list( + # Backend runs: DM_TEST_SRC selects the database backend + # (see tests/testthat/helper-src.R), and the custom after-install action + # provisions the matching server. The variables travel through the generic + # "env" matrix field, which the rcc-full job applies to the environment of + # all steps. SKIP_UPDATE_SNAPSHOTS opts these entries out of the snapshot + # updater, which would otherwise run the test suite a second time against + # the backend and accept backend-specific snapshots. data.frame( os = "ubuntu-22.04", r = "release", - test_src = c( - "test-mssql", - "test-postgres", - "test-maria", - "test-mysql-maria", - "test-duckdb", - "test-sqlite" + env = paste0( + "DM_TEST_SRC=", + c( + "test-mssql", + "test-postgres", + "test-maria", + "test-mysql-maria", + "test-duckdb", + "test-sqlite" + ), + "\nSKIP_UPDATE_SNAPSHOTS=true" ), covr = "true", desc = c( @@ -20,14 +31,14 @@ list( "SQLite with covr" ) ), - # Instrumented validation run. The DM_VALIDATE flag travels through the - # generic "env" matrix field to the custom before/after-install actions, - # which uncomment code marked "# INSTRUMENT: validate" and opt out of the - # snapshot updater. covr then builds and tests the instrumented sources. + # Instrumented validation run. DM_VALIDATE makes the custom after-install + # action uncomment code marked "# INSTRUMENT: validate"; covr then builds + # and tests the instrumented sources. SKIP_UPDATE_SNAPSHOTS opts out of the + # snapshot updater, which would otherwise run the instrumented tests twice. data.frame( os = "ubuntu-22.04", r = "release", - env = "DM_VALIDATE=true", + env = "DM_VALIDATE=true\nSKIP_UPDATE_SNAPSHOTS=true", covr = "true", desc = "instrumented validation" ) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 72645a267..a0883aa16 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -317,12 +317,23 @@ jobs: with: ref: ${{ needs.rcc-smoke.outputs.sha }} + - name: Apply environment variables from the test matrix + # Generic: matrix entries defined in .github/versions-matrix.R can + # carry an "env" field (one KEY=VALUE per line). + # Write it to $GITHUB_ENV here, in the workflow, + # because composite actions cannot read the matrix context. + # The variables then reach every subsequent step, + # including the custom before-install and after-install actions, + # without requiring repos to implement any plumbing of their own. + if: matrix.env != '' + env: + MATRIX_ENV: ${{ matrix.env }} + run: | + printf '%s\n' "$MATRIX_ENV" | tee -a "$GITHUB_ENV" + shell: bash + - uses: ./.github/workflows/custom/before-install if: hashFiles('.github/workflows/custom/before-install/action.yml') != '' - with: - # Generic: forward any per-entry environment variables defined in the - # test matrix (see .github/versions-matrix.R) to the custom action. - env: ${{ matrix.env }} - uses: ./.github/workflows/install with: diff --git a/.github/workflows/custom/before-install/action.yml b/.github/workflows/custom/before-install/action.yml index 06cb36edf..39d5eee2a 100644 --- a/.github/workflows/custom/before-install/action.yml +++ b/.github/workflows/custom/before-install/action.yml @@ -1,45 +1,19 @@ name: 'Custom steps to run before R packages are installed' -inputs: - env: - description: > - Additional environment variables (one KEY=VALUE per line) contributed by - the test matrix. Generic hook used by .github/versions-matrix.R to flag - special matrix entries (e.g. the instrumented-validation entry below). - required: false - default: "" +# Environment variables from the "env" field of test matrix entries +# (see .github/versions-matrix.R: DM_TEST_SRC, DM_VALIDATE, +# SKIP_UPDATE_SNAPSHOTS) are applied by the rcc-full job in R-CMD-check.yaml +# before this action runs, so they are already visible here +# and in all later steps. runs: using: "composite" steps: - - name: Apply environment variables from the test matrix - if: inputs.env != '' - env: - MATRIX_ENV: ${{ inputs.env }} - run: | - printf '%s\n' "$MATRIX_ENV" | tee -a "$GITHUB_ENV" - shell: bash - - # The instrumented-validation matrix entry sets DM_VALIDATE=true via the - # generic "env" field. Opt it out of the snapshot updater, which bypasses - # tests/testthat.R via testthat::test_local() and would otherwise run the - # instrumented tests twice. - - name: Opt out of the snapshot updater (instrumented validation) - if: env.DM_VALIDATE != '' - run: | - echo 'SKIP_UPDATE_SNAPSHOTS=true' | tee -a "$GITHUB_ENV" - shell: bash - - name: Define _R_CHECK_PKG_SIZES_THRESHOLD_ run: | echo '_R_CHECK_PKG_SIZES_THRESHOLD_=10' | tee -a $GITHUB_ENV shell: bash - - name: Define DM_TEST_SRC - run: | - echo "DM_TEST_SRC=${{ matrix.config.test-src }}" | tee -a $GITHUB_ENV - shell: bash - - name: Clean up broken mysql apt # FIXME: Remove if package becomes unavailable if: runner.os == 'Linux' diff --git a/.github/workflows/versions-matrix/action.R b/.github/workflows/versions-matrix/action.R index 30f03f9a2..5a7bd6550 100644 --- a/.github/workflows/versions-matrix/action.R +++ b/.github/workflows/versions-matrix/action.R @@ -67,9 +67,18 @@ if (!is.na(filter)) { to_json <- function(x) { if (nrow(x) == 0) return(character()) + # Minimal JSON string escaping: backslash, double quote, newline. + # Newlines matter for the "env" field, + # which carries one KEY=VALUE per line. + escape <- function(v) { + v <- gsub("\\", "\\\\", v, fixed = TRUE) + v <- gsub('"', '\\"', v, fixed = TRUE) + v <- gsub("\n", "\\n", v, fixed = TRUE) + v + } parallel <- vector("list", length(x)) for (i in seq_along(x)) { - parallel[[i]] <- paste0('"', names(x)[[i]], '":"', x[[i]], '"') + parallel[[i]] <- paste0('"', escape(names(x)[[i]]), '":"', escape(x[[i]]), '"') } paste0("{", do.call(paste, c(parallel, sep = ",")), "}") } From da11af6843614d32ec9aafaa2b1d15e35f9266c8 Mon Sep 17 00:00:00 2001 From: krlmlr Date: Mon, 27 Jul 2026 18:42:36 +0000 Subject: [PATCH 2/4] chore: Auto-update from GitHub Actions Run: https://github.com/cynkra/dm/actions/runs/30294375847 --- tests/testthat/_snaps/dm_sql.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/testthat/_snaps/dm_sql.md b/tests/testthat/_snaps/dm_sql.md index 186645ad1..15813a0f7 100644 --- a/tests/testthat/_snaps/dm_sql.md +++ b/tests/testthat/_snaps/dm_sql.md @@ -2,6 +2,15 @@ Code dm_for_filter() %>% dm_sql(test_src_duckdb()$con) + Message + duckdb keeps downloaded extensions and secrets in a temporary directory: + i /tmp/RtmpA6QeBX/duckdb + This is removed when the R session ends. + * Extensions are re-downloaded each session. + * Secrets are lost. + i Run duckdb(shared_home = TRUE) (or create ~/.duckdb) to keep them (suitable for most users). + i Run duckdb(shared_home = FALSE) to accept the temporary directory (and silence this message). + i See ?duckdb_storage for details and alternatives. Condition Warning: Autoincrementing columns not yet supported for DuckDB, these won't be set in the remote database but are preserved in the `dm` From a57233752d9b1063b6063317303909892ed35386 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 18:55:02 +0000 Subject: [PATCH 3/4] test: Muffle the duckdb storage-location message in test helpers duckdb 1.5.5 (on the binary repositories since today) announces its extensions-and-secrets location whenever a driver is created without an explicit choice, with a session-specific temporary path. The snapshot updater captured that message and path into tests/testthat/_snaps/dm_sql.md, and the subsequent check step failed because its own session used a different path. Suppress the message by its condition class ("duckdb_storage_location") in the two test helpers that create duckdb connections, and restore the snapshot to its pre-pollution state. --- tests/testthat/_snaps/dm_sql.md | 9 --------- tests/testthat/helper-config-db.R | 9 ++++++++- tests/testthat/helper-src.R | 9 ++++++++- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/testthat/_snaps/dm_sql.md b/tests/testthat/_snaps/dm_sql.md index 15813a0f7..186645ad1 100644 --- a/tests/testthat/_snaps/dm_sql.md +++ b/tests/testthat/_snaps/dm_sql.md @@ -2,15 +2,6 @@ Code dm_for_filter() %>% dm_sql(test_src_duckdb()$con) - Message - duckdb keeps downloaded extensions and secrets in a temporary directory: - i /tmp/RtmpA6QeBX/duckdb - This is removed when the R session ends. - * Extensions are re-downloaded each session. - * Secrets are lost. - i Run duckdb(shared_home = TRUE) (or create ~/.duckdb) to keep them (suitable for most users). - i Run duckdb(shared_home = FALSE) to accept the temporary directory (and silence this message). - i See ?duckdb_storage for details and alternatives. Condition Warning: Autoincrementing columns not yet supported for DuckDB, these won't be set in the remote database but are preserved in the `dm` diff --git a/tests/testthat/helper-config-db.R b/tests/testthat/helper-config-db.R index 74918f28a..a6246f7e8 100644 --- a/tests/testthat/helper-config-db.R +++ b/tests/testthat/helper-config-db.R @@ -13,7 +13,14 @@ test_src_duckdb <- function() { testthat::skip_if_not_installed("DBI") testthat::skip_if_not_installed("dbplyr") testthat::skip_if_not_installed("duckdb") - dbplyr::src_dbi(DBI::dbConnect(duckdb::duckdb()), auto_disconnect = TRUE) + # duckdb >= 1.5.5 announces its storage location on driver creation + # (condition class "duckdb_storage_location") with a session-specific path; + # muffle it so it cannot leak into snapshots. + con <- suppressMessages( + DBI::dbConnect(duckdb::duckdb()), + classes = "duckdb_storage_location" + ) + dbplyr::src_dbi(con, auto_disconnect = TRUE) } test_src_postgres <- function() { diff --git a/tests/testthat/helper-src.R b/tests/testthat/helper-src.R index 803bbc20e..af2ce9e7e 100644 --- a/tests/testthat/helper-src.R +++ b/tests/testthat/helper-src.R @@ -107,7 +107,14 @@ duckdb_test_src %<--% testthat::skip("duckdb failing for R < 4.0") } testthat::skip_if_not_installed("duckdb") - dbplyr::src_dbi(DBI::dbConnect(duckdb::duckdb()), auto_disconnect = TRUE) + # duckdb >= 1.5.5 announces its storage location on driver creation + # (condition class "duckdb_storage_location") with a session-specific path; + # muffle it so it cannot leak into snapshots. + con <- suppressMessages( + DBI::dbConnect(duckdb::duckdb()), + classes = "duckdb_storage_location" + ) + dbplyr::src_dbi(con, auto_disconnect = TRUE) } my_db_test_src <- function() { From e2c7bc4f2cf342947965df129f4f00a07e706bb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Jul 2026 20:33:17 +0000 Subject: [PATCH 4/4] test: Drop the duckdb storage-message suppression again Since #2487 the custom before-install action creates ~/.duckdb and points DUCKDB_R_HOME at it, which counts as an explicit storage choice, so duckdb announces nothing on driver creation and the test helpers no longer need to muffle the "duckdb_storage_location" message. Restore the bare connections. --- tests/testthat/helper-config-db.R | 9 +-------- tests/testthat/helper-src.R | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/testthat/helper-config-db.R b/tests/testthat/helper-config-db.R index a6246f7e8..74918f28a 100644 --- a/tests/testthat/helper-config-db.R +++ b/tests/testthat/helper-config-db.R @@ -13,14 +13,7 @@ test_src_duckdb <- function() { testthat::skip_if_not_installed("DBI") testthat::skip_if_not_installed("dbplyr") testthat::skip_if_not_installed("duckdb") - # duckdb >= 1.5.5 announces its storage location on driver creation - # (condition class "duckdb_storage_location") with a session-specific path; - # muffle it so it cannot leak into snapshots. - con <- suppressMessages( - DBI::dbConnect(duckdb::duckdb()), - classes = "duckdb_storage_location" - ) - dbplyr::src_dbi(con, auto_disconnect = TRUE) + dbplyr::src_dbi(DBI::dbConnect(duckdb::duckdb()), auto_disconnect = TRUE) } test_src_postgres <- function() { diff --git a/tests/testthat/helper-src.R b/tests/testthat/helper-src.R index af2ce9e7e..803bbc20e 100644 --- a/tests/testthat/helper-src.R +++ b/tests/testthat/helper-src.R @@ -107,14 +107,7 @@ duckdb_test_src %<--% testthat::skip("duckdb failing for R < 4.0") } testthat::skip_if_not_installed("duckdb") - # duckdb >= 1.5.5 announces its storage location on driver creation - # (condition class "duckdb_storage_location") with a session-specific path; - # muffle it so it cannot leak into snapshots. - con <- suppressMessages( - DBI::dbConnect(duckdb::duckdb()), - classes = "duckdb_storage_location" - ) - dbplyr::src_dbi(con, auto_disconnect = TRUE) + dbplyr::src_dbi(DBI::dbConnect(duckdb::duckdb()), auto_disconnect = TRUE) } my_db_test_src <- function() {