diff --git a/.github/workflows/filter-nonblocking-test-artifacts.sh b/.github/workflows/filter-nonblocking-test-artifacts.sh new file mode 100644 index 00000000000..5a79b26c4c3 --- /dev/null +++ b/.github/workflows/filter-nonblocking-test-artifacts.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# +# Filters downloaded outerloop test artifacts before the scheduled failure +# reporter enumerates gating failures. +# +# Input: one directory containing downloaded artifacts in this shape: +# /logs-*/testresults/ignore-test-failures.marker +# Output: removes each marked logs-* artifact directory and writes the removed +# paths to stdout. It does not create an output file. +# Failure: exits nonzero for invalid arguments, a missing input directory, or a +# marker outside the expected artifact layout. All markers are validated before +# any directory is removed so an unexpected layout cannot partially filter the +# results. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +all_logs_directory="${1%/}" + +if [[ ! -d "$all_logs_directory" ]]; then + echo "Test artifact directory does not exist: $all_logs_directory" >&2 + exit 1 +fi + +marker_list="$(mktemp)" +trap 'rm -f "$marker_list"' EXIT +find "$all_logs_directory" -type f -name ignore-test-failures.marker -print0 > "$marker_list" + +# Validate every marker before deleting anything so an unexpected artifact +# layout fails closed without partially filtering the downloaded results. +while IFS= read -r -d '' marker; do + artifact_directory="$(dirname "$(dirname "$marker")")" + artifact_parent_directory="$(dirname "$artifact_directory")" + artifact_name="$(basename "$artifact_directory")" + expected_marker="$artifact_directory/testresults/ignore-test-failures.marker" + if [[ "$artifact_parent_directory" != "$all_logs_directory" || + "$artifact_name" != logs-* || + "$marker" != "$expected_marker" ]]; then + echo "Unexpected ignore marker location: $marker" >&2 + exit 1 + fi +done < "$marker_list" + +while IFS= read -r -d '' marker; do + artifact_directory="$(dirname "$(dirname "$marker")")" + echo "Excluding non-gating test results from failure classification: $artifact_directory" + rm -rf -- "$artifact_directory" +done < "$marker_list" diff --git a/.github/workflows/native-dashboard-validation.yml b/.github/workflows/native-dashboard-validation.yml index b7b02508f62..faa6f2fc5c5 100644 --- a/.github/workflows/native-dashboard-validation.yml +++ b/.github/workflows/native-dashboard-validation.yml @@ -37,16 +37,19 @@ jobs: - name: Extract and smoke test Native AOT Dashboard shell: pwsh + env: + ARTIFACTS_DIRECTORY: ${{ github.workspace }}/artifacts/native-dashboard + RID: ${{ inputs.rid }} run: | $ErrorActionPreference = 'Stop' - $archive = @(Get-ChildItem -Path '${{ github.workspace }}/artifacts/native-dashboard' -Filter 'aspire-dashboard-${{ inputs.rid }}.zip' -File -Recurse) + $archive = @(Get-ChildItem -Path $env:ARTIFACTS_DIRECTORY -Filter "aspire-dashboard-$env:RID.zip" -File -Recurse) if ($archive.Count -ne 1) { - throw "Expected one Native AOT Dashboard archive for ${{ inputs.rid }}, found $($archive.Count): $($archive.FullName -join ', ')" + throw "Expected one Native AOT Dashboard archive for $env:RID, found $($archive.Count): $($archive.FullName -join ', ')" } - $extractDirectory = '${{ github.workspace }}/artifacts/native-dashboard/extracted' + $extractDirectory = Join-Path $env:ARTIFACTS_DIRECTORY 'extracted' New-Item -ItemType Directory -Path $extractDirectory | Out-Null - if ('${{ runner.os }}' -eq 'Windows') { + if ($env:RUNNER_OS -eq 'Windows') { Expand-Archive -LiteralPath $archive[0].FullName -DestinationPath $extractDirectory $dashboardPath = Join-Path $extractDirectory 'Aspire.Dashboard.exe' } @@ -61,19 +64,40 @@ jobs: Add-Content -LiteralPath $env:GITHUB_ENV -Value "ASPIRE_NATIVE_DASHBOARD_PATH=$dashboardPath" ./eng/scripts/test-native-dashboard.ps1 -DashboardPath $dashboardPath + - name: Test Native AOT Dashboard + if: ${{ inputs.rid == 'win-x64' }} + shell: pwsh + env: + CONFIGURATION: ${{ inputs.configuration }} + NUGET_PACKAGES: ${{ format('{0}/.nuget/packages', github.workspace) }} + run: | + & ./dotnet.cmd test ` + --project tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj ` + -c $env:CONFIGURATION ` + --no-launch-profile ` + /p:ContinuousIntegrationBuild=true ` + /p:RunOuterloopTests=true ` + -- ` + --filter-method "*.NativeDashboard_*" ` + --filter-not-trait "quarantined=true" + + $testExitCode = $LASTEXITCODE + & .github/workflows/normalize-mtp-exit-code.ps1 -ExitCode $testExitCode + - name: Test Native AOT Dashboard interactivity if: ${{ inputs.rid == 'win-x64' }} shell: pwsh env: + CONFIGURATION: ${{ inputs.configuration }} NUGET_PACKAGES: ${{ format('{0}/.nuget/packages', github.workspace) }} run: > ./dotnet.cmd test - --project tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj - -c ${{ inputs.configuration }} + --project tests/Aspire.Dashboard.Playwright.Tests/Aspire.Dashboard.Playwright.Tests.csproj + -c $env:CONFIGURATION --no-launch-profile /p:ContinuousIntegrationBuild=true /p:InstallBrowsersForPlaywright=true /p:RunOuterloopTests=true -- - --filter-method "*.NativeDashboard_LoadsInteractivePageWithoutBrowserErrors" - --filter-not-trait "quarantined=true" \ No newline at end of file + --filter-method "*.NativeDashboard_*" + --filter-not-trait "quarantined=true" diff --git a/.github/workflows/report-specialized-test-failures.js b/.github/workflows/report-specialized-test-failures.js index 5ce5ddebd23..aa33a1b6c66 100644 --- a/.github/workflows/report-specialized-test-failures.js +++ b/.github/workflows/report-specialized-test-failures.js @@ -8,8 +8,8 @@ // auto-close-on-green). // // Two failure kinds: -// - 'test-failures' : outerloop tests failed. Each failed run's comment lists -// the failing tests; a dev can split them into per-test issues later. +// - 'test-failures' : gating outerloop tests failed. Each failed run's comment +// lists the failing tests; a dev can split them into per-test issues later. // - 'infra' : the run broke before/around test execution (build/setup, // missing TRX). Quarantine runs swallow test failures (ignoreTestFailures), // so a *failed* quarantine run is always infra. diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 66ba4a199c9..1ca54528c5f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -34,11 +34,18 @@ on: required: false type: string default: "ubuntu-latest" - # When true, ignores test failures and checks if .trx files have nonzero test count + # When true, known test-runner outcomes do not fail the job. Unknown or + # infrastructure/configuration failures remain blocking. ignoreTestFailures: required: false type: boolean default: false + # Specialized workflows can legitimately produce zero tests for a project/OS + # combination after applying their trait filter. + allowZeroTests: + required: false + type: boolean + default: false jobs: @@ -427,14 +434,14 @@ jobs: ${{ inputs.extraTestArgs }} || TEST_EXIT_CODE=$? # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome - # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode. + # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored. echo "$TEST_EXIT_CODE" > "${{ github.workspace }}/test-exit-code.txt" # Stop heartbeat monitor kill $HEARTBEAT_PID 2>/dev/null || true wait $HEARTBEAT_PID 2>/dev/null || true - # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job + # When ignoreTestFailures is true a known test outcome must not fail the job # here; the authoritative pass/fail decision is made later by "Verify test results exist". if [ "${{ inputs.ignoreTestFailures }}" = "true" ]; then if [ "$TEST_EXIT_CODE" -ne 0 ]; then @@ -490,10 +497,10 @@ jobs: } # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome - # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode. + # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored. Set-Content -Path "${{ github.workspace }}/test-exit-code.txt" -Value $testExitCode - # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job + # When ignoreTestFailures is true a known test outcome must not fail the job # here. The Linux/macOS step neutralizes the exit code inline; this pwsh step captures # $testExitCode instead, so it must explicitly exit 0. Otherwise the non-zero $LASTEXITCODE # survives to the end of the script and GitHub's pwsh wrapper — which appends @@ -555,14 +562,14 @@ jobs: TEST_EXIT_CODE=$NORMALIZED_TEST_EXIT_CODE # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome - # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode. + # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored. echo "$TEST_EXIT_CODE" > "${{ github.workspace }}/test-exit-code.txt" # Stop heartbeat monitor kill $HEARTBEAT_PID 2>/dev/null || true wait $HEARTBEAT_PID 2>/dev/null || true - # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job + # When ignoreTestFailures is true a known test outcome must not fail the job # here; the authoritative pass/fail decision is made later by "Verify test results exist". if [ "${{ inputs.ignoreTestFailures }}" = "true" ]; then if [ "$TEST_EXIT_CODE" -ne 0 ]; then @@ -630,10 +637,10 @@ jobs: } # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome - # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode. + # (test outcome vs. infrastructure failure) even though we exit 0 below when failures are ignored. Set-Content -Path "${{ github.workspace }}/test-exit-code.txt" -Value $testExitCode - # When ignoreTestFailures is true (the quarantine workflow) a failing test must not fail the job + # When ignoreTestFailures is true a known test outcome must not fail the job # here. The Linux/macOS step neutralizes the exit code inline; this pwsh step captures # $testExitCode instead, so it must explicitly exit 0. Otherwise the non-zero $LASTEXITCODE # survives to the end of the script and GitHub's pwsh wrapper — which appends @@ -648,87 +655,30 @@ jobs: exit $testExitCode + - name: Mark ignored test results + if: ${{ always() && inputs.ignoreTestFailures }} + shell: pwsh + env: + TEST_SHORT_NAME: ${{ inputs.testShortName }} + run: | + $testResultsDir = "${{ github.workspace }}/testresults" + New-Item -Path $testResultsDir -ItemType Directory -Force | Out-Null + Set-Content -Path "$testResultsDir/ignore-test-failures.marker" -Value $env:TEST_SHORT_NAME + - name: Verify test results exist if: ${{ inputs.ignoreTestFailures }} shell: pwsh + env: + ALLOW_ZERO_TESTS: ${{ inputs.allowZeroTests }} run: | - # Classify the test run using the MTP exit code captured by the test step. - # https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes - # In quarantine mode (ignoreTestFailures) we tolerate genuine test outcomes — 2 (a test failed), - # 3 (session aborted, e.g. a hangdump timeout), 7 (the test host crashed) and 13 (--maximum-failed-tests - # reached) — because quarantined tests are flaky by definition. But a broken harness must still fail - # the job, otherwise the quarantine run stays permanently green and hides real breakage: - # 4 = invalid extension setup - # 5 = invalid command-line arguments - # 10 = test adapter infrastructure failure (e.g. a fixture could not be created) - # 11 = the test process exited because a dependent process exited - # 12 = no supported protocol version - # (Exit code 8 "zero tests ran" is mapped to 0 either by --ignore-exit-code 8 for direct assembly - # execution or explicitly by the dotnet test steps above, so a genuinely empty run is caught below - # by the .trx/artifact checks rather than by the exit code.) - $exitCodeFile = "${{ github.workspace }}/test-exit-code.txt" - if (Test-Path $exitCodeFile) { - $mtpExitCode = [int]((Get-Content -Path $exitCodeFile -Raw).Trim()) - $fatalInfraCodes = @(4, 5, 10, 11, 12) - if ($fatalInfraCodes -contains $mtpExitCode) { - Write-Host "::error::Test runner exited with code $mtpExitCode, which indicates a test infrastructure/configuration failure rather than a test failure. Failing the job even though ignoreTestFailures is true." - exit 1 - } - Write-Host "Test runner exit code: $mtpExitCode (treated as a test outcome, not an infrastructure failure)" - } else { - Write-Warning "No test exit code file found at $exitCodeFile; relying on .trx/artifact checks only." + $arguments = @{ + TestResultsPath = "${{ github.workspace }}/testresults" + ExitCodePath = "${{ github.workspace }}/test-exit-code.txt" } - - $trxFiles = Get-ChildItem -Path "${{ github.workspace }}/testresults" -Filter *.trx -Recurse -ErrorAction SilentlyContinue - - if ($trxFiles.Count -eq 0) { - # MTP does not produce .trx files when the test run is aborted (e.g., by --hangdump-timeout). - # Check for recordings or other test artifacts that prove tests actually ran before failing. - $hasRecordings = Test-Path "${{ github.workspace }}/testresults/recordings/*.cast" - $hasHangDumps = Get-ChildItem -Path "${{ github.workspace }}/testresults" -Filter *hangdump* -Recurse -ErrorAction SilentlyContinue - if ($hasRecordings -or $hasHangDumps) { - Write-Warning "No .trx files found, but test artifacts exist — test run likely aborted by timeout before trx was flushed." - exit 0 - } - Write-Error "No .trx files found. Tests may not have run due to infrastructure issues." - exit 1 - } - - Write-Host "Found $($trxFiles.Count) .trx file(s)" - - $validFileCount = 0 - foreach ($trxFile in $trxFiles) { - Write-Host "Checking $($trxFile.Name)..." - - try { - [xml]$trxContent = Get-Content -Path $trxFile.FullName - - # Check if the XML is valid and non-empty - if ($trxContent -and $trxContent.TestRun) { - $validFileCount++ - $countersNode = $trxContent.TestRun.ResultSummary.Counters - - if ($countersNode -and $countersNode.total) { - $testCount = [int]$countersNode.total - Write-Host " Total tests in file: $testCount" - } else { - Write-Host " File is valid but has no test count information" - } - } else { - Write-Warning " File appears to be empty or invalid" - } - } - catch { - Write-Warning "Failed to parse $($trxFile.Name): $_" - } + if ([System.Convert]::ToBoolean($env:ALLOW_ZERO_TESTS)) { + $arguments.AllowZeroTests = $true } - - if ($validFileCount -eq 0) { - Write-Error "No valid .trx files found. All .trx files are empty or invalid XML." - exit 1 - } - - Write-Host "✓ Test execution completed successfully (found $validFileCount valid .trx file(s))" + ./eng/scripts/verify-nonblocking-test-results.ps1 @arguments - name: Check for hang dump files if: always() @@ -745,13 +695,8 @@ jobs: # even when all tests passed, so we don't fail on those. $hangDumpFiles = Get-ChildItem -Path $testResultsDir -Filter *hangdump* -Recurse -ErrorAction SilentlyContinue if ($hangDumpFiles.Count -gt 0) { - # When ignoreTestFailures is true (the quarantine workflow), a hang dump must NOT fail the job. - # Quarantined tests are flaky by definition and routinely time out; the whole point of that - # workflow is to surface flakiness without going red, so failing here defeats its purpose and - # leaves the quarantine run permanently broken. Downgrade to a warning instead — the dump is - # still detected, logged, and uploaded as an artifact for investigation. - # For all other callers (normal CI, outerloop) ignoreTestFailures is false, so a hang dump - # still fails the job: it indicates a real timeout that must not be ignored. + # A caller that explicitly ignores test outcomes also treats a test timeout as non-gating. + # The dump remains visible in the logs and uploaded artifacts. All other callers fail here. if ('${{ inputs.ignoreTestFailures }}' -eq 'true') { Write-Host "::warning::Hang dump file(s) detected — a test timed out (ignored because ignoreTestFailures is true):" foreach ($df in $hangDumpFiles) { Write-Host " - $($df.FullName)" } diff --git a/.github/workflows/specialized-test-runner.yml b/.github/workflows/specialized-test-runner.yml index 921dabf8e68..46afeb693f8 100644 --- a/.github/workflows/specialized-test-runner.yml +++ b/.github/workflows/specialized-test-runner.yml @@ -22,11 +22,16 @@ on: enablePlaywrightInstall: required: true type: boolean - # When true, ignores test failures and checks if .trx files have nonzero test count + # When true, known test outcomes are non-blocking while infrastructure failures remain blocking. ignoreTestFailures: required: false type: boolean default: false + # Allows valid zero-test results for trait-filtered project/OS lanes where no tests match. + allowZeroTests: + required: false + type: boolean + default: false jobs: @@ -160,7 +165,8 @@ jobs: os: ${{ matrix.tests.os }} mtpBaseArgs: ${{ matrix.tests.mtpBaseArgs }} extraTestArgs: ${{ inputs.extraTestArgs }} - ignoreTestFailures: ${{ inputs.ignoreTestFailures }} + ignoreTestFailures: ${{ inputs.ignoreTestFailures || matrix.tests.properties.nonBlockingTestFailures }} + allowZeroTests: ${{ inputs.allowZeroTests }} results: diff --git a/.github/workflows/tests-outerloop.yml b/.github/workflows/tests-outerloop.yml index e416dd6e26b..35d6f2ca678 100644 --- a/.github/workflows/tests-outerloop.yml +++ b/.github/workflows/tests-outerloop.yml @@ -2,7 +2,7 @@ # # COPILOT INSTRUCTIONS: # - Keep the shared 'paths:' entries (specialized-test-runner.yml, -# run-tests.yml, build-cli-e2e-image.yml) in sync across +# run-tests.yml, build-cli-e2e-image.yml, and the non-blocking result verifier) in sync across # tests-outerloop.yml and tests-quarantine.yml. Each workflow also lists itself. # - Validate that each path exists in the repository before adding or # updating the list @@ -29,6 +29,9 @@ on: - '.github/workflows/specialized-test-runner.yml' - '.github/workflows/run-tests.yml' - '.github/workflows/build-cli-e2e-image.yml' + - '.github/workflows/filter-nonblocking-test-artifacts.sh' + - 'eng/scripts/verify-nonblocking-test-results.ps1' + - 'eng/testing/CITestsProperties.props' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -81,6 +84,14 @@ jobs: run: | mkdir -p "$RUNNER_TEMP/out" OUT="$RUNNER_TEMP/out/failed-tests.json" + # Project-level NonBlockingTestFailures keeps known test outcomes visible in + # their job summary without making the workflow red. If another job + # fails, exclude those non-gating artifacts so they cannot be reported + # as part of the gating failure issue. + if [ -d "$GITHUB_WORKSPACE/all-logs" ]; then + bash ./.github/workflows/filter-nonblocking-test-artifacts.sh "$GITHUB_WORKSPACE/all-logs" + fi + # extractionFailed distinguishes "we could not read the results" from # "the run produced zero failed tests". The reporter classifies a # genuinely-red run with unknown results as a test failure (not infra), @@ -114,4 +125,3 @@ jobs: script: | const reporter = require('./.github/workflows/report-specialized-test-failures.js'); await require('./.github/workflows/specialized-test-failure-runner.js')({ github, context, core, reporter }); - diff --git a/.github/workflows/tests-quarantine.yml b/.github/workflows/tests-quarantine.yml index c4284dee2f4..121e3320adc 100644 --- a/.github/workflows/tests-quarantine.yml +++ b/.github/workflows/tests-quarantine.yml @@ -2,7 +2,7 @@ # # COPILOT INSTRUCTIONS: # - Keep the shared 'paths:' entries (specialized-test-runner.yml, -# run-tests.yml, build-cli-e2e-image.yml) in sync across +# run-tests.yml, build-cli-e2e-image.yml, and the non-blocking result verifier) in sync across # tests-outerloop.yml and tests-quarantine.yml. Each workflow also lists itself. # - Validate that each path exists in the repository before adding or # updating the list @@ -30,6 +30,8 @@ on: - '.github/workflows/specialized-test-runner.yml' - '.github/workflows/run-tests.yml' - '.github/workflows/build-cli-e2e-image.yml' + - 'eng/scripts/verify-nonblocking-test-results.ps1' + - 'eng/testing/CITestsProperties.props' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -45,9 +47,12 @@ jobs: extraTestArgs: "--filter-trait quarantined=true" enablePlaywrightInstall: true ignoreTestFailures: true + allowZeroTests: true - # Quarantine passes ignoreTestFailures, so failing quarantined tests do NOT red - # the run (run-tests.yml swallows them and only checks that TRX were produced). + # Quarantine passes ignoreTestFailures, so known quarantined test outcomes do + # not red the run. Some project/OS lanes have no matching quarantined tests, + # so allowZeroTests accepts their valid zero-test TRX. The runner still + # validates exit-code and result evidence. # A failed quarantine run therefore means infrastructure broke — file/append a # single deduplicated 'automation-broken' issue. No test-failure issues are # filed here. PR-triggered runs are excluded. @@ -79,4 +84,3 @@ jobs: script: | const reporter = require('./.github/workflows/report-specialized-test-failures.js'); await require('./.github/workflows/specialized-test-failure-runner.js')({ github, context, core, reporter }); - diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f0a404ff49..2c8fc4a1af7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -391,6 +391,7 @@ jobs: extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} properties: ${{ toJson(matrix.properties) }} + ignoreTestFailures: ${{ matrix.properties.nonBlockingTestFailures }} # Split requires-nugets tests by OS so each group depends only on # the CLI archive build for its platform, avoiding cross-OS blocking. @@ -410,6 +411,7 @@ jobs: extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} properties: ${{ toJson(matrix.properties) }} + ignoreTestFailures: ${{ matrix.properties.nonBlockingTestFailures }} tests_requires_nugets_windows: name: Package tests - Windows @@ -427,6 +429,7 @@ jobs: extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} properties: ${{ toJson(matrix.properties) }} + ignoreTestFailures: ${{ matrix.properties.nonBlockingTestFailures }} tests_requires_nugets_macos: name: Package tests - macOS @@ -444,6 +447,7 @@ jobs: extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} properties: ${{ toJson(matrix.properties) }} + ignoreTestFailures: ${{ matrix.properties.nonBlockingTestFailures }} tests_requires_cli_archive: name: CLI archive tests @@ -461,6 +465,7 @@ jobs: extraTestArgs: "--filter-not-trait \"quarantined=true\" --filter-not-trait \"outerloop=true\" ${{ matrix.extraTestArgs }}" versionOverrideArg: ${{ inputs.versionOverrideArg }} properties: ${{ toJson(matrix.properties) }} + ignoreTestFailures: ${{ matrix.properties.nonBlockingTestFailures }} polyglot_validation: name: Polyglot SDK Validation diff --git a/Aspire.slnx b/Aspire.slnx index 7655c568695..d65af014c08 100644 --- a/Aspire.slnx +++ b/Aspire.slnx @@ -546,6 +546,7 @@ + diff --git a/docs/ci/TestingOnCI.md b/docs/ci/TestingOnCI.md index 624a2bf3af6..2be7a51e3bd 100644 --- a/docs/ci/TestingOnCI.md +++ b/docs/ci/TestingOnCI.md @@ -64,7 +64,7 @@ This invokes `eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder. - Writes a `.tests-metadata.json` file to `artifacts/helix/` containing: - `projectName`, `shortName`, `testProjectPath` - `supportedOSes` array (e.g., `["windows", "linux", "macos"]`) - - `properties` object with boolean flags (defined in `eng/testing/CITestsProperties.props`): `requiresNugets`, `requiresTestSdk`, `requiresCliArchive`, `requiresGitHubToken`, `enablePlaywrightInstall` + - `properties` object with boolean flags (defined in `eng/testing/CITestsProperties.props`): `requiresNugets`, `requiresTestSdk`, `requiresCliArchive`, `enablePlaywrightInstall`, `nonBlockingTestFailures`, `requiresJava` - `testSessionTimeout`, `testHangTimeout` values - `uncollectedTestsSessionTimeout`, `uncollectedTestsHangTimeout` values - `splitTests` flag @@ -107,7 +107,8 @@ After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/bui "requiresNugets": true, "requiresTestSdk": true, "requiresCliArchive": false, - "enablePlaywrightInstall": false + "enablePlaywrightInstall": false, + "nonBlockingTestFailures": false }, "testSessionTimeout": "20m", "testHangTimeout": "10m", @@ -122,7 +123,8 @@ After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/bui "requiresNugets": false, "requiresTestSdk": false, "requiresCliArchive": false, - "enablePlaywrightInstall": false + "enablePlaywrightInstall": false, + "nonBlockingTestFailures": false }, "testSessionTimeout": "30m", "extraTestArgs": "--filter-trait \"Partition=Docker\"", @@ -314,6 +316,7 @@ All boolean test properties (such as `RequiresNugets`, `RequiresTestSdk`, `Requi + ``` @@ -331,6 +334,26 @@ Each `CITestsProperty` item has three attributes: 2. **PowerShell scripts** (`build-test-matrix.ps1`) parse the `.props` XML at startup to build the defaults dictionary and copy properties generically — no per-property code blocks. 3. **GitHub Actions workflows** (`run-tests.yml`) read properties from the opaque `properties` JSON string with bespoke `if:` conditions for each property that controls unique workflow behavior. +Both `tests-outerloop.yml` and `tests-quarantine.yml` include +`eng/testing/CITestsProperties.props` in their PR path filters because the +specialized runsheet builder imports this registry. A registry-only change +therefore triggers specialized PR smoke validation of the emitted metadata. +Outerloop-only inputs, such as the advisory artifact filter, do not trigger +quarantine validation. + +`NonBlockingTestFailures` is a narrow opt-in for test projects whose results +should remain visible without gating the workflow. Known test outcomes such as +failed tests, session aborts, and test-host crashes are reported in the job +summary and artifacts, while missing evidence, unknown exit codes, and runner +or configuration failures still fail the job. A project-level opt-in also fails +when zero tests execute. Specialized workflows that intentionally apply a trait +filter across project/OS combinations can explicitly allow a valid zero-test +result. + +The reusable test job reports success only after that classification succeeds, +so aggregate `Final Results` jobs do not need a separate exclusion list. +Infrastructure failures still propagate as failed dependencies. + ### Adding a new boolean test property 1. Add one line to `eng/testing/CITestsProperties.props`: diff --git a/docs/ci/specialized-test-failure-issues.md b/docs/ci/specialized-test-failure-issues.md index acebeabb4d8..daa5828463c 100644 --- a/docs/ci/specialized-test-failure-issues.md +++ b/docs/ci/specialized-test-failure-issues.md @@ -22,13 +22,21 @@ two conclusions where no job (and so no in-pipeline reporter) runs. ### Why quarantine is infra-only `tests-quarantine.yml` passes `ignoreTestFailures: true`. In `run-tests.yml` that -appends `|| true` to the test step and then only checks that `.trx` files with a -nonzero test count were produced. So **failing quarantined tests never red the -run** — they are flaky by definition. A *failed* quarantine run therefore always -means infrastructure broke, and only an `infra` issue is filed. - -`tests-outerloop.yml` does not ignore test failures, so a failed outerloop run is -classified by inspecting the produced `.trx`: +captures the test-runner exit code, makes known test outcomes non-blocking, and +then verifies that the outcome and result evidence are valid. A project/OS +combination with no matching quarantined tests is accepted because +`tests-quarantine.yml` explicitly passes `allowZeroTests: true`; the verifier +still requires a valid zero-test TRX. The specialized trait filter makes that +empty lane expected. +So **failing quarantined tests never red the run** — they are flaky by +definition. A *failed* quarantine run therefore always means infrastructure +broke, and only an `infra` issue is filed. + +`tests-outerloop.yml` generally treats test failures as gating. Individual test +projects can opt into non-gating test outcomes with +`NonBlockingTestFailures`; their results remain visible in the job summary and +artifacts but are excluded from the scheduled failure issue. A failed outerloop +run is classified from the remaining gating `.trx` results: - failed test names present → `test-failures` - clean extraction, zero failed → `infra` (the run broke before/around test @@ -48,7 +56,8 @@ PR-triggered runs from the `paths:` filter never file issues), and only for `microsoft/aspire`. For outerloop it downloads the runner's `logs-*` artifacts (which contain the -`.trx` files) and runs: +`.trx` files), removes artifacts marked as project-level non-gating results, and +runs: ```bash GenerateTestSummary --failed-tests-json diff --git a/docs/ci/test-trigger-map.md b/docs/ci/test-trigger-map.md index bc8f0534115..7d59ed66e96 100644 --- a/docs/ci/test-trigger-map.md +++ b/docs/ci/test-trigger-map.md @@ -226,6 +226,14 @@ Highlights: Dedicated package-input directories use broad `/**` rules when enumerating files or RIDs would let a newly added input silently miss its consumers. +Native Dashboard validation is a custom job, not an MSBuild test-project +selection. Its path rule explicitly covers the native browser fixture's shared +timeout input (`tests/Shared/AsyncTestHelpers.cs`) and telemetry input +(`tests/Shared/Telemetry/TelemetryTestHelpers.cs`). Layer 1 selects the managed +consumers of those files but does not select the packaged native Dashboard job. +Keep these direct runtime dependencies in the native path rule rather than +deriving native validation from every change to the browser test project. + ### Project rules (`affected_project_rules`) An affected **production/non-test** project → a target set, matched by @@ -421,13 +429,14 @@ carry it forward. Never silently regenerate it. the kill switch err toward `ALL`; otherwise the selector relies on Layer 1 for `src` coverage and the convention backstop for non-MSBuild files. - **Independent workflow targets.** `deployment-e2e`, - `Aspire.Deployment.EndToEnd.Tests`, `Aspire.EndToEnd.Tests`, and - `Aspire.Oracle.EntityFrameworkCore.Tests` are not in the regular PR matrix - today; their rules give the *would-be* trigger paths. Their own schedules, - dispatches, or narrow PR workflow triggers decide whether they run. Comments - and job summaries list them separately from work the PR selector can actually - run. The API/ATS baseline regeneration workflows are independently scheduled - or dispatched and ignored as selector targets, as described in `ignore` above. + `Aspire.Dashboard.Playwright.Tests`, `Aspire.Deployment.EndToEnd.Tests`, + `Aspire.EndToEnd.Tests`, and `Aspire.Oracle.EntityFrameworkCore.Tests` are not + in the regular PR matrix today; their rules give the *would-be* trigger paths. + Their own schedules, dispatches, or narrow PR workflow triggers decide whether + they run. Comments and job summaries list them separately from work the PR + selector can actually run. The API/ATS baseline regeneration workflows are + independently scheduled or dispatched and ignored as selector targets, as + described in `ignore` above. - **Integration dirs with no test.** `src/Aspire.Hosting.Orleans`, `Aspire.Hosting.AppHost`, and `Aspire.Hosting.Tasks` have no dedicated test project. Their MSBuild files are owned by Layer 1, and their non-MSBuild files diff --git a/eng/github-ci/test-trigger-map.yml b/eng/github-ci/test-trigger-map.yml index 3c22856c87a..079d1c31bfa 100644 --- a/eng/github-ci/test-trigger-map.yml +++ b/eng/github-ci/test-trigger-map.yml @@ -195,15 +195,18 @@ path_rules: # template fixtures, and test certificates -> err toward ALL. The link-compiled tests/Shared/**/*.cs # are handled precisely by Layer 1 (see the `ignore` section), NOT routed here. - paths: - - tests/Shared/**/*.props - - tests/Shared/**/*.targets - - tests/Shared/**/*.proj - - tests/Shared/**/*.config - - tests/Shared/**/*.json - - tests/Shared/**/*.sh - - tests/Shared/**/*.pfx - - tests/Shared/**/Dockerfile* + - tests/Shared/*.props + - tests/Shared/*.targets + - tests/Shared/*.proj + - tests/Shared/*.config + - tests/Shared/RepoTesting/**/*.props + - tests/Shared/RepoTesting/**/*.targets + - tests/Shared/Docker/**/*.config + - tests/Shared/Docker/**/*.sh + - tests/Shared/Docker/**/Dockerfile* + - tests/Shared/TemplatesTesting/**/*.targets - tests/Shared/TemplatesTesting/data/** + - tests/Shared/TestCertificates/**/*.pfx targets: [ALL] reason: shared test build inputs and fixtures are consumed across multiple test lanes @@ -240,6 +243,7 @@ path_rules: - eng/TestEnumerationRunsheetBuilder/** - tools/ExtractTestPartitions/** # compiled partition extractor used for every split test project - eng/testing/CITestsProperties.props + - eng/scripts/verify-nonblocking-test-results.ps1 - .github/workflows/tests.yml - .github/workflows/run-tests.yml targets: [ALL] @@ -356,7 +360,14 @@ path_rules: - paths: - .github/workflows/native-dashboard-validation.yml - eng/scripts/test-native-dashboard.ps1 - - tests/Aspire.Dashboard.Tests/Integration/Playwright/NativeAotDashboardTests.cs + - tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj + - tests/Aspire.Dashboard.Playwright.Tests/Aspire.Dashboard.Playwright.Tests.csproj + - tests/Aspire.Dashboard*.Tests/**/Native*Dashboard*Tests.cs + - tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/** + - tests/Aspire.Dashboard.Tests/Shared/MockDashboardClient.cs + - tests/Shared/Playwright/** + - tests/Shared/AsyncTestHelpers.cs + - tests/Shared/Telemetry/TelemetryTestHelpers.cs targets: [job:native-dashboard-validation] reason: native Dashboard archive production and packaged runtime validation - paths: @@ -365,6 +376,17 @@ path_rules: reason: validates the Native AOT Dashboard publish warning policy # --- outerloop / e2e test targets: loose paths (production projects in affected_project_rules) -------- + - paths: + - tests/Aspire.Dashboard.Playwright.Tests/** + - tests/Aspire.Dashboard.Tests/Shared/MockDashboardClient.cs + - tests/Aspire.Dashboard.Tests/Shared/TerminalTestProducer.cs + - tests/Shared/Playwright/** + targets: [test:Aspire.Dashboard.Playwright.Tests] + reason: OUTERLOOP-ONLY today (SkipTests on GH Actions unless RunOuterloopTests) + - paths: + - tests/Shared/Playwright/** + targets: [test:Aspire.Templates.Tests] + reason: shared Playwright sources and browser provisioning used by template tests - paths: - tests/Aspire.EndToEnd.Tests/** - tests/testproject/** diff --git a/eng/scripts/verify-nonblocking-test-results.ps1 b/eng/scripts/verify-nonblocking-test-results.ps1 new file mode 100644 index 00000000000..4419b3f14ea --- /dev/null +++ b/eng/scripts/verify-nonblocking-test-results.ps1 @@ -0,0 +1,122 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# +# Validates the evidence from a Microsoft.Testing.Platform run before +# run-tests.yml treats recognized test outcomes as non-blocking. +# +# Inputs: +# - TestResultsPath: directory containing TRX files. +# - ExitCodePath: file containing the test command's numeric exit code. +# - AllowZeroTests: explicitly permits successful runs with valid TRX evidence +# and zero executed tests for specialized trait-filter lanes. +# Output: writes validation details and GitHub Actions annotations to stdout. It +# does not create or modify result files. +# Exit: 0 only when the exit code and available evidence satisfy the policy; +# otherwise 1 so setup, runner, malformed-result, and unexpected empty-run +# failures remain blocking. + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string] $TestResultsPath, + + [Parameter(Mandatory = $true)] + [string] $ExitCodePath, + + [switch] $AllowZeroTests +) + +$ErrorActionPreference = 'Stop' + +function Fail([string] $Message) { + Write-Host "::error::$Message" + exit 1 +} + +# https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes +# Only known test outcomes are non-blocking: 0 (success), 2 (test failure), +# 3 (session aborted), 7 (test host crash), and 13 (--maximum-failed-tests reached). +# Every recognized outcome still requires valid TRX evidence of executed tests. +# Fail closed for every other code so project-level opt-in cannot hide runner, +# configuration, or future unclassified failures. +if (-not (Test-Path -LiteralPath $ExitCodePath)) { + Fail "No test exit code file found at $ExitCodePath. The runner outcome cannot be classified safely." +} + +$rawExitCode = (Get-Content -LiteralPath $ExitCodePath -Raw).Trim() +$mtpExitCode = 0 +if (-not [int]::TryParse($rawExitCode, [ref]$mtpExitCode)) { + Fail "Test exit code '$rawExitCode' is not a valid integer." +} + +$allowedTestOutcomeCodes = @(0, 2, 3, 7, 13) +if ($allowedTestOutcomeCodes -notcontains $mtpExitCode) { + Fail "Test runner exited with unclassified code $mtpExitCode. Failing the job even though test failures are non-blocking." +} +Write-Host "Test runner exit code: $mtpExitCode (classified as a known test outcome)" + +$trxFiles = Get-ChildItem -LiteralPath $TestResultsPath -Filter *.trx -Recurse -ErrorAction SilentlyContinue +if ($trxFiles.Count -eq 0) { + Fail "No .trx files found. Tests may not have run due to infrastructure issues." +} + +Write-Host "Found $($trxFiles.Count) .trx file(s)" + +$validFileCount = 0 +$totalTestCount = 0 +$totalExecutedTestCount = 0 +foreach ($trxFile in $trxFiles) { + Write-Host "Checking $($trxFile.Name)..." + + try { + [xml]$trxContent = Get-Content -LiteralPath $trxFile.FullName -Raw + if ($trxContent -and $trxContent.TestRun) { + $countersNode = $trxContent.TestRun.ResultSummary.Counters + if (-not $countersNode -or $null -eq $countersNode.total) { + Fail "TRX file $($trxFile.Name) does not contain a Counters total value." + } + + $testCount = 0 + if (-not [int]::TryParse([string]$countersNode.total, [ref]$testCount) -or $testCount -lt 0) { + Fail "TRX file $($trxFile.Name) has invalid test count '$($countersNode.total)'. Counters total must be a nonnegative integer." + } + + if ($null -eq $countersNode.executed) { + Fail "TRX file $($trxFile.Name) does not contain a Counters executed value." + } + + $executedTestCount = 0 + if (-not [int]::TryParse([string]$countersNode.executed, [ref]$executedTestCount) -or + $executedTestCount -lt 0 -or + $executedTestCount -gt $testCount) { + Fail "TRX file $($trxFile.Name) has invalid executed test count '$($countersNode.executed)'. Counters executed must be an integer between zero and total." + } + + $validFileCount++ + $totalTestCount += $testCount + $totalExecutedTestCount += $executedTestCount + Write-Host " Tests in file: $executedTestCount executed out of $testCount total" + } + else { + Fail "TRX file $($trxFile.Name) is empty or does not contain a TestRun element." + } + } + catch { + Fail "Failed to parse TRX file $($trxFile.Name): $_" + } +} + +if ($validFileCount -eq 0) { + Fail "No valid .trx files found. All .trx files are empty or invalid XML." +} + +if ($totalExecutedTestCount -eq 0) { + if ($AllowZeroTests -and $mtpExitCode -eq 0) { + Write-Warning "Valid .trx files were produced with zero executed tests, which this specialized workflow explicitly allows." + exit 0 + } + + Fail "Valid .trx files were produced, but they contain zero executed tests." +} + +Write-Host "Test execution completed with $totalExecutedTestCount executed test(s) out of $totalTestCount total in $validFileCount valid .trx file(s)" diff --git a/eng/testing/CITestsProperties.props b/eng/testing/CITestsProperties.props index ec927337b74..3c0cec96fab 100644 --- a/eng/testing/CITestsProperties.props +++ b/eng/testing/CITestsProperties.props @@ -18,6 +18,7 @@ + diff --git a/src/Aspire.Dashboard/Aspire.Dashboard.csproj b/src/Aspire.Dashboard/Aspire.Dashboard.csproj index 75f1e307d3a..428e401227d 100644 --- a/src/Aspire.Dashboard/Aspire.Dashboard.csproj +++ b/src/Aspire.Dashboard/Aspire.Dashboard.csproj @@ -134,6 +134,7 @@ + diff --git a/tests/Aspire.Dashboard.Playwright.Tests/Aspire.Dashboard.Playwright.Tests.csproj b/tests/Aspire.Dashboard.Playwright.Tests/Aspire.Dashboard.Playwright.Tests.csproj new file mode 100644 index 00000000000..53492dd13cb --- /dev/null +++ b/tests/Aspire.Dashboard.Playwright.Tests/Aspire.Dashboard.Playwright.Tests.csproj @@ -0,0 +1,60 @@ + + + + net11.0 + true + true + true + + + $(NoWarn);CS8002 + $(WarningsNotAsErrors);ASPDEPR004;ASPDEPR008;SYSLIB0057 + + true + true + true + + + false + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/AccessibilityTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AccessibilityTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/AccessibilityTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AccessibilityTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/AppBarTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AppBarTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/AppBarTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AppBarTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/AspireMenuButtonFocusTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AspireMenuButtonFocusTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/AspireMenuButtonFocusTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AspireMenuButtonFocusTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/AspireMenuLayoutTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AspireMenuLayoutTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/AspireMenuLayoutTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/AspireMenuLayoutTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/BrowserTokenAuthenticationTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/BrowserTokenAuthenticationTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/BrowserTokenAuthenticationTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/BrowserTokenAuthenticationTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/DashboardInteractionsTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/DashboardInteractionsTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/DashboardInteractionsTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/DashboardInteractionsTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/DesktopNavMenuTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/DesktopNavMenuTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/DesktopNavMenuTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/DesktopNavMenuTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/DashboardServerFixture.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/DashboardServerFixture.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/DashboardServerFixture.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/DashboardServerFixture.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/PlaywrightFixture.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/PlaywrightFixture.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/PlaywrightFixture.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/PlaywrightFixture.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/PlaywrightTestsBase.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/PlaywrightTestsBase.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/PlaywrightTestsBase.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/PlaywrightTestsBase.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/TestTerminalConnectionResolver.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/TestTerminalConnectionResolver.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/TestTerminalConnectionResolver.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/TestTerminalConnectionResolver.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/MobileNavMenuTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/MobileNavMenuTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/MobileNavMenuTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/MobileNavMenuTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/NativeAotDashboardTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/NativeAotDashboardTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/NativeAotDashboardTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/NativeAotDashboardTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/ResourceSelectInteractionTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/ResourceSelectInteractionTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/ResourceSelectInteractionTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/ResourceSelectInteractionTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/ResourcesTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/ResourcesTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/ResourcesTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/ResourcesTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/StructuredLogsLayoutTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/StructuredLogsLayoutTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/StructuredLogsLayoutTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/StructuredLogsLayoutTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/TerminalDockTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/TerminalDockTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/TerminalDockTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/TerminalDockTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/TerminalTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/TerminalTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/TerminalTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/TerminalTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/TracesLayoutTests.cs b/tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/TracesLayoutTests.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/TracesLayoutTests.cs rename to tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/TracesLayoutTests.cs diff --git a/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj b/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj index 644171aa65d..1b509d5433d 100644 --- a/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj +++ b/tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj @@ -8,10 +8,6 @@ $(NoWarn);CS8002 $(WarningsNotAsErrors);ASPDEPR004;ASPDEPR008;SYSLIB0057 - true - true - true - false false @@ -22,8 +18,6 @@ - - @@ -54,6 +48,4 @@ - - diff --git a/tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/MockDashboardClient.cs b/tests/Aspire.Dashboard.Tests/Shared/MockDashboardClient.cs similarity index 100% rename from tests/Aspire.Dashboard.Tests/Integration/Playwright/Infrastructure/MockDashboardClient.cs rename to tests/Aspire.Dashboard.Tests/Shared/MockDashboardClient.cs diff --git a/tests/Aspire.Dashboard.Tests/Shared/TerminalTestHost.cs b/tests/Aspire.Dashboard.Tests/Shared/TerminalTestHost.cs index 323cba6d959..99d2ca3cf87 100644 --- a/tests/Aspire.Dashboard.Tests/Shared/TerminalTestHost.cs +++ b/tests/Aspire.Dashboard.Tests/Shared/TerminalTestHost.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Concurrent; -using System.IO.Pipelines; using System.Net.WebSockets; using Aspire.Dashboard.Configuration; using Aspire.Dashboard.Terminal; @@ -13,7 +12,6 @@ using Grpc.Core; using Hex1b; using Hex1b.Automation; -using Hex1b.Reflow; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -234,119 +232,3 @@ public async Task WriteAsync(TerminalClientFrame message, CancellationToken canc } } } - -internal sealed class TerminalTestProducer : IAsyncDisposable -{ - private readonly ConcurrentBag> _connections = []; - private readonly ConcurrentBag _disconnections = []; - private readonly CancellationTokenSource _stopping = new(); - private readonly Hex1bTerminal _terminal; - private int _disposed; - - public TerminalTestProducer(int width, int height, int scrollback) - { - Workload = new Hex1bAppWorkloadAdapter(); - Presentation = new Hmp1PresentationAdapter(width, height) - .WithReflow(GhosttyReflowStrategy.Instance); - _terminal = Hex1bTerminal.CreateBuilder() - .WithWorkload(Workload) - .WithPresentation(Presentation) - .WithDimensions(width, height) - .WithScrollback(scrollback) - .Build(); - } - - public Hex1bAppWorkloadAdapter Workload { get; } - public Hmp1PresentationAdapter Presentation { get; } - public int Width => Workload.Width; - public int Height => Workload.Height; - public int ConnectionCount => _connections.Count; - - public Stream Connect() - { - ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); - var toClient = new Pipe(); - var toServer = new Pipe(); - var disconnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var server = new DuplexStream(toServer.Reader.AsStream(), toClient.Writer.AsStream(), onDispose: null); - var client = new DuplexStream(toClient.Reader.AsStream(), toServer.Writer.AsStream(), () => disconnected.TrySetResult()); - _disconnections.Add(disconnected.Task); - _connections.Add(Presentation.AddClient(server, _stopping.Token)); - return client; - } - - public async Task WaitForProducerTextAsync(string text, CancellationToken cancellationToken) - { - using var snapshot = await new Hex1bTerminalInputSequenceBuilder() - .WaitUntil(snapshot => snapshot.ContainsText(text), TimeSpan.FromSeconds(10), "Terminal producer output was not applied.") - .Build() - .ApplyAsync(_terminal, cancellationToken); - } - - public Task WaitForPeerHandshakesAsync(CancellationToken cancellationToken) => - Task.WhenAll(_connections).WaitAsync(cancellationToken); - - public Task WaitForAttachmentsReleasedAsync(CancellationToken cancellationToken) => - Task.WhenAll(_disconnections).WaitAsync(cancellationToken); - - internal Task CancelConnectionsAsync() => _stopping.CancelAsync(); - - public async ValueTask DisposeAsync() - { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } - - await _stopping.CancelAsync(); - try - { - foreach (var connection in _connections) - { - try - { - await using var handle = await connection; - } - catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException) - { - // A cancelled request can end during the producer's initial handshake. - } - } - } - finally - { - await _terminal.DisposeAsync(); - _stopping.Dispose(); - } - } - - private sealed class DuplexStream(Stream input, Stream output, Action? onDispose) : Stream - { - private int _disposed; - - public override bool CanRead => input.CanRead; - public override bool CanWrite => output.CanWrite; - public override bool CanSeek => false; - public override long Length => throw new NotSupportedException(); - public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } - public override int Read(byte[] buffer, int offset, int count) => input.Read(buffer, offset, count); - public override void Write(byte[] buffer, int offset, int count) => output.Write(buffer, offset, count); - public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => input.ReadAsync(buffer, cancellationToken); - public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => output.WriteAsync(buffer, cancellationToken); - public override void Flush() => output.Flush(); - public override Task FlushAsync(CancellationToken cancellationToken) => output.FlushAsync(cancellationToken); - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - public override void SetLength(long value) => throw new NotSupportedException(); - - protected override void Dispose(bool disposing) - { - if (disposing && Interlocked.Exchange(ref _disposed, 1) == 0) - { - input.Dispose(); - output.Dispose(); - onDispose?.Invoke(); - } - base.Dispose(disposing); - } - } -} diff --git a/tests/Aspire.Dashboard.Tests/Shared/TerminalTestProducer.cs b/tests/Aspire.Dashboard.Tests/Shared/TerminalTestProducer.cs new file mode 100644 index 00000000000..066d9c18a4e --- /dev/null +++ b/tests/Aspire.Dashboard.Tests/Shared/TerminalTestProducer.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Concurrent; +using System.IO.Pipelines; +using Hex1b; +using Hex1b.Automation; +using Hex1b.Reflow; + +namespace Aspire.Dashboard.Tests.Shared; + +internal sealed class TerminalTestProducer : IAsyncDisposable +{ + private readonly ConcurrentBag> _connections = []; + private readonly ConcurrentBag _disconnections = []; + private readonly CancellationTokenSource _stopping = new(); + private readonly Hex1bTerminal _terminal; + private int _disposed; + + public TerminalTestProducer(int width, int height, int scrollback) + { + Workload = new Hex1bAppWorkloadAdapter(); + Presentation = new Hmp1PresentationAdapter(width, height) + .WithReflow(GhosttyReflowStrategy.Instance); + _terminal = Hex1bTerminal.CreateBuilder() + .WithWorkload(Workload) + .WithPresentation(Presentation) + .WithDimensions(width, height) + .WithScrollback(scrollback) + .Build(); + } + + public Hex1bAppWorkloadAdapter Workload { get; } + public Hmp1PresentationAdapter Presentation { get; } + public int Width => Workload.Width; + public int Height => Workload.Height; + public int ConnectionCount => _connections.Count; + + public Stream Connect() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + var toClient = new Pipe(); + var toServer = new Pipe(); + var disconnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var server = new DuplexStream(toServer.Reader.AsStream(), toClient.Writer.AsStream(), onDispose: null); + var client = new DuplexStream(toClient.Reader.AsStream(), toServer.Writer.AsStream(), () => disconnected.TrySetResult()); + _disconnections.Add(disconnected.Task); + _connections.Add(Presentation.AddClient(server, _stopping.Token)); + return client; + } + + public async Task WaitForProducerTextAsync(string text, CancellationToken cancellationToken) + { + using var snapshot = await new Hex1bTerminalInputSequenceBuilder() + .WaitUntil(snapshot => snapshot.ContainsText(text), TimeSpan.FromSeconds(10), "Terminal producer output was not applied.") + .Build() + .ApplyAsync(_terminal, cancellationToken); + } + + public Task WaitForPeerHandshakesAsync(CancellationToken cancellationToken) => + Task.WhenAll(_connections).WaitAsync(cancellationToken); + + public Task WaitForAttachmentsReleasedAsync(CancellationToken cancellationToken) => + Task.WhenAll(_disconnections).WaitAsync(cancellationToken); + + internal Task CancelConnectionsAsync() => _stopping.CancelAsync(); + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + { + return; + } + + await _stopping.CancelAsync(); + try + { + foreach (var connection in _connections) + { + try + { + await using var handle = await connection; + } + catch (Exception ex) when (ex is OperationCanceledException or IOException or ObjectDisposedException) + { + // A cancelled request can end during the producer's initial handshake. + } + } + } + finally + { + await _terminal.DisposeAsync(); + _stopping.Dispose(); + } + } + + private sealed class DuplexStream(Stream input, Stream output, Action? onDispose) : Stream + { + private int _disposed; + + public override bool CanRead => input.CanRead; + public override bool CanWrite => output.CanWrite; + public override bool CanSeek => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override int Read(byte[] buffer, int offset, int count) => input.Read(buffer, offset, count); + public override void Write(byte[] buffer, int offset, int count) => output.Write(buffer, offset, count); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => input.ReadAsync(buffer, cancellationToken); + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => output.WriteAsync(buffer, cancellationToken); + public override void Flush() => output.Flush(); + public override Task FlushAsync(CancellationToken cancellationToken) => output.FlushAsync(cancellationToken); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing && Interlocked.Exchange(ref _disposed, 1) == 0) + { + input.Dispose(); + output.Dispose(); + onDispose?.Invoke(); + } + base.Dispose(disposing); + } + } +} diff --git a/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs b/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs index 03413357b63..cef48d19f29 100644 --- a/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs +++ b/tests/Infrastructure.Tests/PowerShellScripts/BuildTestMatrixTests.cs @@ -55,6 +55,28 @@ public async Task GeneratesMatrixFromSingleProject() Assert.False(entry.Properties.GetValueOrDefault("requiresNugets")); } + [Fact] + [RequiresTools(["pwsh"])] + public async Task PreservesProjectLevelNonBlockingTestFailures() + { + var artifactsDir = Path.Combine(_workspace.Path, "artifacts"); + Directory.CreateDirectory(artifactsDir); + + TestDataBuilder.CreateTestsMetadataJson( + Path.Combine(artifactsDir, "NonGating.tests-metadata.json"), + projectName: "NonGating", + testProjectPath: "tests/NonGating/NonGating.csproj", + nonBlockingTestFailures: true); + + var outputFile = Path.Combine(_workspace.Path, "matrix.json"); + + var result = await RunScript(artifactsDir, outputFile); + + result.EnsureSuccessful(); + var entry = Assert.Single(ParseCanonicalMatrix(outputFile).Tests); + Assert.True(entry.Properties["nonBlockingTestFailures"]); + } + [Fact] [RequiresTools(["pwsh"])] public async Task GeneratesMatrixFromMultipleProjects() diff --git a/tests/Infrastructure.Tests/PowerShellScripts/VerifyNonBlockingTestResultsTests.cs b/tests/Infrastructure.Tests/PowerShellScripts/VerifyNonBlockingTestResultsTests.cs new file mode 100644 index 00000000000..c02bc0fe73c --- /dev/null +++ b/tests/Infrastructure.Tests/PowerShellScripts/VerifyNonBlockingTestResultsTests.cs @@ -0,0 +1,308 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.TestUtilities; +using Xunit; + +namespace Infrastructure.Tests; + +public class VerifyNonBlockingTestResultsTests : IDisposable +{ + private readonly TemporaryWorkspace _workspace; + private readonly string _scriptPath; + private readonly ITestOutputHelper _output; + + public VerifyNonBlockingTestResultsTests(ITestOutputHelper output) + { + _output = output; + _workspace = TemporaryWorkspace.Create(output); + _scriptPath = Path.Combine(RepoRoot.Path, "eng", "scripts", "verify-nonblocking-test-results.ps1"); + } + + public void Dispose() => _workspace.Dispose(); + + [Theory] + [InlineData(0)] + [InlineData(2)] + [InlineData(3)] + [InlineData(7)] + [InlineData(13)] + [RequiresTools(["pwsh"])] + public async Task AcceptsKnownTestOutcomesWithExecutedTests(int exitCode) + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode, testCount: 1); + + var result = await RunScript(testResultsPath, exitCodePath); + + result.EnsureSuccessful(); + } + + [Theory] + [InlineData(0)] + [InlineData(2)] + [InlineData(3)] + [InlineData(7)] + [InlineData(13)] + [RequiresTools(["pwsh"])] + public async Task RejectsKnownTestOutcomesWithoutTrx(int exitCode) + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode, testCount: null); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("No .trx files found", result.Output); + } + + [Theory] + [InlineData("session.cast")] + [InlineData("test-hangdump.dmp")] + [RequiresTools(["pwsh"])] + public async Task RejectsExecutionArtifactsWithoutTrx(string artifactName) + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 2, testCount: null); + File.WriteAllText(Path.Combine(testResultsPath, artifactName), ""); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("No .trx files found", result.Output); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + [InlineData(5)] + [InlineData(9)] + [InlineData(10)] + [InlineData(11)] + [InlineData(12)] + [InlineData(127)] + [RequiresTools(["pwsh"])] + public async Task RejectsUnknownOrInfrastructureExitCodes(int exitCode) + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode, testCount: 1); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains($"unclassified code {exitCode}", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task RejectsMissingExitCodeEvidence() + { + var testResultsPath = _workspace.CreateDirectory("testresults").FullName; + var exitCodePath = Path.Combine(_workspace.Path, "missing-exit-code.txt"); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("No test exit code file found", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task RejectsZeroExecutedTests() + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: 0); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("contain zero executed tests", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task AcceptsZeroExecutedTestsWhenExplicitlyAllowed() + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: 0); + + var result = await RunScript(testResultsPath, exitCodePath, allowZeroTests: true); + + result.EnsureSuccessful(); + Assert.Contains("explicitly allows", result.Output); + } + + [Theory] + [InlineData(2)] + [InlineData(3)] + [InlineData(7)] + [InlineData(13)] + [RequiresTools(["pwsh"])] + public async Task RejectsZeroExecutedTestsForNonSuccessOutcomesWhenExplicitlyAllowed(int exitCode) + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode, testCount: 0); + + var result = await RunScript(testResultsPath, exitCodePath, allowZeroTests: true); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("contain zero executed tests", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task RejectsAllSkippedTests() + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: 90, executedTestCount: 0); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("contain zero executed tests", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task AcceptsAllSkippedTestsWhenExplicitlyAllowed() + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: 90, executedTestCount: 0); + + var result = await RunScript(testResultsPath, exitCodePath, allowZeroTests: true); + + result.EnsureSuccessful(); + Assert.Contains("explicitly allows", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task RejectsMalformedTrxWhenAnotherTrxIsValid() + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: 1); + File.WriteAllText(Path.Combine(testResultsPath, "malformed.trx"), ""); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("Failed to parse TRX file malformed.trx", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task RejectsMissingTestCountWhenZeroTestsAreAllowed() + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: null); + File.WriteAllText( + Path.Combine(testResultsPath, "incomplete.trx"), + """ + + + + """); + + var result = await RunScript(testResultsPath, exitCodePath, allowZeroTests: true); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("does not contain a Counters total value", result.Output); + } + + [Theory] + [InlineData("-1")] + [InlineData("invalid")] + [RequiresTools(["pwsh"])] + public async Task RejectsInvalidTestCount(string testCount) + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: null); + File.WriteAllText( + Path.Combine(testResultsPath, "invalid-count.trx"), + $""" + + + + + + """); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("Counters total must be a nonnegative integer", result.Output); + } + + [Fact] + [RequiresTools(["pwsh"])] + public async Task RejectsMissingExecutedTestCount() + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: null); + File.WriteAllText( + Path.Combine(testResultsPath, "missing-executed-count.trx"), + """ + + + + + + """); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("does not contain a Counters executed value", result.Output); + } + + [Theory] + [InlineData("-1")] + [InlineData("invalid")] + [InlineData("2")] + [RequiresTools(["pwsh"])] + public async Task RejectsInvalidExecutedTestCount(string executedTestCount) + { + var (testResultsPath, exitCodePath) = CreateInputs(exitCode: 0, testCount: null); + File.WriteAllText( + Path.Combine(testResultsPath, "invalid-executed-count.trx"), + $""" + + + + + + """); + + var result = await RunScript(testResultsPath, exitCodePath); + + Assert.NotEqual(0, result.ExitCode); + Assert.Contains("Counters executed must be an integer between zero and total", result.Output); + } + + private (string TestResultsPath, string ExitCodePath) CreateInputs(int exitCode, int? testCount, int? executedTestCount = null) + { + var testResultsPath = _workspace.CreateDirectory(Guid.NewGuid().ToString("N")).FullName; + var exitCodePath = Path.Combine(testResultsPath, "test-exit-code.txt"); + File.WriteAllText(exitCodePath, exitCode.ToString()); + + if (testCount is not null) + { + var executedCount = executedTestCount ?? testCount.Value; + File.WriteAllText( + Path.Combine(testResultsPath, "results.trx"), + $""" + + + + + + + """); + } + + return (testResultsPath, exitCodePath); + } + + private async Task RunScript(string testResultsPath, string exitCodePath, bool allowZeroTests = false) + { + using var command = new PowerShellCommand(_scriptPath, _output); + var arguments = new List + { + "-TestResultsPath", $"\"{testResultsPath}\"", + "-ExitCodePath", $"\"{exitCodePath}\"" + }; + if (allowZeroTests) + { + arguments.Add("-AllowZeroTests"); + } + + return await command.ExecuteAsync(arguments.ToArray()); + } +} diff --git a/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs b/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs index 1b58f4bd335..6aab40221d6 100644 --- a/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs +++ b/tests/Infrastructure.Tests/Shared/TestDataBuilder.cs @@ -30,6 +30,7 @@ public static string CreateTestsMetadataJson( bool requiresTestSdk = false, bool requiresCliArchive = false, bool enablePlaywrightInstall = false, + bool nonBlockingTestFailures = false, string? extraTestArgs = null, string[]? supportedOSes = null, Dictionary? runners = null) @@ -46,7 +47,8 @@ public static string CreateTestsMetadataJson( ["requiresNugets"] = requiresNugets, ["requiresTestSdk"] = requiresTestSdk, ["requiresCliArchive"] = requiresCliArchive, - ["enablePlaywrightInstall"] = enablePlaywrightInstall + ["enablePlaywrightInstall"] = enablePlaywrightInstall, + ["nonBlockingTestFailures"] = nonBlockingTestFailures }, ExtraTestArgs = extraTestArgs, SupportedOSes = supportedOSes ?? ["windows", "linux", "macos"], @@ -77,6 +79,7 @@ public static string CreateSplitTestsMetadataJson( bool requiresTestSdk = false, bool requiresCliArchive = false, bool enablePlaywrightInstall = false, + bool nonBlockingTestFailures = false, string[]? supportedOSes = null, Dictionary? runners = null) { @@ -93,7 +96,8 @@ public static string CreateSplitTestsMetadataJson( ["requiresNugets"] = requiresNugets, ["requiresTestSdk"] = requiresTestSdk, ["requiresCliArchive"] = requiresCliArchive, - ["enablePlaywrightInstall"] = enablePlaywrightInstall + ["enablePlaywrightInstall"] = enablePlaywrightInstall, + ["nonBlockingTestFailures"] = nonBlockingTestFailures }, SupportedOSes = supportedOSes ?? ["windows", "linux", "macos"], Runners = runners @@ -193,6 +197,7 @@ public static CanonicalMatrixEntry CreateMatrixEntry( bool requiresTestSdk = false, bool requiresCliArchive = false, bool enablePlaywrightInstall = false, + bool nonBlockingTestFailures = false, string[]? supportedOSes = null, Dictionary? runners = null) { @@ -213,7 +218,8 @@ public static CanonicalMatrixEntry CreateMatrixEntry( ["requiresNugets"] = requiresNugets, ["requiresTestSdk"] = requiresTestSdk, ["requiresCliArchive"] = requiresCliArchive, - ["enablePlaywrightInstall"] = enablePlaywrightInstall + ["enablePlaywrightInstall"] = enablePlaywrightInstall, + ["nonBlockingTestFailures"] = nonBlockingTestFailures }, SupportedOSes = supportedOSes ?? ["windows", "linux", "macos"], Runners = runners diff --git a/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsCliTests.cs b/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsCliTests.cs index d773384c5d1..5c358274a78 100644 --- a/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsCliTests.cs +++ b/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsCliTests.cs @@ -649,6 +649,43 @@ public void WritesSelectionJsonArtifactWithPerItemCauses() }); } + [Theory] + [InlineData("tests/Shared/AsyncTestHelpers.cs")] + [InlineData("tests/Shared/Telemetry/TelemetryTestHelpers.cs")] + public void NativeDashboardSharedHelpersSelectNativeValidationInEnforceMode(string path) + { + RunInTempRepo((temporaryRoot, propsPath, output) => + { + var changed = WriteChangedFiles(temporaryRoot, path); + var jsonPath = Path.Combine(temporaryRoot, "selection.json"); + var previousJson = Environment.GetEnvironmentVariable("SELECT_TESTS_JSON_FILE"); + Environment.SetEnvironmentVariable("SELECT_TESTS_JSON_FILE", jsonPath); + try + { + var options = Options(RepoRoot.Path, propsPath, changedFilesPath: changed, enforce: true) with + { + MapPath = Path.Combine(RepoRoot.Path, "eng", "github-ci", "test-trigger-map.yml") + }; + + Assert.Equal(0, Selection.Run(options)); + Assert.Equal("true", output()["run_native_dashboard_validation"]); + + using var json = JsonDocument.Parse(File.ReadAllText(jsonPath)); + var selection = json.RootElement; + Assert.False(selection.GetProperty("selectsAll").GetBoolean()); + Assert.Empty(selection.GetProperty("unattributedFiles").EnumerateArray()); + Assert.Equal( + ["job:native-dashboard-validation"], + selection.GetProperty("jobs").EnumerateArray() + .Select(job => job.GetProperty("name").GetString())); + } + finally + { + Environment.SetEnvironmentVariable("SELECT_TESTS_JSON_FILE", previousJson); + } + }); + } + // Crash traceability hardening: the diagnostics writer must be best-effort. If the step summary // path is unwritable (the exact scenario where diagnostics matter), writing the block must NOT throw // a NEW exception that masks the ORIGINAL failure. The original (FileNotFoundException from the diff --git a/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsWorkflowTests.cs b/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsWorkflowTests.cs index dd28fe84099..daeb790cfa3 100644 --- a/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsWorkflowTests.cs +++ b/tests/Infrastructure.Tests/TestTriggerMap/SelectTestsWorkflowTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.Extensions.FileSystemGlobbing; using Xunit; using YamlDotNet.RepresentationModel; @@ -15,6 +16,45 @@ namespace Infrastructure.Tests.TestTriggerMap; /// public sealed class SelectTestsWorkflowTests { + [Theory] + [InlineData("tests-outerloop.yml", "eng/testing/CITestsProperties.props", true)] + [InlineData("tests-quarantine.yml", "eng/testing/CITestsProperties.props", true)] + [InlineData("tests-outerloop.yml", ".github/workflows/specialized-test-runner.yml", true)] + [InlineData("tests-quarantine.yml", ".github/workflows/specialized-test-runner.yml", true)] + [InlineData("tests-outerloop.yml", ".github/workflows/filter-nonblocking-test-artifacts.sh", true)] + [InlineData("tests-quarantine.yml", ".github/workflows/filter-nonblocking-test-artifacts.sh", false)] + [InlineData("tests-outerloop.yml", "README.md", false)] + [InlineData("tests-quarantine.yml", "README.md", false)] + public void SpecializedWorkflowRunsForChangedInput(string workflow, string changedPath, bool expected) + { + using var reader = File.OpenText(Path.Combine(RepoRoot.Path, ".github", "workflows", workflow)); + var yaml = new YamlStream(); + yaml.Load(reader); + var root = Assert.IsType(Assert.Single(yaml.Documents).RootNode); + var triggers = Assert.IsType(root.Children[new YamlScalarNode("on")]); + var pullRequest = Assert.IsType(triggers.Children[new YamlScalarNode("pull_request")]); + var paths = Assert.IsType(pullRequest.Children[new YamlScalarNode("paths")]); + + // These workflows currently use literal paths; this checks one-file + // eligibility, not full equivalence with GitHub's glob semantics. + // https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#onpushpull_requestpull_request_targetpathspaths-ignore + var matcher = new Matcher(StringComparison.Ordinal, preserveFilterOrder: true); + foreach (var node in paths) + { + var pattern = node.ToString(); + if (pattern.StartsWith('!')) + { + matcher.AddExclude(pattern[1..]); + } + else + { + matcher.AddInclude(pattern); + } + } + + Assert.Equal(expected, matcher.Match(changedPath).HasMatches); + } + // The kill switch is a maintainer-only PR label, not a PR-body token. The action must consume it // as a plain boolean (forceAll) and must NOT re-introduce body scanning (a grep over an untrusted // PR description -- the injection surface this design deliberately removed). @@ -121,30 +161,6 @@ public void NativeArchiveDependencyPackagesUseMatrixRid() Assert.Contains("/p:TargetRids=${{ matrix.targets.rids }}", command); } - [Fact] - public void NativeDashboardInteractivityOptsIntoOuterloopTests() - { - var yaml = new YamlStream(); - using var reader = new StringReader(File.ReadAllText(NativeDashboardValidationWorkflowPath)); - yaml.Load(reader); - - var root = (YamlMappingNode)yaml.Documents[0].RootNode; - var jobs = (YamlMappingNode)root.Children[new YamlScalarNode("jobs")]; - var validationJob = (YamlMappingNode)jobs.Children[new YamlScalarNode("validate")]; - var steps = (YamlSequenceNode)validationJob.Children[new YamlScalarNode("steps")]; - var interactiveTest = Assert.Single( - steps.Cast(), - step => step.Children.TryGetValue(new YamlScalarNode("name"), out var name) && - name.ToString() == "Test Native AOT Dashboard interactivity"); - var command = interactiveTest.Children[new YamlScalarNode("run")].ToString(); - - Assert.Equal("${{ inputs.rid == 'win-x64' }}", interactiveTest.Children[new YamlScalarNode("if")].ToString()); - Assert.Contains("/p:RunOuterloopTests=true --", command); - Assert.EndsWith( - "--filter-method \"*.NativeDashboard_LoadsInteractivePageWithoutBrowserErrors\" --filter-not-trait \"quarantined=true\"", - command.Trim()); - } - // The comment_selection job posts one comment per pushed commit (createComment for a new commit, // updateComment for a re-run of the same commit) and collapses superseded comments with // minimizeComment -- it must never delete. This guard fails if deletion is introduced or the @@ -261,9 +277,6 @@ private static string VersionsPropsPath private static string BuildCliNativeArchivesWorkflowPath => Path.Combine(RepoRoot.Path, ".github", "workflows", "build-cli-native-archives.yml"); - private static string NativeDashboardValidationWorkflowPath - => Path.Combine(RepoRoot.Path, ".github", "workflows", "native-dashboard-validation.yml"); - private static string ExtractCommentSelectionJob() { var testsYml = File.ReadAllText(TestsWorkflowPath); diff --git a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs index d5bc4ab0297..5d1c02248b4 100644 --- a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs +++ b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs @@ -665,9 +665,29 @@ public void DocumentedJobTargetVocabularyMatchesReferencedMapJobs() ["test:Infrastructure.Tests", "job:native-dashboard-validation"] }, { - "tests/Aspire.Dashboard.Tests/Integration/Playwright/NativeAotDashboardTests.cs", + "tests/Aspire.Dashboard.Tests/Aspire.Dashboard.Tests.csproj", ["test:Aspire.Dashboard.Tests", "job:native-dashboard-validation"] }, + { + "tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/NativeAotDashboardTests.cs", + ["test:Aspire.Dashboard.Playwright.Tests", "job:native-dashboard-validation"] + }, + { + "tests/Aspire.Dashboard.Tests/Integration/NativeAotDashboardTests.cs", + ["test:Aspire.Dashboard.Tests", "job:native-dashboard-validation"] + }, + { + "tests/Aspire.Dashboard.Playwright.Tests/Integration/Playwright/Infrastructure/PlaywrightFixture.cs", + ["test:Aspire.Dashboard.Playwright.Tests", "job:native-dashboard-validation"] + }, + { + "tests/Aspire.Dashboard.Playwright.Tests/Aspire.Dashboard.Playwright.Tests.csproj", + ["test:Aspire.Dashboard.Playwright.Tests", "job:native-dashboard-validation"] + }, + { + "tests/Shared/Playwright/Playwright.targets", + ["test:Aspire.Dashboard.Playwright.Tests", "test:Aspire.Templates.Tests", "job:native-dashboard-validation"] + }, { "eng/scripts/get-aspire-cli-pr.ps1", [ @@ -1500,34 +1520,32 @@ public void PrefilterKeepRoutedGlobsAreLiveAndCoverSelectorRoutedDirs() } [Fact] - public void EveryTestsSharedFileIsCsAttributedOrRoutedToAllOrExcluded() + public void EveryTestsSharedFileIsCsAttributedOrExplicitlyRoutedOrExcluded() { // tests/Shared has both link-compiled *.cs (Layer 1 attributes them precisely, so the run-all // fallback treats them as owned — they need no curated rule) and non-source build/fixture infra - // (props/targets/packages/Docker/Playwright/certs -> ALL). A tests/Shared file that is NEITHER a - // *.cs NOR matched by an ALL path rule NOR a doc (excluded by the prefilter) would silently select - // no targeted work: it is not under a project dir (so directory containment can't attribute a - // loose file there) and would force the run-all fallback. Pin the invariant so a new file type - // added under tests/Shared can't quietly expand to ALL. (.md files are dropped by the prefilter's - // **.md.) - var allGlobs = s_map.PathRules - .Where(r => r.Targets.Contains("ALL", StringComparer.Ordinal)) + // (props/targets/packages/Docker/Playwright/certs -> explicit consumers or ALL). A tests/Shared + // file that is NEITHER a *.cs NOR matched by a path rule NOR a doc (excluded by the prefilter) + // would force the run-all fallback because it is not under a project directory. Pin the invariant + // so new shared inputs must declare whether they have precise consumers or truly affect every lane. + // (.md files are dropped by the prefilter's **.md.) + var routedGlobs = s_map.PathRules .SelectMany(r => r.Paths) .ToList(); - bool RoutedToAll(string file) => allGlobs.Any(g => TestTriggerMap.GlobMatches(g, file)); + bool IsRouted(string file) => routedGlobs.Any(g => TestTriggerMap.GlobMatches(g, file)); var orphaned = s_trackedFiles .Where(f => f.StartsWith("tests/Shared/", StringComparison.Ordinal)) .Where(f => !f.EndsWith(".cs", StringComparison.Ordinal)) .Where(f => !f.EndsWith(".md", StringComparison.Ordinal)) - .Where(f => !RoutedToAll(f)) + .Where(f => !IsRouted(f)) .Order(StringComparer.Ordinal) .ToList(); Assert.True(orphaned.Count == 0, - $"tests/Shared files that are neither *.cs (Layer 1-attributed), *.md (prefiltered), nor routed " + - $"to ALL (they would silently select nothing): {string.Join(", ", orphaned)}"); + $"tests/Shared files that are neither *.cs (Layer 1-attributed), *.md (prefiltered), nor " + + $"explicitly routed (they would force the run-all fallback): {string.Join(", ", orphaned)}"); } [Fact] diff --git a/tests/Infrastructure.Tests/WorkflowScripts/FilterNonBlockingTestArtifactsTests.cs b/tests/Infrastructure.Tests/WorkflowScripts/FilterNonBlockingTestArtifactsTests.cs new file mode 100644 index 00000000000..e503fa28e6a --- /dev/null +++ b/tests/Infrastructure.Tests/WorkflowScripts/FilterNonBlockingTestArtifactsTests.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.TestUtilities; +using Xunit; + +namespace Infrastructure.Tests; + +public sealed class FilterNonBlockingTestArtifactsTests(ITestOutputHelper output) +{ + private static readonly string s_scriptPath = Path.Combine( + RepoRoot.Path, + ".github", + "workflows", + "filter-nonblocking-test-artifacts.sh"); + + [Fact] + [RequiresTools(["bash"])] + public async Task RemovesMarkedArtifactAndPreservesGatingArtifact() + { + using var workspace = TemporaryWorkspace.Create(output); + var allLogsPath = Path.Combine(workspace.Path, "all-logs"); + var advisoryResultsPath = Path.Combine(allLogsPath, "logs-advisory", "testresults"); + var gatingResultsPath = Path.Combine(allLogsPath, "logs-gating", "testresults"); + Directory.CreateDirectory(advisoryResultsPath); + Directory.CreateDirectory(gatingResultsPath); + File.WriteAllText(Path.Combine(advisoryResultsPath, "ignore-test-failures.marker"), "Dashboard Playwright"); + File.WriteAllText(Path.Combine(advisoryResultsPath, "advisory.trx"), "advisory failure"); + File.WriteAllText(Path.Combine(gatingResultsPath, "gating.trx"), "gating failure"); + + var result = await ProcessRunner.RunAsync(output, "bash", [s_scriptPath, allLogsPath], RepoRoot.Path); + + Assert.Equal(0, result.ExitCode); + Assert.Equal( + ["logs-gating"], + Directory.EnumerateDirectories(allLogsPath) + .Select(Path.GetFileName) + .Order(StringComparer.Ordinal) + .ToArray()); + Assert.Equal( + ["gating.trx"], + Directory.EnumerateFiles(gatingResultsPath) + .Select(Path.GetFileName) + .Order(StringComparer.Ordinal) + .ToArray()); + Assert.Contains("Excluding non-gating test results from failure classification", result.StandardOutput); + } + + [Fact] + [RequiresTools(["bash"])] + public async Task RejectsMarkerOutsideArtifactDirectory() + { + using var workspace = TemporaryWorkspace.Create(output); + var allLogsPath = Path.Combine(workspace.Path, "all-logs"); + var advisoryResultsPath = Path.Combine(allLogsPath, "logs-advisory", "testresults"); + var unexpectedResultsPath = Path.Combine(allLogsPath, "unexpected", "testresults"); + Directory.CreateDirectory(advisoryResultsPath); + Directory.CreateDirectory(unexpectedResultsPath); + File.WriteAllText(Path.Combine(advisoryResultsPath, "ignore-test-failures.marker"), "Dashboard Playwright"); + var markerPath = Path.Combine(unexpectedResultsPath, "ignore-test-failures.marker"); + File.WriteAllText(markerPath, "unexpected"); + + var result = await ProcessRunner.RunAsync(output, "bash", [s_scriptPath, allLogsPath], RepoRoot.Path); + + Assert.Equal(1, result.ExitCode); + Assert.Equal($"Unexpected ignore marker location: {markerPath}\n", result.StandardError); + Assert.Equal( + ["logs-advisory", "unexpected"], + Directory.EnumerateDirectories(allLogsPath) + .Select(Path.GetFileName) + .Order(StringComparer.Ordinal) + .ToArray()); + } + + [Fact] + [RequiresTools(["bash"])] + public async Task RejectsMarkerOutsideTestResultsDirectory() + { + using var workspace = TemporaryWorkspace.Create(output); + var allLogsPath = Path.Combine(workspace.Path, "all-logs"); + var advisoryResultsPath = Path.Combine(allLogsPath, "logs-advisory", "testresults"); + var misplacedResultsPath = Path.Combine(allLogsPath, "logs-gating", "not-testresults"); + Directory.CreateDirectory(advisoryResultsPath); + Directory.CreateDirectory(misplacedResultsPath); + File.WriteAllText(Path.Combine(advisoryResultsPath, "ignore-test-failures.marker"), "Dashboard Playwright"); + var markerPath = Path.Combine(misplacedResultsPath, "ignore-test-failures.marker"); + File.WriteAllText(markerPath, "unexpected"); + + var result = await ProcessRunner.RunAsync(output, "bash", [s_scriptPath, allLogsPath], RepoRoot.Path); + + Assert.Equal(1, result.ExitCode); + Assert.Equal($"Unexpected ignore marker location: {markerPath}\n", result.StandardError); + Assert.Equal( + ["logs-advisory", "logs-gating"], + Directory.EnumerateDirectories(allLogsPath) + .Select(Path.GetFileName) + .Order(StringComparer.Ordinal) + .ToArray()); + } + + [Fact] + [RequiresTools(["bash"])] + public async Task RejectsMarkerInNestedArtifactDirectory() + { + using var workspace = TemporaryWorkspace.Create(output); + var allLogsPath = Path.Combine(workspace.Path, "all-logs"); + var advisoryResultsPath = Path.Combine(allLogsPath, "logs-advisory", "testresults"); + var nestedResultsPath = Path.Combine(allLogsPath, "logs-gating", "nested", "testresults"); + Directory.CreateDirectory(advisoryResultsPath); + Directory.CreateDirectory(nestedResultsPath); + File.WriteAllText(Path.Combine(advisoryResultsPath, "ignore-test-failures.marker"), "Dashboard Playwright"); + var markerPath = Path.Combine(nestedResultsPath, "ignore-test-failures.marker"); + File.WriteAllText(markerPath, "unexpected"); + + var result = await ProcessRunner.RunAsync(output, "bash", [s_scriptPath, allLogsPath], RepoRoot.Path); + + Assert.Equal(1, result.ExitCode); + Assert.Equal($"Unexpected ignore marker location: {markerPath}\n", result.StandardError); + Assert.Equal( + ["logs-advisory", "logs-gating"], + Directory.EnumerateDirectories(allLogsPath) + .Select(Path.GetFileName) + .Order(StringComparer.Ordinal) + .ToArray()); + } +} diff --git a/tools/SelectTests/Program.cs b/tools/SelectTests/Program.cs index 658b85b243e..81649d3c058 100644 --- a/tools/SelectTests/Program.cs +++ b/tools/SelectTests/Program.cs @@ -139,6 +139,7 @@ internal static class Selection new Dictionary(StringComparer.Ordinal) { ["Aspire.Deployment.EndToEnd.Tests"] = "deployment workflow-only", + ["Aspire.Dashboard.Playwright.Tests"] = "outerloop-only", ["Aspire.EndToEnd.Tests"] = "outerloop-only", ["Aspire.Oracle.EntityFrameworkCore.Tests"] = "outerloop-only", };