diff --git a/.github/workflows/deployment-tests.yml b/.github/workflows/deployment-tests.yml index 0550b3b3510..680486b7559 100644 --- a/.github/workflows/deployment-tests.yml +++ b/.github/workflows/deployment-tests.yml @@ -580,19 +580,17 @@ jobs: Azure__Location: westus3 GH_TOKEN: ${{ github.token }} run: | - # --ignore-exit-code 8: MTP returns exit code 8 ("zero tests ran") when the single - # matrix test is dynamically skipped (e.g. on a transient Azure regional-capacity error), - # which would otherwise be treated as a failure below. Treating 8 as success — the repo - # convention baked into MtpBaseArgs (eng/Testing.targets) — keeps a skipped deploy green. - ./dotnet.sh test --project tests/Aspire.Deployment.EndToEnd.Tests/Aspire.Deployment.EndToEnd.Tests.csproj \ - -c Release \ - --results-directory ${{ github.workspace }}/testresults \ - -- \ - --report-trx --report-trx-filename "${{ matrix.shortname }}.trx" \ - --filter-not-trait "quarantined=true" \ - --ignore-exit-code 8 \ - ${{ matrix.extraTestArgs }} \ - || echo "test_failed=true" >> $GITHUB_OUTPUT + # MTP returns exit code 8 ("zero tests ran") when the single matrix test is dynamically + # skipped — e.g. the opt-in Connector Namespace scenario + # (ASPIRE_DEPLOYMENT_TEST_ENABLE_CONNECTOR_NAMESPACE=false) or a transient Azure + # regional-capacity skip — so 8 must be treated as success to keep a skipped deploy green. + # The MTP `--ignore-exit-code 8` flag cannot do that here: it only remaps the exit code of + # the test module itself, while `dotnet test` computes its own exit code from the + # aggregated run and still reports 8. So the exit code is classified explicitly instead. + bash .github/workflows/run-deployment-test.sh \ + "${{ github.workspace }}/testresults" \ + "${{ matrix.shortname }}" \ + ${{ matrix.extraTestArgs }} - name: Collect Aspire CLI logs if: always() diff --git a/.github/workflows/normalize-mtp-exit-code.ps1 b/.github/workflows/normalize-mtp-exit-code.ps1 new file mode 100644 index 00000000000..14629945b12 --- /dev/null +++ b/.github/workflows/normalize-mtp-exit-code.ps1 @@ -0,0 +1,23 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +param( + [Parameter(Mandatory)] + [ValidateRange(0, 255)] + [int] $ExitCode +) + +# Starting with .NET 11, `dotnet test` decides the zero-tests result for the whole run from +# aggregated results. `--ignore-exit-code 8` still makes direct test-module execution return 0, +# but an all-empty or all-skipped `dotnet test` run returns 8 from the orchestrator. +# This temporarily works around https://github.com/dotnet/sdk/issues/56214. Remove it after Aspire +# upgrades to a .NET 11 SDK build containing https://github.com/dotnet/sdk/pull/56296 or the +# equivalent https://github.com/dotnet/sdk/pull/56219 fix. Cleanup is tracked by +# https://github.com/microsoft/aspire/issues/20565. +# https://learn.microsoft.com/dotnet/core/tools/dotnet-test-mtp#whole-run-and-per-module-minimums +if ($ExitCode -eq 8) { + Write-Host "All selected tests were skipped; treating MTP exit code 8 as success." + exit 0 +} + +exit $ExitCode diff --git a/.github/workflows/normalize-mtp-exit-code.sh b/.github/workflows/normalize-mtp-exit-code.sh new file mode 100644 index 00000000000..f1159cf6cc6 --- /dev/null +++ b/.github/workflows/normalize-mtp-exit-code.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +set -euo pipefail + +# Starting with .NET 11, `dotnet test` decides the zero-tests result for the whole run from +# aggregated results. `--ignore-exit-code 8` still makes direct test-module execution return 0, +# but an all-empty or all-skipped `dotnet test` run returns 8 from the orchestrator. +# This temporarily works around https://github.com/dotnet/sdk/issues/56214. Remove it after Aspire +# upgrades to a .NET 11 SDK build containing https://github.com/dotnet/sdk/pull/56296 or the +# equivalent https://github.com/dotnet/sdk/pull/56219 fix. Cleanup is tracked by +# https://github.com/microsoft/aspire/issues/20565. +# https://learn.microsoft.com/dotnet/core/tools/dotnet-test-mtp#whole-run-and-per-module-minimums +if [ "$#" -ne 1 ] || [[ ! "$1" =~ ^[0-9]+$ ]]; then + echo "Usage: $0 " >&2 + exit 5 +fi + +if [ "$1" -eq 8 ]; then + echo "All selected tests were skipped; treating MTP exit code 8 as success." + exit 0 +fi + +exit "$1" diff --git a/.github/workflows/run-deployment-test.sh b/.github/workflows/run-deployment-test.sh new file mode 100644 index 00000000000..f8639213ea4 --- /dev/null +++ b/.github/workflows/run-deployment-test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +set -euo pipefail + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 [extra test arguments...]" >&2 + exit 5 +fi + +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must identify the workflow step output file}" + +results_directory=$1 +test_short_name=$2 +shift 2 + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +dotnet_script=${DOTNET_SCRIPT:-./dotnet.sh} +test_exit_code=0 + +bash "$dotnet_script" test \ + --project tests/Aspire.Deployment.EndToEnd.Tests/Aspire.Deployment.EndToEnd.Tests.csproj \ + -c Release \ + --results-directory "$results_directory" \ + -- \ + --report-trx \ + --report-trx-filename "${test_short_name}.trx" \ + --filter-not-trait "quarantined=true" \ + "$@" || test_exit_code=$? + +echo "Test run exit code: $test_exit_code" + +normalized_exit_code=0 +bash "$SCRIPT_DIR/normalize-mtp-exit-code.sh" "$test_exit_code" || normalized_exit_code=$? + +if [ "$normalized_exit_code" -ne 0 ]; then + echo "test_failed=true" >> "$GITHUB_OUTPUT" +fi diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index edb755d4dbf..ca877e95334 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -549,6 +549,12 @@ jobs: --results-directory ${{ github.workspace }}/testresults \ ${{ inputs.extraTestArgs }} || TEST_EXIT_CODE=$? + # `--ignore-exit-code 8` only remaps the test module's exit code. Normalize the + # aggregated `dotnet test` result with the same helper exercised by Infrastructure.Tests. + NORMALIZED_TEST_EXIT_CODE=0 + bash .github/workflows/normalize-mtp-exit-code.sh "$TEST_EXIT_CODE" || NORMALIZED_TEST_EXIT_CODE=$? + TEST_EXIT_CODE=$NORMALIZED_TEST_EXIT_CODE + # Persist the MTP exit code so the "Verify test results exist" gate can classify the outcome # (test failure vs. infrastructure/crash) even though we exit 0 below in quarantine mode. echo "$TEST_EXIT_CODE" > "${{ github.workspace }}/test-exit-code.txt" @@ -612,6 +618,11 @@ jobs: --results-directory ${{ github.workspace }}/testresults ` ${{ inputs.extraTestArgs }} $testExitCode = $LASTEXITCODE + + # `--ignore-exit-code 8` only remaps the test module's exit code. Normalize the + # aggregated `dotnet test` result with the same helper exercised by Infrastructure.Tests. + & .github/workflows/normalize-mtp-exit-code.ps1 -ExitCode $testExitCode + $testExitCode = $LASTEXITCODE } finally { # Stop heartbeat monitor @@ -654,8 +665,9 @@ jobs: # 10 = test adapter infrastructure failure (e.g. a fixture could not be created) # 11 = the test process exited because a dependent process exited # 12 = no supported protocol version - # (Exit code 8 "zero tests ran" is already mapped to 0 by --ignore-exit-code 8 in MtpBaseArgs, so a - # genuinely empty run is caught below by the .trx/artifact checks rather than by the exit code.) + # (Exit code 8 "zero tests ran" is mapped to 0 either by --ignore-exit-code 8 for direct assembly + # execution or explicitly by the dotnet test steps above, so a genuinely empty run is caught below + # by the .trx/artifact checks rather than by the exit code.) $exitCodeFile = "${{ github.workspace }}/test-exit-code.txt" if (Test-Path $exitCodeFile) { $mtpExitCode = [int]((Get-Content -Path $exitCodeFile -Raw).Trim()) diff --git a/docs/ci/mtp-args-pipeline.md b/docs/ci/mtp-args-pipeline.md index 6a2dc33b144..3cec6e5e292 100644 --- a/docs/ci/mtp-args-pipeline.md +++ b/docs/ci/mtp-args-pipeline.md @@ -53,7 +53,7 @@ overrides declared in an individual test project are reflected in the final argu | Arg | Purpose | |-----|---------| -| `--ignore-exit-code 8` | Don't fail the test run when zero tests match filters ([MTP exit codes](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-exit-codes)) | +| `--ignore-exit-code 8` | Don't fail the test run when zero tests match filters ([MTP exit codes](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-exit-codes)). This only remaps the exit code of the test module itself: `dotnet test` computes its own exit code from the aggregated run and still reports 8, so a caller that must treat "zero tests ran" as success has to classify the exit code itself. | | `--crashdump` | Collect crash dumps on test host crash | | `--hangdump` | Enable hang detection and hang dump handling | | `--hangdump-type none` | Disable hang dump file creation; hang detection and timeout handling still occur, but no dump file is generated | @@ -141,6 +141,22 @@ dotnet test --project --no-build -- \ MTP MSBuild integration injects `TestingPlatformCommandLineArguments` automatically (includes `--filter-not-trait "category=failing"` and TRX filename). These args are complementary to the explicit `-- `. +Starting with the .NET 11 SDK, `dotnet test` decides the zero-tests result for +the whole run from its +[aggregated results](https://learn.microsoft.com/dotnet/core/tools/dotnet-test-mtp#whole-run-and-per-module-minimums). +`--ignore-exit-code 8` still makes direct test-module execution return 0, but an +all-empty or all-skipped `dotnet test` run returns 8 from the orchestrator. The +non-NuGet paths in `run-tests.yml` therefore normalize the result through +`normalize-mtp-exit-code.sh` or `normalize-mtp-exit-code.ps1`. The Deployment +E2E workflow runs its complete test command through `run-deployment-test.sh`, +which uses the same Bash normalizer before deciding whether to set its failure +output. Other nonzero exit codes remain failures. This is a temporary workaround +for [dotnet/sdk#56214](https://github.com/dotnet/sdk/issues/56214) and should be +removed after Aspire upgrades to a .NET 11 SDK build containing +[dotnet/sdk#56296](https://github.com/dotnet/sdk/pull/56296), or the equivalent +[dotnet/sdk#56219](https://github.com/dotnet/sdk/pull/56219) fix. Cleanup is +tracked by [microsoft/aspire#20565](https://github.com/microsoft/aspire/issues/20565). + ## Backward compatibility The `run-tests.yml` `mtpBaseArgs` input has a default value that includes diagnostic flags (crashdump, hangdump, exit-code handling) but does not include timeout arguments. Timeout values are baked into `mtpBaseArgs` at build time by MSBuild (via `eng/Testing.targets`) and flow through the test matrix metadata. Callers that bypass the matrix and don't pass `mtpBaseArgs` should include the timeout arguments explicitly if needed. diff --git a/eng/github-ci/test-trigger-map.yml b/eng/github-ci/test-trigger-map.yml index 96dba197214..e83045c4b42 100644 --- a/eng/github-ci/test-trigger-map.yml +++ b/eng/github-ci/test-trigger-map.yml @@ -241,6 +241,11 @@ path_rules: targets: [ALL] reason: matrix, enumeration, and runner infrastructure can affect any regular PR test lane + - paths: + - .github/workflows/normalize-mtp-exit-code.* + targets: [ALL, job:deployment-e2e] + reason: shared MTP exit normalization is consumed by regular test lanes and Deployment E2E + - paths: - .github/workflows/build-packages.yml targets: [BUILD_PACKAGE_CONSUMERS] @@ -330,6 +335,7 @@ path_rules: - tests/Aspire.Deployment.EndToEnd.Tests/** - src/Aspire.ProjectTemplates/** - .github/workflows/deployment-tests.yml + - .github/workflows/run-deployment-test.sh targets: [job:deployment-e2e] reason: SCHEDULE/DISPATCH-ONLY today; ProjectTemplates is a Layer-1 blind spot; Azure/Cli/TypeSystem/Managed production projects are in affected_project_rules - paths: diff --git a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs index d34a6dedba4..72d69d77653 100644 --- a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs +++ b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs @@ -72,6 +72,27 @@ public void WarningPolicyChangesSelectAllTests(string path) Assert.Equal(["ALL"], targets); } + [Fact] + public void MtpExitCodeNormalizerRuleSelectsAllConsumers() + { + var rule = Assert.Single( + s_map.PathRules, + rule => rule.Paths.Any(path => path.Contains("normalize-mtp-exit-code", StringComparison.Ordinal))); + + Assert.Equal(["ALL", "job:deployment-e2e"], rule.Targets); + } + + [Fact] + public void DeploymentTestRunnerSelectsDeploymentWorkflow() + { + const string path = ".github/workflows/run-deployment-test.sh"; + var targets = s_map.PathRules + .Where(rule => rule.Paths.Any(glob => TestTriggerMap.GlobMatches(glob, path))) + .SelectMany(rule => rule.Targets); + + Assert.Contains("job:deployment-e2e", targets); + } + [Fact] public void ExtensionUnitWorkflowChangesSelectUnitAndE2eJobs() { diff --git a/tests/Infrastructure.Tests/WorkflowScripts/DeploymentTestScriptTests.cs b/tests/Infrastructure.Tests/WorkflowScripts/DeploymentTestScriptTests.cs new file mode 100644 index 00000000000..cad1291ccb0 --- /dev/null +++ b/tests/Infrastructure.Tests/WorkflowScripts/DeploymentTestScriptTests.cs @@ -0,0 +1,122 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Aspire.TestUtilities; +using Xunit; + +namespace Infrastructure.Tests; + +/// +/// Guards the command and exit-code behavior of the nightly Deployment E2E test step. +/// +public sealed class DeploymentTestScriptTests(ITestOutputHelper output) : IDisposable +{ + private static readonly string s_runnerPath = Path.Combine( + RepoRoot.Path, + ".github", + "workflows", + "run-deployment-test.sh"); + + private readonly TemporaryWorkspace _workspace = TemporaryWorkspace.Create(output); + + [Theory] + [InlineData(0, false)] // all tests passed + [InlineData(8, false)] // MTP "zero tests ran": the only test was dynamically skipped + [InlineData(2, true)] // a test failed + [InlineData(7, true)] // the test host crashed + [RequiresTools(["bash"])] + public async Task RunsDeploymentTestAndMarksOnlyGenuineFailures(int testExitCode, bool expectFailure) + { + var outputFile = Path.Combine(_workspace.Path, "github-output.txt"); + var argumentsFile = Path.Combine(_workspace.Path, "dotnet-arguments.txt"); + var fakeDotnetScript = Path.Combine(_workspace.Path, "dotnet.sh"); + await File.WriteAllTextAsync( + fakeDotnetScript, + """ + #!/usr/bin/env bash + printf '%s\n' "$@" > "$FAKE_DOTNET_ARGUMENTS" + exit "$FAKE_DOTNET_EXIT_CODE" + """); + + var resultsDirectory = Path.Combine(_workspace.Path, "test results"); + var result = await RunTestStepAsync( + testExitCode, + outputFile, + argumentsFile, + fakeDotnetScript, + resultsDirectory); + + Assert.Equal(0, result.ExitCode); + Assert.Equal( + [ + "test", + "--project", + "tests/Aspire.Deployment.EndToEnd.Tests/Aspire.Deployment.EndToEnd.Tests.csproj", + "-c", + "Release", + "--results-directory", + resultsDirectory, + "--", + "--report-trx", + "--report-trx-filename", + "Deployment.EndToEnd-ExampleTests.trx", + "--filter-not-trait", + "quarantined=true", + "--filter-class", + "Aspire.ExampleTests", + ], + await File.ReadAllLinesAsync(argumentsFile)); + + var stepOutputs = File.Exists(outputFile) ? await File.ReadAllTextAsync(outputFile) : string.Empty; + if (expectFailure) + { + Assert.Equal($"test_failed=true{Environment.NewLine}", stepOutputs); + } + else + { + Assert.Equal(string.Empty, stepOutputs); + } + } + + private async Task RunTestStepAsync( + int testExitCode, + string outputFile, + string argumentsFile, + string fakeDotnetScript, + string resultsDirectory) + { + using var process = new Process(); + process.StartInfo.FileName = "bash"; + process.StartInfo.ArgumentList.Add(s_runnerPath); + process.StartInfo.ArgumentList.Add(resultsDirectory); + process.StartInfo.ArgumentList.Add("Deployment.EndToEnd-ExampleTests"); + process.StartInfo.ArgumentList.Add("--filter-class"); + process.StartInfo.ArgumentList.Add("Aspire.ExampleTests"); + process.StartInfo.WorkingDirectory = _workspace.Path; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.UseShellExecute = false; + process.StartInfo.Environment["GITHUB_OUTPUT"] = outputFile; + process.StartInfo.Environment["DOTNET_SCRIPT"] = fakeDotnetScript; + process.StartInfo.Environment["FAKE_DOTNET_ARGUMENTS"] = argumentsFile; + process.StartInfo.Environment["FAKE_DOTNET_EXIT_CODE"] = testExitCode.ToString(); + + process.Start(); + + // Read both streams concurrently to avoid deadlock when a pipe buffer fills. + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await process.WaitForExitAsync(timeout.Token); + + var result = new CommandResult(process.ExitCode, await stdoutTask + await stderrTask); + output.WriteLine(result.Output); + + return result; + } + + public void Dispose() => _workspace.Dispose(); + + private sealed record CommandResult(int ExitCode, string Output); +} diff --git a/tests/Infrastructure.Tests/WorkflowScripts/MtpExitCodeScriptTests.cs b/tests/Infrastructure.Tests/WorkflowScripts/MtpExitCodeScriptTests.cs new file mode 100644 index 00000000000..226b3697066 --- /dev/null +++ b/tests/Infrastructure.Tests/WorkflowScripts/MtpExitCodeScriptTests.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using Aspire.TestUtilities; +using Xunit; + +namespace Infrastructure.Tests; + +public sealed class MtpExitCodeScriptTests(ITestOutputHelper output) +{ + private static readonly string s_unixNormalizerPath = Path.Combine( + RepoRoot.Path, + ".github", + "workflows", + "normalize-mtp-exit-code.sh"); + private static readonly string s_windowsNormalizerPath = Path.Combine( + RepoRoot.Path, + ".github", + "workflows", + "normalize-mtp-exit-code.ps1"); + + [Theory] + [InlineData(0, 0)] + [InlineData(8, 0)] + [InlineData(2, 2)] + [InlineData(7, 7)] + [RequiresTools(["bash"])] + public async Task UnixDotnetTestExitCodeClassificationPreservesFailures(int testExitCode, int expectedExitCode) + { + var result = await RunProcessAsync("bash", [s_unixNormalizerPath, testExitCode.ToString()]); + + Assert.Equal(expectedExitCode, result.ExitCode); + } + + [Theory] + [InlineData(0, 0)] + [InlineData(8, 0)] + [InlineData(2, 2)] + [InlineData(7, 7)] + [RequiresTools(["pwsh"])] + public async Task WindowsDotnetTestExitCodeClassificationPreservesFailures(int testExitCode, int expectedExitCode) + { + var result = await RunProcessAsync( + "pwsh", + ["-NoProfile", "-File", s_windowsNormalizerPath, "-ExitCode", testExitCode.ToString()]); + + Assert.Equal(expectedExitCode, result.ExitCode); + } + + private async Task RunProcessAsync(string fileName, IReadOnlyList arguments) + { + using var process = new Process(); + process.StartInfo.FileName = fileName; + process.StartInfo.WorkingDirectory = RepoRoot.Path; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.UseShellExecute = false; + foreach (var argument in arguments) + { + process.StartInfo.ArgumentList.Add(argument); + } + + process.Start(); + + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await process.WaitForExitAsync(timeout.Token); + + var result = new CommandResult(process.ExitCode, await stdoutTask + await stderrTask); + output.WriteLine(result.Output); + + return result; + } + + private sealed record CommandResult(int ExitCode, string Output); +}