diff --git a/Actions/.Modules/ReadSettings.psm1 b/Actions/.Modules/ReadSettings.psm1 index 8109baf299..aa489dd58c 100644 --- a/Actions/.Modules/ReadSettings.psm1 +++ b/Actions/.Modules/ReadSettings.psm1 @@ -174,6 +174,8 @@ function GetDefaultSettings "doNotBuildTests" = $false "doNotPerformUpgrade" = $false "doNotRunTests" = $false + "useSeparateTestAction" = $false + "testType" = "" "doNotRunBcptTests" = $false "doNotRunPageScriptingTests" = $false "doNotPublishApps" = $false diff --git a/Actions/.Modules/settings.schema.json b/Actions/.Modules/settings.schema.json index 4a1cd415d3..7dc62f194f 100644 --- a/Actions/.Modules/settings.schema.json +++ b/Actions/.Modules/settings.schema.json @@ -344,6 +344,16 @@ "doNotRunTests": { "type": "boolean" }, + "useSeparateTestAction": { + "type": "boolean", + "default": false, + "description": "PREVIEW: When set to true, normal tests (testFolders) run in a separate RunTests action against the build container kept alive by RunPipeline. Builds with additionalCountries continue to run normal tests inside RunPipeline so every country is tested. See https://aka.ms/ALGoSettings#useSeparateTestAction" + }, + "testType": { + "type": "string", + "default": "", + "description": "Optional test type used by the separate RunTests action. The built-in AlTool runner supports UnitTest, IntegrationTest, and Uncategorized; blank runs all test types. Custom RunTestsInBcContainer overrides may interpret other values. See https://aka.ms/ALGoSettings#testType" + }, "doNotRunBcptTests": { "type": "boolean" }, diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 086d05f5ae..18687a77c7 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -185,11 +185,14 @@ function GetTestResultSummaryMD { $suiteFailureNode = [FailureNode]::new($false) $suiteFailureNode.summaryDetails = "$($suite.name), $($suite.tests) tests, $($suite.failures) failed, $($suite.skipped) skipped, $($suite.time) seconds" foreach($testcase in $suite.testcase) { - if ($testcase.ChildNodes.Count -gt 0) { + $failureNodes = @($testcase.ChildNodes | Where-Object { + $_.NodeType -eq [System.Xml.XmlNodeType]::Element -and $_.LocalName -eq 'failure' + }) + if ($failureNodes.Count -gt 0) { Write-Host " - $($testcase.name), Failure, $($testcase.time) seconds" $testCaseFailureNode = [FailureNode]::new($false) $testCaseFailureNode.summaryDetails = "$($testcase.name), Failure" - foreach($failure in $testcase.ChildNodes) { + foreach($failure in $failureNodes) { Write-Host " - Error: $($failure.message)" Write-Host " Stacktrace:" Write-Host " $($failure.InnerText.Trim().Replace("`n","`n "))" diff --git a/Actions/RunPipeline/README.md b/Actions/RunPipeline/README.md index 2a790e94df..350e43f81f 100644 --- a/Actions/RunPipeline/README.md +++ b/Actions/RunPipeline/README.md @@ -32,6 +32,8 @@ Run pipeline in AL-Go repository | Name | Description | | :-- | :-- | | containerName | Container name of a container used during build | +| containerCredential | Masked, base64-encoded JSON credential for reconnecting to a container kept alive for the RunTests action | +| runTestsInSeparateAction | True only when RunPipeline kept a single local build container alive for the RunTests action | ## OUTPUT variables diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 07c125d2cf..3c92cf91fd 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -19,6 +19,20 @@ Param( [string] $previousAppsPath = '' ) +function New-KeepAliveContainerCredential { + <# + .SYNOPSIS + Creates a credential for a build container kept alive for the RunTests action. + .DESCRIPTION + Returns a random administrator credential so the RunTests action can reconnect to the + container after RunPipeline completes. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'A container password must be generated as plain text to build a reusable credential')] + param() + $password = "Pass!$([GUID]::NewGuid().ToString())" + return (New-Object pscredential 'admin', (ConvertTo-SecureString -String $password -AsPlainText -Force)) +} + $containerBaseFolder = $null $projectPath = $null @@ -473,6 +487,30 @@ try { $runAlPipelineParams["preprocessorsymbols"] = $settings.preprocessorSymbols $runAlPipelineParams["features"] = $settings.features + # The separate action needs one local container that remains alive after RunPipeline. Multi-country + # builds keep normal tests here because Run-AlPipeline creates and tests a container per country. + $runTestsInSeparateAction = $settings.useSeparateTestAction -and -not $settings.doNotRunTests -and -not $settings.doNotPublishApps -and @($additionalCountries).Count -eq 0 + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "runTestsInSeparateAction=$runTestsInSeparateAction" + + if ($runTestsInSeparateAction) { + Write-Host "useSeparateTestAction is enabled: skipping normal test execution in RunPipeline and keeping the container alive for the RunTests action" + $runAlPipelineParams["doNotRunTests"] = $true + + # Surface a reusable credential so RunTests can reconnect to the kept-alive container. + $containerCredential = New-KeepAliveContainerCredential + $runAlPipelineParams["credential"] = $containerCredential + + $containerCredentialPassword = $containerCredential.GetNetworkCredential().Password + $containerCredentialJson = @{ "username" = $containerCredential.UserName; "password" = $containerCredentialPassword } | ConvertTo-Json -Compress + $containerCredentialBase64 = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($containerCredentialJson)) + Write-Host "::add-mask::$containerCredentialPassword" + Write-Host "::add-mask::$containerCredentialBase64" + Add-Content -Encoding UTF8 -Path $env:GITHUB_ENV -Value "containerCredential=$containerCredentialBase64" + } + elseif ($settings.useSeparateTestAction -and -not $settings.doNotRunTests) { + Write-Host "::Notice::useSeparateTestAction is enabled, but either additionalCountries is configured or no local build container is created. The separate RunTests action will be skipped." + } + Write-Host "Invoke Run-AlPipeline with buildmode $buildMode" Run-AlPipeline @runAlPipelineParams ` -accept_insiderEula ` @@ -518,6 +556,7 @@ try { -pageScriptingTestResultsFolder (Join-Path $buildArtifactFolder 'PageScriptingTestResultDetails') ` -CreateRuntimePackages:$CreateRuntimePackages ` -appVersion ($versionNumber.MajorMinorVersion) -appBuild ($versionNumber.BuildNumber) -appRevision ($versionNumber.RevisionNumber) ` + -keepContainer:$runTestsInSeparateAction ` -uninstallRemovedApps if ($containerBaseFolder) { diff --git a/Actions/RunTests/AlToolTestRunner.psm1 b/Actions/RunTests/AlToolTestRunner.psm1 new file mode 100644 index 0000000000..e1ec5d3253 --- /dev/null +++ b/Actions/RunTests/AlToolTestRunner.psm1 @@ -0,0 +1,945 @@ +<# +.SYNOPSIS + Executes tests with AlTool and produces JUnit XML compatible with AL-Go AnalyzeTests. + +.DESCRIPTION + This is the default RunTests executor when no RunTestsInBcContainer override is supplied. + BcContainerHelper provides app metadata, container configuration, company discovery, and test + enumeration. AlTool runs each app's enabled methods as one batch and writes the outcomes as JUnit. +#> + +$ErrorActionPreference = "Stop" + +$script:AlToolPackageId = "Microsoft.Dynamics.BusinessCentral.Development.Tools" + +<# +.SYNOPSIS + Invokes a native executable and returns its output streams and exit code. +.DESCRIPTION + Keeps stdout separate from native stderr and restores the caller's error preference immediately + after invocation. Windows PowerShell 5 stderr can include PowerShell formatting metadata. + Command resolution and invocation failures remain terminating. +.PARAMETER FilePath + The native executable name or path. +.PARAMETER ArgumentList + Arguments passed to the native executable. +.OUTPUTS + [pscustomobject] with StandardOutput, StandardError, combined Output, and ExitCode properties. +#> +function Invoke-AlNativeCommand { + param( + [Parameter(Mandatory = $true)][string] $FilePath, + [string[]] $ArgumentList = @() + ) + + $nativeCommand = Get-Command -Name $FilePath -CommandType Application -ErrorAction Stop + $standardErrorPath = Join-Path ([System.IO.Path]::GetTempPath()) "altool-stderr-$([Guid]::NewGuid().ToString('N')).txt" + $originalErrorActionPreference = $ErrorActionPreference + $nativeErrorPreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + $originalNativeErrorPreference = if ($nativeErrorPreference) { $nativeErrorPreference.Value } else { $null } + try { + try { + $ErrorActionPreference = "Continue" + if ($nativeErrorPreference) { + $PSNativeCommandUseErrorActionPreference = $false + } + $standardOutput = & $nativeCommand.Source @ArgumentList 2> $standardErrorPath + [int] $exitCode = $LASTEXITCODE + } + finally { + $ErrorActionPreference = $originalErrorActionPreference + if ($nativeErrorPreference) { + $PSNativeCommandUseErrorActionPreference = $originalNativeErrorPreference + } + } + + [string[]] $standardOutputLines = @($standardOutput | ForEach-Object { "$_" }) + if (Test-Path -LiteralPath $standardErrorPath) { + [string[]] $standardErrorLines = @(Get-Content -LiteralPath $standardErrorPath | ForEach-Object { "$_" }) + } + else { + [string[]] $standardErrorLines = @() + } + + return [PSCustomObject]@{ + StandardOutput = $standardOutputLines + StandardError = $standardErrorLines + Output = [string[]] (@($standardOutputLines) + @($standardErrorLines)) + ExitCode = $exitCode + } + } + finally { + Remove-Item -LiteralPath $standardErrorPath -Force -ErrorAction SilentlyContinue + } +} + +<# +.SYNOPSIS + Ensures the `al` CLI is available on PATH, installing the prerelease dotnet global tool. +.DESCRIPTION + Installs the AL developer tools when unavailable. A named mutex prevents concurrent jobs from + modifying the shared tool store at the same time. +.OUTPUTS + [string] The resolved `al` version string. +#> +function Install-AlTool { + param() + + $userProfile = [Environment]::GetFolderPath([Environment+SpecialFolder]::UserProfile) + if ([string]::IsNullOrWhiteSpace($userProfile)) { + throw "Could not resolve the current user's profile directory required for the dotnet global tools path." + } + $toolsPath = Join-Path (Join-Path $userProfile ".dotnet") "tools" + if (($env:PATH -split [System.IO.Path]::PathSeparator) -notcontains $toolsPath) { + $env:PATH = "$env:PATH$([System.IO.Path]::PathSeparator)$toolsPath" + } + + # Serialize install/update across processes with a named mutex and re-check availability after + # acquiring it (another job may have just installed it). + $mutex = New-Object System.Threading.Mutex($false, "Global\AL-Go-AlTool-Install") + $acquired = $false + try { + try { $acquired = $mutex.WaitOne([TimeSpan]::FromMinutes(10)) } catch [System.Threading.AbandonedMutexException] { $acquired = $true } + if (-not $acquired) { + throw "Timed out after 10 minutes waiting to acquire the AlTool installation mutex." + } + + $alAvailable = $null -ne (Get-Command al -ErrorAction SilentlyContinue) + + if (-not $alAvailable) { + Write-Host "Installing '$script:AlToolPackageId' (prerelease) as a dotnet global tool..." + $installResult = Invoke-AlNativeCommand -FilePath "dotnet" -ArgumentList @( + "tool", "install", $script:AlToolPackageId, "--global", "--prerelease" + ) + $installResult.Output | ForEach-Object { Write-Host $_ } + if ($installResult.ExitCode -ne 0) { + # A concurrent job may have installed it first; treat as success if `al` now resolves, + # otherwise fall back to an update. + if ($null -eq (Get-Command al -ErrorAction SilentlyContinue)) { + $updateResult = Invoke-AlNativeCommand -FilePath "dotnet" -ArgumentList @( + "tool", "update", $script:AlToolPackageId, "--global", "--prerelease" + ) + $updateResult.Output | ForEach-Object { Write-Host $_ } + if ($updateResult.ExitCode -ne 0) { + throw "Failed to install or update '$script:AlToolPackageId'. The fallback dotnet tool update exited with code $($updateResult.ExitCode). Output: $($updateResult.Output -join [Environment]::NewLine)" + } + } + } + } + } + finally { + if ($acquired) { $mutex.ReleaseMutex() } + $mutex.Dispose() + } + + if (-not (Get-Command al -ErrorAction SilentlyContinue)) { + throw "The 'al' CLI is not available after installation. Ensure '$toolsPath' is on PATH and that the runner can reach nuget.org." + } + + $versionResult = Invoke-AlNativeCommand -FilePath "al" -ArgumentList @("--version") + if ($versionResult.ExitCode -ne 0) { + throw "Failed to run 'al --version'. The command exited with code $($versionResult.ExitCode). Output: $($versionResult.Output -join [Environment]::NewLine)" + } + if ($versionResult.StandardOutput.Count -eq 0) { + throw "Failed to run 'al --version'. The command returned no output." + } + $version = $versionResult.StandardOutput[0] + Write-Host "Using al CLI version: $version" + return "$version" +} + +<# +.SYNOPSIS + Resolves the on-prem connection settings (server URL, instance, dev-service port) for a container. +.DESCRIPTION + Reads the container server configuration required by AlTool and falls back to conventional + defaults when it is unavailable. +.PARAMETER ContainerName + The name of the build container. +.OUTPUTS + [hashtable] @{ Server; ServerInstance; Port } +#> +function Get-AlToolConnection { + param( + [Parameter(Mandatory = $true)][string] $ContainerName + ) + + $server = "http://$ContainerName" + $instance = "BC" + $port = 7049 + + try { + $config = Get-BcContainerServerConfiguration -ContainerName $ContainerName + if ($config) { + if ($config.ServerInstance) { $instance = "$($config.ServerInstance)" } + if ($config.DeveloperServicesPort) { $port = [int]$config.DeveloperServicesPort } + } + } + catch { + Write-Host "WARNING: Could not read server configuration for '$ContainerName' ($($_.Exception.Message)). Falling back to $server/${instance}:$port." + } + + return @{ Server = $server; ServerInstance = $instance; Port = $port } +} + +<# +.SYNOPSIS + Resolves the company `al runtests` should target. +.DESCRIPTION + Uses an explicitly requested company or selects a container company, preferring an evaluation + company. +.PARAMETER ContainerName + The name of the build container. +.PARAMETER Tenant + The tenant to connect to. +.PARAMETER CompanyName + The company name requested by the caller (optional). +.OUTPUTS + [string] Company name, or empty string if none could be resolved. +#> +function Get-AlToolCompany { + param( + [Parameter(Mandatory = $true)][string] $ContainerName, + [Parameter(Mandatory = $true)][string] $Tenant, + [string] $CompanyName = "" + ) + + if (-not [string]::IsNullOrWhiteSpace($CompanyName)) { + return $CompanyName + } + + try { + $companies = @(Get-CompanyInBcContainer -containerName $ContainerName -tenant $Tenant) + if ($companies.Count -gt 0) { + $preferred = $companies | Where-Object { $_.evaluationCompany -eq $true } | Select-Object -First 1 + $company = if ($preferred) { $preferred.companyName } else { $companies[0].companyName } + return "$company" + } + } + catch { + Write-Host "WARNING: Could not enumerate companies for '$ContainerName' ($($_.Exception.Message))." + } + return "" +} + +<# +.SYNOPSIS + Builds case-insensitive disabled-method and disabled-codeunit lookups. +.DESCRIPTION + A `*` method disables the complete codeunit instead of a method named `*`. +.PARAMETER DisabledTests + Array of disabled-test hashtables. +.OUTPUTS + [hashtable] @{ Methods = ::">; Codeunits = "> } +#> +function Get-DisabledTestKeySet { + param( + [AllowEmptyCollection()][hashtable[]] $DisabledTests = @() + ) + + $methodSet = @{} + $codeunitSet = @{} + foreach ($entry in $DisabledTests) { + if (-not $entry) { continue } + $cuName = "$($entry['codeunitName'])".ToLowerInvariant() + $methods = @() + if ($entry.ContainsKey('method') -and $entry['method']) { $methods = @($entry['method']) } + foreach ($m in $methods) { + if ("$m" -eq '*') { + $codeunitSet[$cuName] = $true + } + else { + $methodSet["$cuName::$("$m".ToLowerInvariant())"] = $true + } + } + } + return @{ Methods = $methodSet; Codeunits = $codeunitSet } +} + +<# +.SYNOPSIS + Enumerates enabled test methods for an app in the container. +.DESCRIPTION + Removes configured disabled methods and codeunits before AlTool execution. +.PARAMETER ContainerName + Container whose tests are enumerated. +.PARAMETER Credential + Credential used to access the container. +.PARAMETER ExtensionId + App ID whose test codeunits are enumerated. +.PARAMETER Tenant + Tenant used for test enumeration. +.PARAMETER TestType + Optional platform test type. Supported values are UnitTest, IntegrationTest, and Uncategorized. + Blank enumerates all test types. +.PARAMETER DisabledTests + Hashtable entries describing test methods or codeunits excluded from the run. +.OUTPUTS + [object[]] Codeunit objects with .Id, .Name, .Tests (enabled method name array). +#> +function Get-AlToolTestCodeunits { + param( + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $ContainerName, + [Parameter(Mandatory = $true)][System.Management.Automation.PSCredential] $Credential, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $ExtensionId, + [string] $Tenant = "default", + [string] $TestType = "", + [AllowEmptyCollection()][hashtable[]] $DisabledTests = @() + ) + + $getTestsParams = @{ + containerName = $ContainerName + tenant = $Tenant + credential = $Credential + extensionId = $ExtensionId + ignoreGroups = $true + } + + if (-not [string]::IsNullOrWhiteSpace($TestType)) { + $supportedTestTypes = @("UnitTest", "IntegrationTest", "Uncategorized") + $matchingTestType = @($supportedTestTypes | Where-Object { $_ -eq $TestType }) + if ($matchingTestType.Count -eq 0) { + throw "Unsupported testType '$TestType' for the built-in AlTool runner. Supported values are UnitTest, IntegrationTest, Uncategorized, or blank." + } + $getTestsParams["testType"] = $matchingTestType[0] + } + + $codeunits = @(Get-TestsFromBcContainer @getTestsParams) + + $disabledMethods = @{} + $disabledCodeunits = @{} + if ($DisabledTests.Count -gt 0) { + $lookup = Get-DisabledTestKeySet -DisabledTests $DisabledTests + $disabledMethods = $lookup.Methods + $disabledCodeunits = $lookup.Codeunits + } + + $result = @() + $disabledCount = 0 + foreach ($cu in $codeunits) { + $cuNameLower = "$($cu.Name)".ToLowerInvariant() + $methods = @($cu.Tests | ForEach-Object { "$_" }) + + if ($disabledCodeunits.ContainsKey($cuNameLower)) { + $disabledCount += $methods.Count + continue + } + + if ($disabledMethods.Count -gt 0) { + $enabled = @($methods | Where-Object { -not $disabledMethods.ContainsKey("$cuNameLower::$("$_".ToLowerInvariant())") }) + $disabledCount += ($methods.Count - $enabled.Count) + $methods = $enabled + } + if ($methods.Count -gt 0) { + $result += [PSCustomObject]@{ Id = $cu.Id; Name = $cu.Name; Tests = $methods } + } + } + + if ($disabledCount -gt 0) { + Write-Host "Excluded $disabledCount disabled test method(s) from altool enumeration." + } + return @($result) +} + +function ConvertFrom-AlFailureOutput { + param( + [AllowEmptyString()][string] $Output + ) + + $messageLines = @() + $stackLines = @() + $inStack = $false + foreach ($line in @("$Output" -split "\r?\n")) { + if ($line -match '^\s*AL Callstack:\s*$') { + $inStack = $true + continue + } + if ($line.Trim().Length -eq 0) { continue } + if ($inStack) { + $stackLines += $line.Trim() + } + else { + $messageLines += $line.Trim() + } + } + + return @{ + Message = ($messageLines -join ' ').Trim() + Stacktrace = ($stackLines -join ';') + } +} + +<# +.SYNOPSIS + Parses an AlTool test-groups ToolResponse into result occurrences. +.DESCRIPTION + Parses every structurally valid result occurrence from the default ToolResponse JSON contract. + A CLI or input failure before ToolResponse serialization has empty OutputLines; its diagnostics + are emitted on stderr by the native command. +.PARAMETER OutputLines + The complete structured stdout from `al runtests --testgroups`. +.OUTPUTS + [hashtable] containing the parsed envelope state, result occurrences, and message. +.EXAMPLE + $outputLines = @' + { + "succeeded": true, + "message": "Test run completed.", + "data": { + "success": true, + "results": [ + { "codeunitId": 130001, "methodName": "TestOne[Case A]", "status": "passed", "output": "", "durationMs": 12 } + ] + }, + "nextSteps": [], + "warnings": [] + } + '@ + ConvertFrom-AlTestGroupsOutput -OutputLines $outputLines +.EXAMPLE + $outputLines = @' + { + "succeeded": false, + "message": "One or more tests failed.", + "data": { + "success": false, + "results": [ + { "codeunitId": 130001, "methodName": "TestOne", "status": "failed", "output": "Assertion failed.\nAL Callstack:\nTestOne line 10", "durationMs": 25 } + ] + }, + "nextSteps": [], + "errorDetails": { + "code": "TestRunFailed", + "description": "One or more tests failed.", + "possibleCauses": [], + "suggestedActions": ["Review the failed test output."], + "alternatives": [], + "missingPrerequisites": [], + "diagnosticHints": ["Inspect the AL callstack."], + "retryable": false + }, + "warnings": [] + } + '@ + ConvertFrom-AlTestGroupsOutput -OutputLines $outputLines +.EXAMPLE + $outputLines = @' + { + "succeeded": false, + "message": "The server connection is not configured.", + "nextSteps": ["Configure the Business Central server connection."], + "errorDetails": { + "code": "ConnectionConfigurationMissing", + "description": "No server connection configuration was found.", + "possibleCauses": ["The project configuration is incomplete."], + "suggestedActions": ["Add the missing server settings."], + "alternatives": [], + "missingPrerequisites": ["Business Central server connection"], + "diagnosticHints": ["Verify the project launch configuration."], + "retryable": false + }, + "warnings": ["No test run was started."] + } + '@ + ConvertFrom-AlTestGroupsOutput -OutputLines $outputLines +#> +function ConvertFrom-AlTestGroupsOutput { + param( + [Parameter(Mandatory = $true)][AllowEmptyCollection()][AllowEmptyString()][string[]] $OutputLines + ) + + $json = ($OutputLines -join [Environment]::NewLine).Trim() + try { + $toolResponse = ConvertTo-HashTable -object ($json | ConvertFrom-Json) -recurse + } + catch { + throw "The structured stdout could not be parsed as JSON: $($_.Exception.Message)" + } + + if (-not $toolResponse.ContainsKey("succeeded") -or $toolResponse.succeeded -isnot [bool]) { + throw "The structured response does not contain a valid Boolean ToolResponse succeeded value." + } + + $responseMessage = "" + if ($toolResponse.ContainsKey("message") -and $toolResponse.message -is [string]) { + $responseMessage = "$($toolResponse.message)".Trim() + } + + $entries = @() + $hasResults = $false + if ($toolResponse.ContainsKey("data") -and $null -ne $toolResponse.data) { + if ($toolResponse.data -isnot [hashtable]) { + throw "The structured response contains an invalid ToolResponse data object." + } + if ($toolResponse.data.ContainsKey("results")) { + if ($toolResponse.data.results -isnot [array]) { + throw "The structured response contains a ToolResponse data.results value that is not an array." + } + $entries = @($toolResponse.data.results) + $hasResults = $true + } + } + if ($toolResponse.succeeded -and -not $hasResults) { + throw "The successful structured response does not contain a ToolResponse data.results array." + } + + $results = @{} + foreach ($entry in $entries) { + [int] $codeunitId = $entry.codeunitId + $methodName = "$($entry.methodName)" + $status = "$($entry.status)" + $outcome = switch ($status.ToLowerInvariant()) { + "passed" { "Pass" } + "failed" { "Fail" } + "skipped" { "Skip" } + default { $null } + } + if (-not $outcome) { + throw "Result $codeunitId/$methodName has unknown status '$status'." + } + + [long] $durationMs = $entry.durationMs + $message = "" + $callstackText = "" + if ($outcome -eq "Fail") { + $failure = ConvertFrom-AlFailureOutput -Output "$($entry.output)" + $message = $failure.Message + $callstackText = $failure.Stacktrace + } + + $codeunitKey = "$codeunitId" + if (-not $results.ContainsKey($codeunitKey)) { + $results[$codeunitKey] = @() + } + $results[$codeunitKey] += @{ + MethodName = $methodName + Outcome = $outcome + Ms = $durationMs + Message = $message + Stacktrace = $callstackText + } + } + + return @{ + Results = $results + Succeeded = $toolResponse.succeeded + Message = $responseMessage + } +} + +<# +.SYNOPSIS + Creates an AlTool test-groups input file. +.DESCRIPTION + Writes the JSON file required by `al runtests --testgroups`, containing each codeunit and its + enabled test methods. +.PARAMETER Codeunits + Codeunits with Id and Tests properties to serialize. +.OUTPUTS + [string] Path to the temporary JSON file. +.EXAMPLE + $codeunits = @([pscustomobject]@{ Id = "130001"; Tests = @("TestOne", "TestTwo") }) + New-AlTestGroupsFile -Codeunits $codeunits + + # Generated JSON: + # [{ "codeunitId": 130001, "testMethods": ["TestOne", "TestTwo"] }] +#> +function New-AlTestGroupsFile { + param( + [Parameter(Mandatory = $true)][object[]] $Codeunits + ) + + $groups = @() + foreach ($codeunit in $Codeunits) { + $codeunitIdText = "$($codeunit.Id)" + [int] $codeunitId = 0 + if ([string]::IsNullOrWhiteSpace($codeunitIdText) -or + -not [int]::TryParse( + $codeunitIdText, + [System.Globalization.NumberStyles]::Integer, + [System.Globalization.CultureInfo]::InvariantCulture, + [ref] $codeunitId + )) { + throw "Test codeunit ID '$codeunitIdText' must be a valid Int32 value." + } + + $methods = @($codeunit.Tests | ForEach-Object { "$_" }) + $groups += [ordered]@{ + codeunitId = $codeunitId + testMethods = [string[]] $methods + } + } + + $testGroupsFile = Join-Path ([System.IO.Path]::GetTempPath()) "altool-testgroups-$([Guid]::NewGuid().ToString('N')).json" + ConvertTo-Json -InputObject @($groups) -Depth 5 -Compress | + Set-Content -LiteralPath $testGroupsFile -Encoding UTF8 + return $testGroupsFile +} + +<# +.SYNOPSIS + Runs all enabled test groups for one app through one AlTool connection. +.DESCRIPTION + Valid structured results are returned for JUnit generation. Invalid structured output terminates + as one protocol failure. +#> +function Invoke-AlRunTestsBatch { + param( + [Parameter(Mandatory = $true)][object[]] $Codeunits, + [Parameter(Mandatory = $true)][string] $Company, + [Parameter(Mandatory = $true)][string] $Tenant, + [Parameter(Mandatory = $true)][hashtable] $Connection + ) + + $testGroupsFile = New-AlTestGroupsFile -Codeunits $Codeunits + try { + $alArgs = @( + 'runtests', + '--testgroups', $testGroupsFile, + '--company', $Company, + '--server', $Connection.Server, + '--serverinstance', $Connection.ServerInstance, + '--port', "$($Connection.Port)", + '--environmenttype', 'OnPrem', + '--authentication', 'UserPassword', + '--tenant', $Tenant + ) + + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $nativeResult = Invoke-AlNativeCommand -FilePath "al" -ArgumentList $alArgs + $sw.Stop() + + $standardOutputText = ($nativeResult.StandardOutput -join [Environment]::NewLine).Trim() + $standardErrorText = ($nativeResult.StandardError -join [Environment]::NewLine).Trim() + if (-not [string]::IsNullOrWhiteSpace($standardErrorText)) { + OutputDebug -message "al runtests stderr:$([Environment]::NewLine)$standardErrorText" + } + + if ($nativeResult.ExitCode -notin @(0, 1)) { + $details = @("al runtests exited with unexpected code $($nativeResult.ExitCode).") + if (-not [string]::IsNullOrWhiteSpace($standardErrorText)) { + $details += "stderr: $standardErrorText" + } + if (-not [string]::IsNullOrWhiteSpace($standardOutputText)) { + $details += "stdout: $standardOutputText" + } + throw "AlTool process failure. $($details -join [Environment]::NewLine)" + } + + if ([string]::IsNullOrWhiteSpace($standardOutputText)) { + if (-not [string]::IsNullOrWhiteSpace($standardErrorText)) { + throw "al runtests failed: $standardErrorText (exit code $($nativeResult.ExitCode))." + } + throw "al runtests returned no structured stdout or stderr (exit code $($nativeResult.ExitCode))." + } + + try { + $parsed = ConvertFrom-AlTestGroupsOutput -OutputLines @($nativeResult.StandardOutput) + } + catch { + $details = @($_.Exception.Message, "stdout: $standardOutputText") + if (-not [string]::IsNullOrWhiteSpace($standardErrorText)) { + $details += "stderr: $standardErrorText" + } + throw "AlTool protocol failure. $($details -join [Environment]::NewLine)" + } + + if (-not $parsed.Succeeded -and $parsed.Results.Count -eq 0) { + $details = @() + if (-not [string]::IsNullOrWhiteSpace($parsed.Message)) { + $details += $parsed.Message + } + if (-not [string]::IsNullOrWhiteSpace($standardErrorText) -and + -not [string]::Equals($standardErrorText, $parsed.Message, [StringComparison]::OrdinalIgnoreCase)) { + $details += "stderr: $standardErrorText" + } + if ($details.Count -eq 0) { + $details += "No failure message or stderr was returned." + } + throw "al runtests failed: $($details -join [Environment]::NewLine)" + } + + return @{ + Results = $parsed.Results + Succeeded = [bool] ($parsed.Succeeded -and ($nativeResult.ExitCode -eq 0)) + ElapsedSec = [Math]::Round($sw.Elapsed.TotalSeconds, 3) + } + } + finally { + Remove-Item -LiteralPath $testGroupsFile -Force -ErrorAction SilentlyContinue + } +} + +<# +.SYNOPSIS + Appends an AL-Go AnalyzeTests-compatible JUnit for one codeunit to the given + document. +.PARAMETER Doc + The JUnit XmlDocument being built. +.PARAMETER TestSuitesNode + The root element to append to. +.PARAMETER Codeunit + The codeunit object (.Id, .Name). +.PARAMETER RequestedMethods + The method names that were requested for this codeunit. +.PARAMETER MethodResults + The parsed result occurrences for this codeunit. +.PARAMETER ExtensionId + The extension (app) id. +.PARAMETER AppName + The app name. +.PARAMETER Hostname + The runner host name. +.OUTPUTS + [int] Number of failing result occurrences in this codeunit. +#> +function Add-JUnitTestSuite { + param( + [Parameter(Mandatory = $true)][System.Xml.XmlDocument] $Doc, + [Parameter(Mandatory = $true)][System.Xml.XmlElement] $TestSuitesNode, + [Parameter(Mandatory = $true)] $Codeunit, + [Parameter(Mandatory = $true)][string[]] $RequestedMethods, + [Parameter(Mandatory = $true)][AllowEmptyCollection()][object[]] $MethodResults, + [Parameter(Mandatory = $true)][string] $ExtensionId, + [Parameter(Mandatory = $true)][string] $AppName, + [Parameter(Mandatory = $true)][string] $Hostname + ) + + $ci = [System.Globalization.CultureInfo]::InvariantCulture + $suiteName = "$($Codeunit.Id) $($Codeunit.Name)" + + $suite = $Doc.CreateElement("testsuite") + $suite.SetAttribute("name", $suiteName) + $suite.SetAttribute("timestamp", (Get-Date -Format s)) + $suite.SetAttribute("hostname", $Hostname) + + $props = $Doc.CreateElement("properties") + $suite.AppendChild($props) | Out-Null + $extProp = $Doc.CreateElement("property") + $extProp.SetAttribute("name", "extensionid") + $extProp.SetAttribute("value", $ExtensionId) + $props.AppendChild($extProp) | Out-Null + if ($AppName) { + $appProp = $Doc.CreateElement("property") + $appProp.SetAttribute("name", "appName") + $appProp.SetAttribute("value", $AppName) + $props.AppendChild($appProp) | Out-Null + } + + $failed = 0 + $skipped = 0 + $suiteMs = 0.0 + $testCount = 0 + $requestedMethodLookup = @{} + $resultsByRequestedMethod = @{} + foreach ($method in $RequestedMethods) { + $requestedMethodLookup[$method] = $true + $resultsByRequestedMethod[$method] = @() + } + + foreach ($res in $MethodResults) { + $resultName = "$($res.MethodName)" + $requestedMethod = $null + if ($requestedMethodLookup.ContainsKey($resultName)) { + $requestedMethod = $resultName + } + elseif ($resultName -match '^([^\[]+)\[(.+)\]$' -and + $requestedMethodLookup.ContainsKey($Matches[1])) { + $requestedMethod = $Matches[1] + } + if ($null -ne $requestedMethod) { + $resultsByRequestedMethod[$requestedMethod] += $res + } + } + + foreach ($method in $RequestedMethods) { + $methodResults = @($resultsByRequestedMethod[$method]) + if ($methodResults.Count -eq 0) { + $tc = $Doc.CreateElement("testcase") + $tc.SetAttribute("classname", $suiteName) + $tc.SetAttribute("name", $method) + # Missing results remain failures in the final JUnit output. + $tc.SetAttribute("time", "0") + $failure = $Doc.CreateElement("failure") + $failure.SetAttribute("message", "No result produced by al runtests") + $failure.InnerText = "" + $tc.AppendChild($failure) | Out-Null + $failed++ + $testCount++ + $suite.AppendChild($tc) | Out-Null + } + else { + foreach ($res in $methodResults) { + $tc = $Doc.CreateElement("testcase") + $tc.SetAttribute("classname", $suiteName) + $tc.SetAttribute("name", "$($res.MethodName)") + $suiteMs += [double] $res.Ms + $tc.SetAttribute("time", ([Math]::Round($res.Ms / 1000.0, 3)).ToString($ci)) + switch ($res.Outcome) { + 'Fail' { + $failure = $Doc.CreateElement("failure") + $failure.SetAttribute("message", "$($res.Message)") + $failure.InnerText = "$($res.Stacktrace)".Replace(";", "`n") + $tc.AppendChild($failure) | Out-Null + $failed++ + } + 'Skip' { + $sk = $Doc.CreateElement("skipped") + $tc.AppendChild($sk) | Out-Null + $skipped++ + } + } + $testCount++ + $suite.AppendChild($tc) | Out-Null + } + } + } + + $suite.SetAttribute("tests", "$testCount") + $suite.SetAttribute("errors", "0") + $suite.SetAttribute("failures", "$failed") + $suite.SetAttribute("skipped", "$skipped") + $suite.SetAttribute("time", ([Math]::Round($suiteMs / 1000.0, 3)).ToString($ci)) + + $TestSuitesNode.AppendChild($suite) | Out-Null + return $failed +} + +<# +.SYNOPSIS + Runs all of a single app's test codeunits through `al runtests` and writes a JUnit results file. +.DESCRIPTION + Runs the app in one test-groups batch and appends JUnit output compatible with AL-Go AnalyzeTests. +.PARAMETER ContainerName + Container hosting the app under test. +.PARAMETER Credential + Credential used to run tests. +.PARAMETER ExtensionId + App ID whose tests are executed. +.PARAMETER AppName + App name included in logs and JUnit output. +.PARAMETER CompanyName + Company used for the test run. The container default is used when omitted. +.PARAMETER Tenant + Tenant used for container discovery and AlTool execution. +.PARAMETER TestType + Optional platform test type used by BcContainerHelper for server-side test enumeration. Supported + values are UnitTest, IntegrationTest, and Uncategorized. Blank enumerates all test types. +.PARAMETER DisabledTests + Hashtable entries describing test methods or codeunits excluded from the run. +.PARAMETER JUnitResultFileName + Required JUnit file to create or append. +.OUTPUTS + [bool] $true if all executed methods passed; $false otherwise. +#> +function Invoke-AlToolTestRun { + param( + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $ContainerName, + [Parameter(Mandatory = $true)][System.Management.Automation.PSCredential] $Credential, + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $ExtensionId, + [string] $AppName = "", + [string] $CompanyName = "", + [string] $Tenant = "default", + [string] $TestType = "", + [AllowEmptyCollection()][hashtable[]] $DisabledTests = @(), + [Parameter(Mandatory = $true)][ValidateNotNullOrEmpty()][string] $JUnitResultFileName + ) + + if ([string]::IsNullOrWhiteSpace($ExtensionId)) { + throw "Invoke-AlToolTestRun requires a nonblank ExtensionId." + } + if ([string]::IsNullOrWhiteSpace($JUnitResultFileName)) { + throw "Invoke-AlToolTestRun requires a nonblank JUnitResultFileName." + } + if ([string]::IsNullOrWhiteSpace($Tenant)) { + $Tenant = "default" + } + + try { + $env:BC_SERVER_USERNAME = $Credential.UserName + $env:BC_SERVER_PASSWORD = $Credential.GetNetworkCredential().Password + + $codeunits = @(Get-AlToolTestCodeunits -ContainerName $ContainerName -Credential $Credential ` + -ExtensionId $ExtensionId -Tenant $Tenant -TestType $TestType -DisabledTests $DisabledTests) + Write-Host "Enumerated $($codeunits.Count) test codeunit(s) for app '$AppName'." + if ($codeunits.Count -eq 0) { + Write-Host "No test codeunits to run for app '$AppName'; nothing to do." + return $true + } + + if (-not (Get-Command al -ErrorAction SilentlyContinue)) { + Install-AlTool | Out-Null + } + + $connection = Get-AlToolConnection -ContainerName $ContainerName + $company = Get-AlToolCompany -ContainerName $ContainerName -Tenant $Tenant -CompanyName $CompanyName + if ([string]::IsNullOrWhiteSpace($company)) { + throw "Could not resolve a company to run tests against in container '$ContainerName'." + } + + Write-Host "altool run: app='$AppName' extensionId=$ExtensionId company='$company' server='$($connection.Server)' instance='$($connection.ServerInstance)' port=$($connection.Port) tenant='$Tenant'" + + $hostname = [System.Net.Dns]::GetHostName() + + # Multiple test apps append to the same result file. + $doc = New-Object System.Xml.XmlDocument + if (Test-Path -LiteralPath $JUnitResultFileName) { + try { + $doc.Load($JUnitResultFileName) + } + catch { + $message = "Could not load existing JUnit file '$JUnitResultFileName': $($_.Exception.Message)" + throw [System.IO.InvalidDataException]::new($message, $_.Exception) + } + + $suites = $doc.DocumentElement + if (-not $suites -or $suites.LocalName -ne 'testsuites') { + $rootName = if ($suites) { "'$($suites.LocalName)'" } else { "no root element" } + throw "Existing JUnit file '$JUnitResultFileName' has $rootName; expected a 'testsuites' root element." + } + } + else { + $doc.AppendChild($doc.CreateXmlDeclaration("1.0", "UTF-8", $null)) | Out-Null + $suites = $doc.CreateElement("testsuites") + $doc.AppendChild($suites) | Out-Null + } + + $batch = Invoke-AlRunTestsBatch -Codeunits $codeunits -Company $company ` + -Tenant $Tenant -Connection $connection + $batchResults = $batch.Results + $allPassed = [bool] $batch.Succeeded + + $idx = 0 + foreach ($cu in $codeunits) { + $idx++ + $methods = @($cu.Tests | ForEach-Object { "$_" }) + $cuResults = $batchResults["$($cu.Id)"] + if ($null -eq $cuResults) { $cuResults = @() } + + $failed = Add-JUnitTestSuite -Doc $doc -TestSuitesNode $suites -Codeunit $cu ` + -RequestedMethods $methods -MethodResults $cuResults -ExtensionId $ExtensionId ` + -AppName $AppName -Hostname $hostname + + if ($failed -gt 0) { $allPassed = $false } + + Write-Host ("[{0}/{1}] cu {2} '{3}' -> {4} failed result(s) from {5} requested method(s)" -f ` + $idx, $codeunits.Count, $cu.Id, $cu.Name, $failed, $methods.Count) + } + Write-Host ("Run for app '{0}': {1} codeunit(s) in {2}s real al wall-clock." -f ` + $AppName, $codeunits.Count, [Math]::Round([double] $batch.ElapsedSec, 2)) + + $dir = [System.IO.Path]::GetDirectoryName($JUnitResultFileName) + if ($dir -and -not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } + $doc.Save($JUnitResultFileName) + Write-Host "Wrote JUnit results for app '$AppName' to $JUnitResultFileName" + + return $allPassed + } + finally { + # Do not retain container credentials after the run. + Remove-Item Env:\BC_SERVER_USERNAME -ErrorAction SilentlyContinue + Remove-Item Env:\BC_SERVER_PASSWORD -ErrorAction SilentlyContinue + } +} + +Export-ModuleMember -Function Install-AlTool, Invoke-AlToolTestRun diff --git a/Actions/RunTests/README.md b/Actions/RunTests/README.md new file mode 100644 index 0000000000..832eb4a430 --- /dev/null +++ b/Actions/RunTests/README.md @@ -0,0 +1,42 @@ +# Run tests + +Run normal tests (`testFolders`) against the build container kept alive by the RunPipeline action. + +Enable this action with the `useSeparateTestAction` setting. It is used when normal tests are enabled and RunPipeline creates one local build container. Builds with `additionalCountries` or without a local build container continue to run normal tests in RunPipeline. BCPT and page scripting tests always remain in RunPipeline. + +The action keeps `TestResults.xml` in the project folder for AnalyzeTests and copies a produced result to `.buildartifacts/TestResults.xml` for artifact upload. It does not create a result artifact when no result is produced. After the run, it refreshes `ContainerEventLog.evtx` in the project folder so failure diagnostics include test-time events. + +Compiled apps are selected by matching their app IDs to `testFolders`, which excludes BCPT-only apps from the shared test-app artifact. When `runTestsInAllInstalledTestApps` is enabled, apps from `installTestAppsJson` are also included. The action honors `disabledTests.json` files found recursively under the matching test folder and project-wide `.disabledTests.json` files for both the default runner and overrides. + +## Test runner + +By default, AlTool runs enabled normal tests for each app through one batch and connection while BcContainerHelper provides app metadata, container configuration, company discovery, and server-side test enumeration. The optional `testType` setting limits enumeration to `UnitTest`, `IntegrationTest`, or `Uncategorized`; blank runs all test types. The action writes JUnit output compatible with AL-Go AnalyzeTests and downstream processing. + +To use another test runner, add a `RunTestsInBcContainer` override script under the project's `.AL-Go` folder. The override replaces AlTool execution and receives the standard BcContainerHelper test parameters once per test app, including a configured `testType`. Custom overrides may interpret additional values such as `Legacy`. + +### Known limitations + +The built-in AlTool execution path does not run `Legacy` test-type codeunits or tests that require UI or client-callback interaction. Repositories that rely on those should execute their tests through BcContainerHelper by supplying a `RunTestsInBcContainer` override script as described above. + +## INPUT + +### ENV variables + +| Name | Description | +| :-- | :-- | +| Settings | env.Settings must be set by a prior call to the ReadSettings Action | +| containerName | env.containerName is set by the RunPipeline action and identifies the container to run tests against (the container name is otherwise derived from the project) | +| containerCredential | env.containerCredential is set by the RunPipeline action and contains the masked, base64-encoded JSON credential used to reconnect to the kept-alive container | + +### Parameters + +| Name | Required | Description | Default value | +| :-- | :-: | :-- | :-- | +| shell | | The shell (powershell or pwsh) in which the PowerShell script should run | powershell | +| token | | The GitHub token running the action and exposed to test override scripts | github.token | +| project | | Project folder | '.' | +| installTestAppsJson | | Path to a JSON file containing a list of test apps to run tests in | '' | + +## OUTPUT + +None diff --git a/Actions/RunTests/RunTests.ps1 b/Actions/RunTests/RunTests.ps1 new file mode 100644 index 0000000000..76681a0be5 --- /dev/null +++ b/Actions/RunTests/RunTests.ps1 @@ -0,0 +1,101 @@ +Param( + [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [string] $token, + [Parameter(HelpMessage = "Project folder", Mandatory = $false)] + [string] $project = "", + [Parameter(HelpMessage = "A path to a JSON-formatted list of test apps to run tests in", Mandatory = $false)] + [string] $installTestAppsJson = '' +) + +<# +.SYNOPSIS + Runs the normal tests (testFolders) for an AL-Go project against the build container + created and kept alive by the RunPipeline action. +.DESCRIPTION + Runs normal tests in eligible separate-test builds. Test results remain in the project folder + for AnalyzeTests and are copied to .buildartifacts when produced. The action also refreshes the + project container event log for failure diagnostics. BCPT and page scripting tests remain in + RunPipeline. +.PARAMETER token + The GitHub token running the action. It is exposed to test override scripts as _token. +.PARAMETER project + Project folder. +.PARAMETER installTestAppsJson + A path to a JSON-formatted list of test apps (produced by previous jobs) to run tests in. +.EXAMPLE + RunTests.ps1 -project 'MyProject' +#> + +# Make the token available to RunTestsInBcContainer overrides. +$ENV:_token = $token + +. (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve) +Import-Module (Join-Path $PSScriptRoot '..\TelemetryHelper.psm1' -Resolve) +Import-Module (Join-Path $PSScriptRoot 'RunTests.psm1' -Resolve) -DisableNameChecking -Force + +function Get-TestRunnerCredential { + <# + .SYNOPSIS + Returns the credential used by the test runner to connect to the build container. + .DESCRIPTION + Uses the credential supplied by RunPipeline and fails when that pipeline output is unavailable. + #> + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'The container credential is surfaced by RunPipeline as plain text')] + param() + + if ([string]::IsNullOrWhiteSpace($ENV:containerCredential)) { + throw "RunPipeline-to-RunTests wiring error: the kept container credential was not provided." + } + + try { + $credentialJson = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ENV:containerCredential)) | + ConvertFrom-Json | + ConvertTo-HashTable -recurse + if ([string]::IsNullOrWhiteSpace("$($credentialJson.username)") -or [string]::IsNullOrWhiteSpace("$($credentialJson.password)")) { + throw "The credential does not contain a username and password." + } + $securePassword = ConvertTo-SecureString -String $credentialJson.password -AsPlainText -Force + return New-Object System.Management.Automation.PSCredential($credentialJson.username, $securePassword) + } + catch { + throw "RunPipeline-to-RunTests wiring error: the kept container credential was not provided in the expected format." + } +} + +function Get-TestRunnerContainerName { + <# + .SYNOPSIS + Returns the kept container name supplied by RunPipeline. + #> + param() + + if ([string]::IsNullOrWhiteSpace($ENV:containerName)) { + throw "RunPipeline-to-RunTests wiring error: the kept container name was not provided." + } + + return $ENV:containerName +} + +if ($project -eq ".") { $project = "" } + +$baseFolder = $ENV:GITHUB_WORKSPACE +$projectPath = Join-Path $baseFolder $project +$containerName = Get-TestRunnerContainerName +$credential = Get-TestRunnerCredential + +DownloadAndImportBcContainerHelper +Write-Host "Use settings" +$settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable -recurse + +$settings = AnalyzeRepo -settings $settings -baseFolder $baseFolder -project $project -doNotCheckArtifactSetting + +# A RunTestsInBcContainer override script, if present, replaces the built-in AlTool test runner. +$overrideParams = Get-ScriptOverrides -ALGoFolderName (Join-Path $projectPath ".AL-Go") -OverrideScriptNames @("RunTestsInBcContainer") + +Invoke-AlGoTestRun ` + -settings $settings ` + -projectPath $projectPath ` + -containerName $containerName ` + -credential $credential ` + -installTestAppsJson $installTestAppsJson ` + -runTestsOverride $overrideParams['RunTestsInBcContainer'] diff --git a/Actions/RunTests/RunTests.psm1 b/Actions/RunTests/RunTests.psm1 new file mode 100644 index 0000000000..469630b3a8 --- /dev/null +++ b/Actions/RunTests/RunTests.psm1 @@ -0,0 +1,380 @@ +<# +.SYNOPSIS + Helper module for the RunTests action. +.DESCRIPTION + Selects and runs normal test apps against the kept-alive build container. AlTool is the default + executor; a RunTestsInBcContainer override can replace it. +#> + +Import-Module (Join-Path $PSScriptRoot 'AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + +function Get-TestAppsToRun { + <# + .SYNOPSIS + Determines the selected test apps and their validated metadata. + .DESCRIPTION + Selects compiled apps matching normal testFolders and excludes BCPT-only apps. When + runTestsInAllInstalledTestApps is enabled, installed test apps are included independently. + Parentheses around installed app paths are removed to match RunPipeline behavior. Returns + each selected app as a record containing Path, Id, and Name. + .PARAMETER settings + The (analyzed) AL-Go settings hashtable. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER installTestAppsJson + Path to a JSON file with the list of installed test apps. + #> + Param( + [hashtable] $settings, + [string] $projectPath, + [string] $installTestAppsJson = '' + ) + + $testAppOutputFolder = Join-Path (Join-Path $projectPath ".buildartifacts") "TestApps" + + $testApps = @() + $selectedAppPaths = @{} + $normalTestAppIds = @{} + foreach ($testFolder in @($settings.testFolders)) { + $appJsonPath = Join-Path (Join-Path $projectPath $testFolder) "app.json" + try { + $sourceAppJson = Get-Content -Path $appJsonPath -Raw -Encoding UTF8 -ErrorAction Stop | + ConvertFrom-Json | + ConvertTo-HashTable -recurse + $sourceAppId = "$($sourceAppJson.id)" + if ([string]::IsNullOrWhiteSpace($sourceAppId)) { + throw "The app.json file does not contain an app ID." + } + } + catch { + throw "Failed to read normal test app metadata from '$appJsonPath'. Error: $($_.Exception.Message)" + } + $normalTestAppIds[$sourceAppId] = $true + } + + if (Test-Path $testAppOutputFolder) { + $selectedCompiledAppIds = @{} + foreach ($compiledApp in @(Get-ChildItem -Path $testAppOutputFolder -Filter "*.app" -File -ErrorAction Stop | Sort-Object FullName)) { + try { + $compiledAppJson = Get-AppJsonFromAppFile -appFile $compiledApp.FullName + $compiledAppId = "$($compiledAppJson.id)" + } + catch { + throw "Failed to read compiled test app metadata from '$($compiledApp.FullName)'. Error: $($_.Exception.Message)" + } + + if ($normalTestAppIds.ContainsKey($compiledAppId) -and -not $selectedCompiledAppIds.ContainsKey($compiledAppId)) { + $compiledAppName = "$($compiledAppJson.name)" + if ([string]::IsNullOrWhiteSpace($compiledAppName)) { + throw "Failed to read compiled test app metadata from '$($compiledApp.FullName)'. Error: The compiled app metadata does not contain an app name." + } + $testApps += [PSCustomObject]@{ + Path = $compiledApp.FullName + Id = $compiledAppId + Name = $compiledAppName + } + $selectedCompiledAppIds[$compiledAppId] = $true + $selectedAppPaths[$compiledApp.FullName] = $true + } + } + } + + if ($settings.runTestsInAllInstalledTestApps -and $installTestAppsJson) { + try { + $installedTestApps = Get-Content -Path $installTestAppsJson -Raw -Encoding UTF8 -ErrorAction Stop | ConvertFrom-Json + } + catch { + throw "Failed to parse JSON file at path '$installTestAppsJson'. Error: $($_.Exception.Message)" + } + foreach ($installedTestApp in @($installedTestApps)) { + $installedTestAppPath = "$installedTestApp".TrimStart("(").TrimEnd(")") + if ([string]::IsNullOrWhiteSpace($installedTestAppPath)) { + throw "The installed test app list '$installTestAppsJson' contains a blank path." + } + try { + $installedAppJson = Get-AppJsonFromAppFile -appFile $installedTestAppPath + $installedAppId = "$($installedAppJson.id)" + $installedAppName = "$($installedAppJson.name)" + if ([string]::IsNullOrWhiteSpace($installedAppId) -or [string]::IsNullOrWhiteSpace($installedAppName)) { + throw "The installed app metadata does not contain an app ID and name." + } + } + catch { + throw "Failed to read installed test app metadata from '$installedTestAppPath'. Error: $($_.Exception.Message)" + } + if (-not $selectedAppPaths.ContainsKey($installedTestAppPath)) { + $testApps += [PSCustomObject]@{ + Path = $installedTestAppPath + Id = $installedAppId + Name = $installedAppName + } + $selectedAppPaths[$installedTestAppPath] = $true + } + } + } + + return @($testApps) +} + +function Get-DisabledTestsForApp { + <# + .SYNOPSIS + Gets the disabled tests configured for a test app. + .DESCRIPTION + Loads disabledTests.json files recursively under the matching test folder and project-wide + .disabledTests.json files. + .PARAMETER settings + The analyzed AL-Go settings hashtable. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER appId + The ID of the test app. + #> + Param( + [hashtable] $settings, + [string] $projectPath, + [string] $appId + ) + + $disabledTestFiles = @() + foreach ($testFolder in @($settings.testFolders)) { + $testFolderPath = Join-Path $projectPath $testFolder + $appJsonPath = Join-Path $testFolderPath "app.json" + if (-not (Test-Path $appJsonPath -PathType Leaf)) { + continue + } + + $testAppJson = Get-Content -Path $appJsonPath -Raw -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -recurse + if ("$($testAppJson.id)" -eq $appId) { + $disabledTestFiles += @(Get-ChildItem -LiteralPath $testFolderPath -Filter "disabledTests.json" -File -Recurse -Force | ForEach-Object { $_.FullName }) + } + } + + $disabledTestFiles += @(Get-ChildItem -LiteralPath $projectPath -Filter "$appId.disabledTests.json" -File -Recurse -Force | ForEach-Object { $_.FullName }) + + $disabledTests = @() + foreach ($disabledTestFile in @($disabledTestFiles | Sort-Object -Unique)) { + try { + $disabledTestsJson = Get-Content -Path $disabledTestFile -Raw -Encoding UTF8 + $parsedDisabledTests = $disabledTestsJson | ConvertFrom-Json + foreach ($disabledTest in $parsedDisabledTests) { + $disabledTests += @(ConvertTo-HashTable -object $disabledTest -recurse) + } + } + catch { + throw "Failed to parse disabled tests JSON file '$disabledTestFile'. Error: $($_.Exception.Message)" + } + } + + return @($disabledTests) +} + +function Copy-TestResultsToBuildArtifacts { + <# + .SYNOPSIS + Copies an existing test result file to the project build artifacts folder. + .DESCRIPTION + Preserves the project result for AnalyzeTests and creates an artifact copy only when a result + exists. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER testResultsFile + The canonical test result file in the project root. + #> + Param( + [string] $projectPath, + [string] $testResultsFile + ) + + $buildArtifactsFolder = Join-Path $projectPath ".buildartifacts" + $artifactTestResultsFile = Join-Path $buildArtifactsFolder "TestResults.xml" + try { + if (-not (Test-Path -Path $testResultsFile -PathType Leaf -ErrorAction Stop)) { + return + } + New-Item -Path $buildArtifactsFolder -ItemType Directory -Force -ErrorAction Stop | Out-Null + Copy-Item -Path $testResultsFile -Destination $artifactTestResultsFile -Force -ErrorAction Stop + } + catch { + throw "Failed to copy test results from '$testResultsFile' to '$artifactTestResultsFile'. Error: $($_.Exception.Message)" + } +} + +function Export-AlGoContainerEventLog { + <# + .SYNOPSIS + Exports the kept-alive container event log to the project folder. + .DESCRIPTION + Replaces ContainerEventLog.evtx only after a readable export is available, preserving an + existing diagnostic when export fails. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER containerName + The name of the kept-alive build container. + #> + Param( + [string] $projectPath, + [string] $containerName + ) + + $containerEventLogFile = Join-Path $projectPath "ContainerEventLog.evtx" + try { + $exportedEventLogFile = Get-BcContainerEventLog -containerName $containerName -doNotOpen + if ([string]::IsNullOrWhiteSpace("$exportedEventLogFile") -or -not (Test-Path -Path $exportedEventLogFile -PathType Leaf -ErrorAction Stop)) { + throw "Get-BcContainerEventLog did not return a readable event log file." + } + + Copy-Item -Path $exportedEventLogFile -Destination $containerEventLogFile -Force -ErrorAction Stop + } + catch { + throw "Failed to capture event log from container '$containerName' to '$containerEventLogFile'. Error: $($_.Exception.Message)" + } +} + +function Invoke-AlGoTestRun { + <# + .SYNOPSIS + Runs the normal tests for an AL-Go project against a kept-alive build container. + .DESCRIPTION + Runs each selected test app with AlTool or a RunTestsInBcContainer override. Preserves + TestResults.xml for AnalyzeTests, copies produced results to .buildartifacts, and refreshes + ContainerEventLog.evtx after every outcome. Event-log capture failures are warnings and do + not change the test outcome. + .PARAMETER settings + The (analyzed) AL-Go settings hashtable. + .PARAMETER projectPath + The full path to the project folder. + .PARAMETER containerName + The name of the build container to run the tests against. + .PARAMETER credential + The credential used to connect to the build container. + .PARAMETER installTestAppsJson + Path to a JSON file with the list of installed test apps. + .PARAMETER runTestsOverride + Optional scriptblock overriding the built-in AlTool test runner (RunTestsInBcContainer). + #> + Param( + [hashtable] $settings, + [string] $projectPath, + [string] $containerName, + [System.Management.Automation.PSCredential] $credential, + [string] $installTestAppsJson = '', + [scriptblock] $runTestsOverride = $null + ) + + try { + $testApps = Get-TestAppsToRun -settings $settings -projectPath $projectPath ` + -installTestAppsJson $installTestAppsJson + if (@($testApps).Count -eq 0) { + Write-Host "No test apps found to run tests in. Skipping test execution." + return + } + + Write-Host "Running tests against container '$containerName'" + + $testResultsFile = Join-Path $projectPath "TestResults.xml" + $artifactTestResultsFile = Join-Path (Join-Path $projectPath ".buildartifacts") "TestResults.xml" + foreach ($previousResultFile in @($testResultsFile, $artifactTestResultsFile)) { + if (Test-Path $previousResultFile) { + Remove-Item $previousResultFile -Force + } + } + + if (-not $runTestsOverride) { + Install-AlTool | Out-Null + } + + # Test failures surface as warnings when treatTestFailuresAsWarnings is set, otherwise as errors. + $gitHubActionsSeverity = if ($settings.treatTestFailuresAsWarnings) { 'warning' } else { 'error' } + $testType = if ($settings.ContainsKey("testType")) { "$($settings.testType)" } else { "" } + + $allTestsPassed = $true + $testRunError = $null + Push-Location $projectPath + try { + foreach ($testApp in $testApps) { + Write-Host "Running tests in $($testApp.Name) ($($testApp.Id))" + $disabledTests = @(Get-DisabledTestsForApp -settings $settings -projectPath $projectPath -appId "$($testApp.Id)") + + if ($runTestsOverride) { + $runTestsParams = @{ + "containerName" = $containerName + "credential" = $credential + "companyName" = $settings.companyName + "extensionId" = $testApp.Id + "appName" = $testApp.Name + "disabledTests" = $disabledTests + "JUnitResultFileName" = $testResultsFile + "AppendToJUnitResultFile" = $true + "detailed" = $true + "GitHubActions" = $gitHubActionsSeverity + "returnTrueIfAllPassed" = $true + } + if (-not [string]::IsNullOrWhiteSpace($testType)) { + $runTestsParams["testType"] = $testType + } + $passed = & $runTestsOverride -parameters $runTestsParams + } + else { + $passed = Invoke-AlToolTestRun ` + -ContainerName $containerName ` + -Credential $credential ` + -ExtensionId "$($testApp.Id)" ` + -AppName "$($testApp.Name)" ` + -CompanyName "$($settings.companyName)" ` + -Tenant "default" ` + -TestType $testType ` + -DisabledTests @($disabledTests) ` + -JUnitResultFileName $testResultsFile + } + + if (-not $passed) { + $allTestsPassed = $false + } + } + } + catch { + $testRunError = $_ + } + finally { + Pop-Location + } + + try { + Copy-TestResultsToBuildArtifacts -projectPath $projectPath -testResultsFile $testResultsFile + } + catch { + if ($testRunError) { + OutputWarning -message "The test run failed and produced test results could not be copied to build artifacts. $($_.Exception.Message)" + } + else { + throw + } + } + + if ($testRunError) { + throw $testRunError + } + + if (-not $allTestsPassed) { + if ($settings.treatTestFailuresAsWarnings) { + OutputWarning -message "There are test failures, but they are treated as warnings (treatTestFailuresAsWarnings is set)." + } + else { + throw "There are test failures." + } + } + } + finally { + try { + Export-AlGoContainerEventLog ` + -projectPath $projectPath ` + -containerName $containerName + } + catch { + OutputWarning -message "The post-test container event log could not be captured. $($_.Exception.Message)" + } + } +} + +Export-ModuleMember -Function Invoke-AlGoTestRun, Get-TestAppsToRun diff --git a/Actions/RunTests/action.yaml b/Actions/RunTests/action.yaml new file mode 100644 index 0000000000..4cd99644f4 --- /dev/null +++ b/Actions/RunTests/action.yaml @@ -0,0 +1,35 @@ +name: Run Tests +author: Microsoft Corporation +inputs: + shell: + description: Shell in which you want to run the action (powershell or pwsh) + required: false + default: powershell + token: + description: The GitHub token running the action + required: false + default: ${{ github.token }} + project: + description: Project folder + required: false + default: '.' + installTestAppsJson: + description: A path to a JSON-formatted list of test apps to run tests in + required: false + default: '' +runs: + using: composite + steps: + - name: run + shell: ${{ inputs.shell }} + env: + _token: ${{ inputs.token }} + _project: ${{ inputs.project }} + _installTestAppsJson: ${{ inputs.installTestAppsJson }} + run: | + ${{ github.action_path }}/../Invoke-AlGoAction.ps1 -ActionName "RunTests" -Action { + ${{ github.action_path }}/RunTests.ps1 -token $ENV:_token -project $ENV:_project -installTestAppsJson $ENV:_installTestAppsJson + } +branding: + icon: terminal + color: blue diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1daebdd442..2a54b31b58 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,3 +1,12 @@ +### Separate test execution from RunPipeline (PREVIEW) + +A new `useSeparateTestAction` setting (default `false`) runs normal tests (`testFolders`) in a dedicated `RunTests` action. The separate action is used when tests are enabled and the build uses one local container. Builds with `additionalCountries` or without a local build container continue to run normal tests in `RunPipeline`. BCPT and page scripting tests remain in `RunPipeline`. + +The action uses AlTool by default, running each app's enabled normal tests in one batch and connection. It honors `disabledTests.json` definitions and the optional `testType` setting. The built-in runner supports `UnitTest`, `IntegrationTest`, and `Uncategorized`, while custom `RunTestsInBcContainer` overrides may interpret other values such as `Legacy`. Test results remain available to AnalyzeTests and are included in the build artifacts when produced. Failure diagnostics include a refreshed container event log with events from the separate test run. + +> [!NOTE] +> The built-in AlTool execution path does not run `Legacy` test-type codeunits or tests that require UI or client-callback interaction. Projects that rely on those should supply a `RunTestsInBcContainer` override script to execute their tests through BcContainerHelper instead. + ### Expanded AL-Go telemetry dashboard The starter Azure Data Explorer dashboard now includes dedicated views for workflow reliability, run exploration, test quality, workflow duration, runner efficiency, and AL-Go maintenance. It also provides repository, workflow, branch, and repository-type filtering, clearer empty states, and repository-level runtime supportability information. diff --git a/Scenarios/settings.md b/Scenarios/settings.md index c0118f92d4..bf3bb4c275 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -244,6 +244,8 @@ Please read the release notes carefully when installing new versions of AL-Go fo | doNotBuildTests | This setting forces the pipeline to NOT build and run the tests and performance tests in testFolders and bcptTestFolders | false | | doNotRunTests | This setting forces the pipeline to NOT run the tests in testFolders. Tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | | doNotRunBcptTests | This setting forces the pipeline to NOT run the performance tests in testFolders. Performance tests are still being built and published. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false | +| useSeparateTestAction | PREVIEW: When set to true, normal tests from testFolders run in a separate RunTests action against the build container kept alive by RunPipeline. BCPT and page scripting tests remain in RunPipeline. Builds with additionalCountries continue to run normal tests inside RunPipeline so every country is tested. Existing behavior is unchanged when this setting is false. | false | +| testType | Optional string used by the separate RunTests action to select a test type. The built-in AlTool runner supports `UnitTest`, `IntegrationTest`, and `Uncategorized`; blank runs all test types. Custom `RunTestsInBcContainer` overrides may interpret other values such as `Legacy`. RunPipeline does not use this setting. | | | memoryLimit | Specifies the memory limit for the build container. By default, this is left to BcContainerHelper to handle and will currently be set to 8G | 8G | | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles (**deprecated**) | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index bc3ef290ea..6d30336dc8 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -229,6 +229,16 @@ jobs: baselineWorkflowSHA: ${{ inputs.baselineWorkflowSHA }} previousAppsPath: ${{ steps.DownloadPreviousRelease.outputs.PreviousAppsPath }} + - name: Run Tests + uses: microsoft/AL-Go-Actions/RunTests@main + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.runTestsInSeparateAction == 'True' + env: + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' + with: + shell: ${{ inputs.shell }} + project: ${{ inputs.project }} + installTestAppsJson: ${{ steps.DownloadProjectDependencies.outputs.DownloadedTestApps }} + - name: Sign id: sign if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && inputs.signArtifacts && env.doNotSignApps == 'False' && (env.keyVaultCodesignCertificateName != '' || (fromJson(env.trustedSigning).Endpoint != '' && fromJson(env.trustedSigning).Account != '' && fromJson(env.trustedSigning).CertificateProfile != '')) && (hashFiles(format('{0}/.buildartifacts/Apps/*.app',inputs.project)) != '') diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index bc3ef290ea..6d30336dc8 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -229,6 +229,16 @@ jobs: baselineWorkflowSHA: ${{ inputs.baselineWorkflowSHA }} previousAppsPath: ${{ steps.DownloadPreviousRelease.outputs.PreviousAppsPath }} + - name: Run Tests + uses: microsoft/AL-Go-Actions/RunTests@main + if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && env.runTestsInSeparateAction == 'True' + env: + Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' + with: + shell: ${{ inputs.shell }} + project: ${{ inputs.project }} + installTestAppsJson: ${{ steps.DownloadProjectDependencies.outputs.DownloadedTestApps }} + - name: Sign id: sign if: steps.DetermineBuildProject.outputs.BuildIt == 'True' && inputs.signArtifacts && env.doNotSignApps == 'False' && (env.keyVaultCodesignCertificateName != '' || (fromJson(env.trustedSigning).Endpoint != '' && fromJson(env.trustedSigning).Account != '' && fromJson(env.trustedSigning).CertificateProfile != '')) && (hashFiles(format('{0}/.buildartifacts/Apps/*.app',inputs.project)) != '') diff --git a/Tests/AlToolTestRunner.Test.ps1 b/Tests/AlToolTestRunner.Test.ps1 new file mode 100644 index 0000000000..92c15b967c --- /dev/null +++ b/Tests/AlToolTestRunner.Test.ps1 @@ -0,0 +1,1651 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'Mock/callback parameters must match function signatures')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'Test-only credential')] +param() + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +Import-Module (Join-Path $PSScriptRoot '../Actions/.Modules/DebugLogHelper.psm1' -Resolve) -DisableNameChecking -Force +Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + +Describe 'AlToolTestRunner.psm1 Tests' { + + BeforeAll { + . (Join-Path -Path $PSScriptRoot -ChildPath "../Actions/AL-Go-Helper.ps1" -Resolve) + $script:stubbedGlobalCommands = @() + $stubDefinitions = @{ + 'ConvertTo-HashTable' = ${function:ConvertTo-HashTable} + 'Get-TestsFromBcContainer' = { + param( + [string] $containerName, + [string] $tenant, + [System.Management.Automation.PSCredential] $credential, + [string] $extensionId, + [string] $testType, + [switch] $ignoreGroups + ) + throw 'Get-TestsFromBcContainer must be mocked by the test.' + } + 'Get-BcContainerServerConfiguration' = { + param([string] $containerName) + throw 'Get-BcContainerServerConfiguration must be mocked by the test.' + } + 'Get-CompanyInBcContainer' = { + param( + [string] $containerName, + [string] $tenant + ) + throw 'Get-CompanyInBcContainer must be mocked by the test.' + } + } + foreach ($commandName in $stubDefinitions.Keys) { + if (-not (Test-Path -LiteralPath "Function:\global:$commandName")) { + Set-Item -LiteralPath "Function:\global:$commandName" -Value $stubDefinitions[$commandName] + $script:stubbedGlobalCommands += $commandName + } + } + + # Re-import in the run phase so the module functions are guaranteed to be available even when + # this file runs in the same Invoke-Pester session as RunTests.Test.ps1. RunTests.psm1 + # imports AlToolTestRunner.psm1 as a nested module with -Force, which removes the standalone + # module's functions from the global scope during discovery. + Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' -Resolve) -DisableNameChecking -Force + } + + AfterAll { + foreach ($commandName in $script:stubbedGlobalCommands) { + Remove-Item "Function:\global:$commandName" -Force -ErrorAction SilentlyContinue + } + } + + Context 'Module exports' { + It 'Exports only the RunTests integration functions' { + @(Get-Command -Module AlToolTestRunner).Name | Sort-Object | + Should -Be @('Install-AlTool', 'Invoke-AlToolTestRun') + } + } + + Context 'Invoke-AlNativeCommand' { + It 'Captures native stdout, redirected stderr and a nonzero exit code in Windows PowerShell 5' { + if ([System.Environment]::OSVersion.Platform -ne [System.PlatformID]::Win32NT) { + Set-ItResult -Skipped -Because 'Windows PowerShell 5 is only available on Windows' + return + } + + $exitCode = 1 + $modulePath = (Resolve-Path (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1')).Path + $windowsPowerShell = (Get-Command (Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe') -ErrorAction Stop).Source + $childScript = "[Console]::Out.WriteLine('native-stdout'); [Console]::Error.WriteLine('native-stderr'); exit $ExitCode" + $encodedChildScript = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($childScript)) + $escapedModulePath = $modulePath.Replace("'", "''") + $escapedWindowsPowerShell = $windowsPowerShell.Replace("'", "''") + + $parentScript = @" +`$ErrorActionPreference = 'Stop' +`$module = Import-Module '$escapedModulePath' -Force -PassThru +`$result = & `$module { + Invoke-AlNativeCommand -FilePath '$escapedWindowsPowerShell' -ArgumentList @( + '-NoLogo', '-NoProfile', '-NonInteractive', '-EncodedCommand', '$encodedChildScript' + ) +} +@{ + StandardOutput = @(`$result.StandardOutput) + StandardError = @(`$result.StandardError) + Output = @(`$result.Output) + ExitCode = `$result.ExitCode + ErrorActionPreference = "`$ErrorActionPreference" +} | ConvertTo-Json -Compress +"@ + $encodedParentScript = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($parentScript)) + + $parentOutput = & $windowsPowerShell -NoLogo -NoProfile -EncodedCommand $encodedParentScript 2>&1 + $parentExitCode = $LASTEXITCODE + + $parentExitCode | Should -Be 0 + $payload = ($parentOutput -join "`n") | ConvertFrom-Json + $payload.ExitCode | Should -Be $ExitCode + $payload.ErrorActionPreference | Should -Be 'Stop' + @($payload.StandardOutput) | Should -Be @('native-stdout') + ($payload.StandardError -join "`n") | Should -Match 'native-stderr' + ($payload.Output -join "`n") | Should -Match 'native-stdout' + ($payload.Output -join "`n") | Should -Match 'native-stderr' + } + + It 'Captures LASTEXITCODE immediately after the stderr-redirected native invocation' { + InModuleScope AlToolTestRunner { + $functionAst = (Get-Command Invoke-AlNativeCommand).ScriptBlock.Ast + $outputAssignment = @($functionAst.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.AssignmentStatementAst] -and + $node.Left.Extent.Text -eq '$standardOutput' + }, $true))[0] + $statements = @($outputAssignment.Parent.Statements) + $outputIndex = [Array]::IndexOf($statements, $outputAssignment) + $nativeCommand = $outputAssignment.Right.PipelineElements[0] + $errorRedirection = $nativeCommand.Redirections[0] + $exitCodeAssignment = $statements[$outputIndex + 1] + + $nativeCommand | Should -BeOfType ([System.Management.Automation.Language.CommandAst]) + $nativeCommand.InvocationOperator | Should -Be ([System.Management.Automation.Language.TokenKind]::Ampersand) + $nativeCommand.Redirections.Count | Should -Be 1 + $errorRedirection | Should -BeOfType ([System.Management.Automation.Language.FileRedirectionAst]) + $errorRedirection.FromStream | Should -Be ([System.Management.Automation.Language.RedirectionStream]::Error) + $exitCodeAssignment | Should -BeOfType ([System.Management.Automation.Language.AssignmentStatementAst]) + $exitCodeAssignment.Left.Extent.Text | Should -Be '[int] $exitCode' + $exitCodeAssignment.Right.Expression.VariablePath.UserPath | Should -Be 'LASTEXITCODE' + } + } + + It 'Preserves stdout, captures stderr and restores preferences after a nonzero exit' { + $exitCode = 1 + $powerShell = (Get-Process -Id $PID).Path + $childScript = "[Console]::Out.WriteLine('native-stdout'); [Console]::Error.WriteLine('native-stderr'); exit $ExitCode" + $encodedChildScript = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($childScript)) + + InModuleScope AlToolTestRunner -Parameters @{ + PowerShellPath = $powerShell + EncodedScript = $encodedChildScript + ExpectedExit = $ExitCode + } { + $originalErrorActionPreference = $ErrorActionPreference + $nativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + $originalNativePreference = if ($nativePreference) { $nativePreference.Value } else { $null } + $existingTempFiles = @(Get-ChildItem -LiteralPath ([System.IO.Path]::GetTempPath()) -Filter 'altool-stderr-*.txt' | + ForEach-Object { $_.FullName }) + try { + $ErrorActionPreference = 'Stop' + if ($nativePreference) { + $PSNativeCommandUseErrorActionPreference = $true + } + + $result = Invoke-AlNativeCommand -FilePath $PowerShellPath -ArgumentList @( + '-NoLogo', '-NoProfile', '-NonInteractive', '-EncodedCommand', $EncodedScript + ) + + $result.ExitCode | Should -Be $ExpectedExit + $result.StandardOutput | Should -Be @('native-stdout') + ($result.StandardError -join "`n") | Should -Match 'native-stderr' + $result.Output[0] | Should -Be 'native-stdout' + ($result.Output -join "`n") | Should -Match 'native-stderr' + $ErrorActionPreference | Should -Be 'Stop' + if ($nativePreference) { + $PSNativeCommandUseErrorActionPreference | Should -BeTrue + } + $remainingTempFiles = @(Get-ChildItem -LiteralPath ([System.IO.Path]::GetTempPath()) -Filter 'altool-stderr-*.txt' | + ForEach-Object { $_.FullName }) + @($remainingTempFiles | Where-Object { $_ -notin $existingTempFiles }).Count | Should -Be 0 + } + finally { + $ErrorActionPreference = $originalErrorActionPreference + if ($nativePreference) { + $PSNativeCommandUseErrorActionPreference = $originalNativePreference + } + } + } + } + + It 'Does not swallow command-not-found errors' { + InModuleScope AlToolTestRunner { + { Invoke-AlNativeCommand -FilePath 'al-go-command-that-does-not-exist' } | + Should -Throw + } + } + + It 'Does not swallow native invocation failures' { + if ([System.Environment]::OSVersion.Platform -ne [System.PlatformID]::Win32NT) { + Set-ItResult -Skipped -Because 'The invalid Windows executable fixture is Windows-specific' + return + } + + $invalidExecutable = Join-Path $TestDrive 'invalid.exe' + Set-Content -Path $invalidExecutable -Value 'not an executable' -Encoding ASCII + + InModuleScope AlToolTestRunner -Parameters @{ InvalidExecutable = $invalidExecutable } { + $originalErrorActionPreference = $ErrorActionPreference + $nativePreference = Get-Variable -Name PSNativeCommandUseErrorActionPreference -ErrorAction SilentlyContinue + $originalNativePreference = if ($nativePreference) { $nativePreference.Value } else { $null } + $existingTempFiles = @(Get-ChildItem -LiteralPath ([System.IO.Path]::GetTempPath()) -Filter 'altool-stderr-*.txt' | + ForEach-Object { $_.FullName }) + try { + $ErrorActionPreference = 'Stop' + if ($nativePreference) { + $PSNativeCommandUseErrorActionPreference = $true + } + + { Invoke-AlNativeCommand -FilePath $InvalidExecutable } | Should -Throw + + $ErrorActionPreference | Should -Be 'Stop' + if ($nativePreference) { + $PSNativeCommandUseErrorActionPreference | Should -BeTrue + } + $remainingTempFiles = @(Get-ChildItem -LiteralPath ([System.IO.Path]::GetTempPath()) -Filter 'altool-stderr-*.txt' | + ForEach-Object { $_.FullName }) + @($remainingTempFiles | Where-Object { $_ -notin $existingTempFiles }).Count | Should -Be 0 + } + finally { + $ErrorActionPreference = $originalErrorActionPreference + if ($nativePreference) { + $PSNativeCommandUseErrorActionPreference = $originalNativePreference + } + } + } + } + } + + Context 'Install-AlTool native command handling' { + InModuleScope AlToolTestRunner { + BeforeAll { + function Get-TestInstallMutex { + param( + [ValidateSet('Acquired', 'Timeout')] + [string] $WaitBehavior + ) + + $mutex = [PSCustomObject]@{ + WaitBehavior = $WaitBehavior + ReleaseCount = 0 + DisposeCount = 0 + } + $mutex | Add-Member -MemberType ScriptMethod -Name WaitOne -Value { + param([TimeSpan] $Timeout) + $null = $Timeout + return $this.WaitBehavior -eq 'Acquired' + } + $mutex | Add-Member -MemberType ScriptMethod -Name ReleaseMutex -Value { + $this.ReleaseCount = [int] $this.ReleaseCount + 1 + } + $mutex | Add-Member -MemberType ScriptMethod -Name Dispose -Value { + $this.DisposeCount = [int] $this.DisposeCount + 1 + } + return $mutex + } + } + + It 'Stops before native work when the installation mutex times out' { + $script:testInstallMutex = Get-TestInstallMutex -WaitBehavior Timeout + Mock -ModuleName AlToolTestRunner New-Object { return $script:testInstallMutex } -ParameterFilter { + $TypeName -eq 'System.Threading.Mutex' + } + Mock -ModuleName AlToolTestRunner Get-Command { return $null } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand {} + + { Install-AlTool } | Should -Throw '*Timed out after 10 minutes*AlTool installation mutex*' + + $script:testInstallMutex.ReleaseCount | Should -Be 0 + $script:testInstallMutex.DisposeCount | Should -Be 1 + Should -Invoke -ModuleName AlToolTestRunner Get-Command -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlNativeCommand -Times 0 -Exactly + } + + It 'Releases the installation mutex after normal acquisition' { + $script:testInstallMutex = Get-TestInstallMutex -WaitBehavior Acquired + Mock -ModuleName AlToolTestRunner New-Object { return $script:testInstallMutex } -ParameterFilter { + $TypeName -eq 'System.Threading.Mutex' + } + Mock -ModuleName AlToolTestRunner Get-Command { + return [PSCustomObject]@{ Source = 'al' } + } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + return [PSCustomObject]@{ + StandardOutput = [string[]]@('1.2.3') + StandardError = [string[]]@() + Output = [string[]]@('1.2.3') + ExitCode = [int] 0 + } + } + + Install-AlTool | Should -Be '1.2.3' + + $script:testInstallMutex.ReleaseCount | Should -Be 1 + $script:testInstallMutex.DisposeCount | Should -Be 1 + } + + It 'Adds the platform dotnet global tools directory to PATH' { + $userProfile = [Environment]::GetFolderPath([Environment+SpecialFolder]::UserProfile) + $userProfile | Should -Not -BeNullOrEmpty + $expectedToolsPath = Join-Path (Join-Path $userProfile '.dotnet') 'tools' + $previousPath = $env:PATH + try { + $env:PATH = @($env:PATH -split [System.IO.Path]::PathSeparator | + Where-Object { $_ -ne $expectedToolsPath }) -join [System.IO.Path]::PathSeparator + Mock -ModuleName AlToolTestRunner Get-Command { + return [PSCustomObject]@{ Source = 'al' } + } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + return [PSCustomObject]@{ + StandardOutput = [string[]]@('1.2.3') + StandardError = [string[]]@() + Output = [string[]]@('1.2.3') + ExitCode = [int] 0 + } + } + + Install-AlTool | Should -Be '1.2.3' + + @($env:PATH -split [System.IO.Path]::PathSeparator) | Should -Contain $expectedToolsPath + } + finally { + $env:PATH = $previousPath + } + } + + It 'Falls back to update after install failure only when al is still unavailable' { + $script:availabilityChecks = 0 + Mock -ModuleName AlToolTestRunner Get-Command { + $script:availabilityChecks++ + if ($script:availabilityChecks -eq 1) { return $null } + return [PSCustomObject]@{ Source = 'al' } + } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + if ($FilePath -eq 'dotnet') { + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@('install failed') + Output = [string[]]@('install failed') + ExitCode = [int] 1 + } + } + return [PSCustomObject]@{ + StandardOutput = [string[]]@('1.2.3') + StandardError = [string[]]@() + Output = [string[]]@('1.2.3') + ExitCode = [int] 0 + } + } + + Install-AlTool | Should -Be '1.2.3' + + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlNativeCommand -Times 0 -Exactly -ParameterFilter { + $FilePath -eq 'dotnet' -and $ArgumentList[1] -eq 'update' + } + } + + It 'Uses a successful fallback update after install fails and al remains unavailable' { + $script:availabilityChecks = 0 + Mock -ModuleName AlToolTestRunner Get-Command { + $script:availabilityChecks++ + if ($script:availabilityChecks -lt 3) { return $null } + return [PSCustomObject]@{ Source = 'al' } + } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + if ($FilePath -eq 'dotnet') { + $exitCode = if ($ArgumentList[1] -eq 'install') { 1 } else { 0 } + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@() + Output = [string[]]@() + ExitCode = [int] $exitCode + } + } + return [PSCustomObject]@{ + StandardOutput = [string[]]@('1.2.3') + StandardError = [string[]]@() + Output = [string[]]@('1.2.3') + ExitCode = [int] 0 + } + } + + Install-AlTool | Should -Be '1.2.3' + + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlNativeCommand -Times 1 -Exactly -ParameterFilter { + $FilePath -eq 'dotnet' -and $ArgumentList[1] -eq 'update' + } + } + + It 'Fails when al remains unavailable after a successful installation command' { + Mock -ModuleName AlToolTestRunner Get-Command { return $null } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@() + Output = [string[]]@() + ExitCode = [int] 0 + } + } + + { Install-AlTool } | Should -Throw "*'al' CLI is not available after installation*" + } + + It 'Reports a failed fallback update clearly' { + Mock -ModuleName AlToolTestRunner Get-Command { return $null } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + if ($ArgumentList[1] -eq 'install') { + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@('install failed') + Output = [string[]]@('install failed') + ExitCode = [int] 1 + } + } + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@('update stderr') + Output = [string[]]@('update stderr') + ExitCode = [int] 17 + } + } + + { Install-AlTool } | Should -Throw '*fallback dotnet tool update exited with code 17*update stderr*' + } + + It 'Reports al version failure explicitly' { + Mock -ModuleName AlToolTestRunner Get-Command { return [PSCustomObject]@{ Source = 'al' } } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@('version stderr') + Output = [string[]]@('version stderr') + ExitCode = [int] 11 + } + } + + { Install-AlTool } | Should -Throw "*'al --version'*exited with code 11*version stderr*" + } + } + } + + Context 'Get-DisabledTestKeySet' { + InModuleScope AlToolTestRunner { + It 'Reads production hashtable entries with scalar, array, duplicate, and wildcard methods' { + $disabled = @( + @{ codeunitName = 'My Tests'; method = 'TestOne' }, + @{ codeunitName = 'MY TESTS'; method = @('TestTwo', 'TESTTHREE') }, + @{ codeunitName = 'my tests'; method = 'testone' }, + @{ codeunitName = 'Whole CU'; method = '*' } + ) + + $lookup = Get-DisabledTestKeySet -DisabledTests $disabled + + @($lookup.Methods.Keys | Sort-Object) | + Should -Be @('my tests::testone', 'my tests::testthree', 'my tests::testtwo') + @($lookup.Codeunits.Keys) | Should -Be @('whole cu') + $lookup.Methods.ContainsKey('whole cu::*') | Should -BeFalse + } + + It 'Returns empty sets for an empty list' { + $lookup = Get-DisabledTestKeySet -DisabledTests @() + $lookup.Methods.Count | Should -Be 0 + $lookup.Codeunits.Count | Should -Be 0 + } + } + } + + Context 'Get-AlToolTestCodeunits' { + InModuleScope AlToolTestRunner { + It 'Enumerates codeunits and filters disabled methods and whole codeunits' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @( + [PSCustomObject]@{ Id = '130001'; Name = 'My Tests'; Tests = @('TestOne', 'TestTwo') }, + [PSCustomObject]@{ Id = '130002'; Name = 'Whole CU'; Tests = @('X', 'Y') } + ) + } + $testCodeunitParams = @{ + ContainerName = 'test' + Credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + ExtensionId = [Guid]::NewGuid().ToString() + DisabledTests = @( + @{ codeunitName = 'My Tests'; method = 'TestTwo' }, + @{ codeunitName = 'Whole CU'; method = '*' } + ) + } + + $codeunits = @(Get-AlToolTestCodeunits @testCodeunitParams) + + $codeunits.Count | Should -Be 1 + $codeunits[0].Name | Should -Be 'My Tests' + @($codeunits[0].Tests).Count | Should -Be 1 + $codeunits[0].Tests[0] | Should -Be 'TestOne' + + $groupsPath = InModuleScope AlToolTestRunner -Parameters @{ Codeunits = $codeunits } { + New-AlTestGroupsFile -Codeunits $Codeunits + } + try { + $groupsJson = Get-Content -LiteralPath $groupsPath -Raw -Encoding UTF8 + $groupsJson.TrimStart() | Should -Match '^\[' + $groupsJson.Trim() | + Should -Be '[{"codeunitId":130001,"testMethods":["TestOne"]}]' + } + finally { + Remove-Item -LiteralPath $groupsPath -Force -ErrorAction SilentlyContinue + } + } + + It 'Filters the production hashtable shape case-insensitively while preserving enabled order' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @( + [PSCustomObject]@{ + Id = 130001; Name = 'My Tests'; Tests = @('TestOne', 'TestTwo', 'TestThree', 'TestFour') + }, + [PSCustomObject]@{ Id = 130002; Name = 'Whole CU'; Tests = @('X', 'Y') }, + [PSCustomObject]@{ Id = 130003; Name = 'Other Tests'; Tests = @('First', 'Second') } + ) + } + $testCodeunitParams = @{ + ContainerName = 'test' + Credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + ExtensionId = [Guid]::NewGuid().ToString() + DisabledTests = @( + @{ codeunitName = 'MY TESTS'; method = 'testtwo' }, + @{ codeunitName = 'my tests'; method = @('TESTTHREE') }, + @{ codeunitName = 'My Tests'; method = 'TestTwo' }, + @{ codeunitName = 'whole cu'; method = '*' } + ) + } + + $codeunits = @(Get-AlToolTestCodeunits @testCodeunitParams) + + $codeunits.Count | Should -Be 2 + @($codeunits.Name) | Should -Be @('My Tests', 'Other Tests') + @($codeunits[0].Tests) | Should -Be @('TestOne', 'TestFour') + @($codeunits[1].Tests) | Should -Be @('First', 'Second') + } + + It 'Omits testType when blank and returns every codeunit' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @([PSCustomObject]@{ Id = 130001; Name = 'My Tests'; Tests = @('TestOne') }) + } + $testCodeunitParams = @{ + ContainerName = 'test' + Credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + ExtensionId = [Guid]::NewGuid().ToString() + TestType = '' + } + + $codeunits = @(Get-AlToolTestCodeunits @testCodeunitParams) + $codeunits.Count | Should -Be 1 + Should -Invoke -ModuleName AlToolTestRunner Get-TestsFromBcContainer -Times 1 -Exactly -ParameterFilter { + -not $PSBoundParameters.ContainsKey('testType') + } + } + + It 'Passes a supported testType to BcContainerHelper' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + @([PSCustomObject]@{ Id = 130001; Name = 'Integration Tests'; Tests = @('TestOne') }) + } + $testCodeunitParams = @{ + ContainerName = 'test' + Credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + ExtensionId = [Guid]::NewGuid().ToString() + TestType = 'IntegrationTest' + } + + $codeunits = @(Get-AlToolTestCodeunits @testCodeunitParams) + + $codeunits.Count | Should -Be 1 + Should -Invoke -ModuleName AlToolTestRunner Get-TestsFromBcContainer -Times 1 -Exactly -ParameterFilter { + $testType -eq 'IntegrationTest' + } + } + + It 'Rejects an unsupported built-in testType before calling BcContainerHelper' { + Mock -ModuleName AlToolTestRunner Get-TestsFromBcContainer { + throw 'BcContainerHelper should not be called' + } + $testCodeunitParams = @{ + ContainerName = 'test' + Credential = (New-Object System.Management.Automation.PSCredential('admin', (ConvertTo-SecureString 'password' -AsPlainText -Force))) + ExtensionId = [Guid]::NewGuid().ToString() + TestType = 'Legacy' + } + + { Get-AlToolTestCodeunits @testCodeunitParams } | + Should -Throw "*Unsupported testType 'Legacy'*UnitTest*IntegrationTest*Uncategorized*" + Should -Invoke -ModuleName AlToolTestRunner Get-TestsFromBcContainer -Times 0 -Exactly + } + } + } + + Context 'Get-AlToolConnection' { + InModuleScope AlToolTestRunner { + It 'Reads server instance and developer services port from the container configuration' { + Mock -ModuleName AlToolTestRunner Get-BcContainerServerConfiguration { + [PSCustomObject]@{ ServerInstance = 'MyBC'; DeveloperServicesPort = 7145 } + } + + $connection = Get-AlToolConnection -ContainerName 'mycontainer' + + $connection.Server | Should -Be 'http://mycontainer' + $connection.ServerInstance | Should -Be 'MyBC' + $connection.Port | Should -Be 7145 + } + + It 'Falls back to conventional defaults when configuration cannot be read' { + Mock -ModuleName AlToolTestRunner Get-BcContainerServerConfiguration { throw 'no such container' } + + $connection = Get-AlToolConnection -ContainerName 'mycontainer' + + $connection.Server | Should -Be 'http://mycontainer' + $connection.ServerInstance | Should -Be 'BC' + $connection.Port | Should -Be 7049 + } + } + } + + Context 'Get-AlToolCompany' { + InModuleScope AlToolTestRunner { + It 'Honors an explicitly requested company name without querying the container' { + Mock -ModuleName AlToolTestRunner Get-CompanyInBcContainer { throw 'should not be called' } + $company = Get-AlToolCompany -ContainerName 'test' -Tenant 'default' -CompanyName 'CRONUS' + $company | Should -Be 'CRONUS' + } + + It 'Falls back to the container default company, preferring an evaluation company' { + Mock -ModuleName AlToolTestRunner Get-CompanyInBcContainer { + @( + [PSCustomObject]@{ companyName = 'CRONUS Real'; evaluationCompany = $false }, + [PSCustomObject]@{ companyName = 'CRONUS Eval'; evaluationCompany = $true } + ) + } + $company = Get-AlToolCompany -ContainerName 'test' -Tenant 'default' + $company | Should -Be 'CRONUS Eval' + } + } + } + + Context 'ConvertFrom-AlTestGroupsOutput' { + InModuleScope AlToolTestRunner { + It 'Maps pass, fail, and skip result occurrences by codeunit' { + $response = @{ + succeeded = $false + message = 'One or more tests failed.' + data = @{ + results = @( + @{ codeunitId = 130001; methodName = 'SameName'; status = 'passed'; output = ''; durationMs = 11 }, + @{ codeunitId = 130001; methodName = 'Fails'; status = 'failed'; output = "assertion failed`nAL Callstack:`nline one`nline two"; durationMs = 12 }, + @{ codeunitId = 130002; methodName = 'SameName'; status = 'skipped'; output = ''; durationMs = 0 } + ) + } + } | ConvertTo-Json -Depth 6 + + $parsed = ConvertFrom-AlTestGroupsOutput -OutputLines @($response) + + $parsed.Succeeded | Should -BeFalse + $parsed.Message | Should -Be 'One or more tests failed.' + @($parsed.Results['130001']).Count | Should -Be 2 + ($parsed.Results['130001'] | Where-Object MethodName -eq 'SameName').Outcome | Should -Be 'Pass' + ($parsed.Results['130002'] | Where-Object MethodName -eq 'SameName').Outcome | Should -Be 'Skip' + $failedResult = $parsed.Results['130001'] | Where-Object MethodName -eq 'Fails' + $failedResult.Outcome | Should -Be 'Fail' + $failedResult.Message | Should -Be 'assertion failed' + $failedResult.Stacktrace | Should -Be 'line one;line two' + } + + It 'Preserves every duplicate result occurrence in response order' { + $response = @{ + succeeded = $true + data = @{ + results = @( + @{ codeunitId = 130001; methodName = 'Duplicate'; status = 'passed'; output = ''; durationMs = 1 }, + @{ codeunitId = 130001; methodName = 'Duplicate'; status = 'failed'; output = 'failed'; durationMs = 2 }, + @{ codeunitId = 130001; methodName = 'Duplicate'; status = 'passed'; output = ''; durationMs = 3 } + ) + } + } | ConvertTo-Json -Depth 6 + + $parsed = ConvertFrom-AlTestGroupsOutput -OutputLines @($response) + + @($parsed.Results['130001']).Count | Should -Be 3 + @($parsed.Results['130001'].Outcome) | Should -Be @('Pass', 'Fail', 'Pass') + @($parsed.Results['130001'].Ms) | Should -Be @(1, 2, 3) + } + + It 'Parses durationMs values beyond the Int32 range as Int64' { + $response = @{ + succeeded = $true + data = @{ + results = @( + @{ + codeunitId = 130001 + methodName = 'LongRunning' + status = 'passed' + output = '' + durationMs = 3000000000 + } + ) + } + } | ConvertTo-Json -Depth 6 + + $parsed = ConvertFrom-AlTestGroupsOutput -OutputLines @($response) + + $parsed.Results['130001'][0].Ms | Should -Be ([long] 3000000000) + $parsed.Results['130001'][0].Ms.GetType() | Should -Be ([long]) + } + + It 'Accepts a skipped missing-codeunit sentinel with a blank method name' { + $response = @{ + succeeded = $true + data = @{ + results = @( + @{ codeunitId = 0; methodName = ''; status = 'skipped'; output = ''; durationMs = 0 } + ) + } + } | ConvertTo-Json -Depth 6 + + $parsed = ConvertFrom-AlTestGroupsOutput -OutputLines @($response) + + $parsed.Results['0'][0].MethodName | Should -Be '' + $parsed.Results['0'][0].Outcome | Should -Be 'Skip' + } + } + } + + Context 'New-AlTestGroupsFile' { + It 'Rejects an invalid codeunit ID' -TestCases @( + @{ Id = ' ' } + @{ Id = 'not-a-number' } + @{ Id = '2147483648' } + ) { + param($Id) + + InModuleScope AlToolTestRunner -Parameters @{ CodeunitId = $Id } { + { + New-AlTestGroupsFile -Codeunits @( + [PSCustomObject]@{ Id = $CodeunitId; Tests = @('TestOne') } + ) + } | Should -Throw "*must be a valid Int32 value*" + } + } + } + + Context 'Invoke-AlRunTestsBatch' { + BeforeEach { + $script:batchCodeunits = @( + [PSCustomObject]@{ Id = '130001'; Name = 'First Tests'; Tests = @('TestOne', 'TestTwo') }, + [PSCustomObject]@{ Id = '130002'; Name = 'Second Tests'; Tests = @('TestThree') } + ) + $script:batchConnection = @{ Server = 'http://test'; ServerInstance = 'BC'; Port = 7049 } + $script:capturedBatchArguments = $null + $script:capturedTestGroupsPath = $null + $script:capturedTestGroupsJson = $null + Mock -ModuleName AlToolTestRunner OutputDebug {} + } + + It 'Uses one testgroups invocation with the exact enabled method lists and removes the temporary file' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $script:capturedBatchArguments = @($ArgumentList) + $testGroupsIndex = [Array]::IndexOf($ArgumentList, '--testgroups') + $script:capturedTestGroupsPath = $ArgumentList[$testGroupsIndex + 1] + $script:capturedTestGroupsJson = Get-Content -LiteralPath $script:capturedTestGroupsPath -Raw -Encoding UTF8 + $stdout = @{ + succeeded = $true + data = @{ + results = @( + @{ codeunitId = 130001; methodName = 'TestOne'; status = 'passed'; output = ''; durationMs = 1 }, + @{ codeunitId = 130001; methodName = 'TestTwo'; status = 'passed'; output = ''; durationMs = 2 }, + @{ codeunitId = 130002; methodName = 'TestThree'; status = 'passed'; output = ''; durationMs = 3 } + ) + } + } | ConvertTo-Json -Depth 6 + return [PSCustomObject]@{ + StandardOutput = [string[]]@($stdout) + StandardError = [string[]]@() + Output = [string[]]@($stdout) + ExitCode = [int] 0 + } + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + $result = Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + $result.Results['130001'].Count | Should -Be 2 + $result.Results['130002'].Count | Should -Be 1 + $result.ElapsedSec | Should -BeGreaterOrEqual 0 + } + + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlNativeCommand -Times 1 -Exactly + $script:capturedBatchArguments[0] | Should -Be 'runtests' + $script:capturedBatchArguments | Should -Contain '--testgroups' + $script:capturedBatchArguments | Should -Not -Contain '--project' + $script:capturedBatchArguments | Should -Not -Contain '--raw' + $script:capturedBatchArguments | Should -Not -Contain '--testmethods' + $script:capturedBatchArguments | Should -Not -Contain '130001' + $script:capturedBatchArguments | Should -Not -Contain '130002' + $script:capturedTestGroupsJson.Trim() | + Should -Be '[{"codeunitId":130001,"testMethods":["TestOne","TestTwo"]},{"codeunitId":130002,"testMethods":["TestThree"]}]' + Test-Path -LiteralPath $script:capturedTestGroupsPath | Should -BeFalse + Should -Invoke -ModuleName AlToolTestRunner OutputDebug -Times 0 -Exactly + } + + It 'Writes successful native stderr only to debug output' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $stdout = @{ + succeeded = $true + data = @{ + results = @( + @{ codeunitId = 130001; methodName = 'TestOne'; status = 'passed'; output = ''; durationMs = 1 }, + @{ codeunitId = 130001; methodName = 'TestTwo'; status = 'passed'; output = ''; durationMs = 2 }, + @{ codeunitId = 130002; methodName = 'TestThree'; status = 'passed'; output = ''; durationMs = 3 } + ) + } + } | ConvertTo-Json -Depth 6 + return [PSCustomObject]@{ + StandardOutput = [string[]]@($stdout) + StandardError = [string[]]@('informational diagnostic', 'server trace') + Output = [string[]]@($stdout, 'informational diagnostic', 'server trace') + ExitCode = [int] 0 + } + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + $result = Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + $result.Succeeded | Should -BeTrue + } + + Should -Invoke -ModuleName AlToolTestRunner OutputDebug -Times 1 -Exactly -ParameterFilter { + $message -like 'al runtests stderr:*informational diagnostic*server trace*' + } + } + + It 'Parses valid failed-test JSON when AlTool exits with code one' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $stdout = @{ + succeeded = $false + data = @{ + results = @( + @{ codeunitId = 130001; methodName = 'TestOne'; status = 'failed'; output = 'failed'; durationMs = 4 }, + @{ codeunitId = 130001; methodName = 'TestTwo'; status = 'passed'; output = ''; durationMs = 5 }, + @{ codeunitId = 130002; methodName = 'TestThree'; status = 'skipped'; output = ''; durationMs = 0 } + ) + } + } | ConvertTo-Json -Depth 6 + return [PSCustomObject]@{ + StandardOutput = [string[]]@($stdout) + StandardError = [string[]]@('test diagnostics') + Output = [string[]]@($stdout, 'test diagnostics') + ExitCode = [int] 1 + } + } + Mock -ModuleName AlToolTestRunner Write-Host {} + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + $result = Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + $result.Succeeded | Should -BeFalse + ($result.Results['130001'] | Where-Object MethodName -eq 'TestOne').Outcome | Should -Be 'Fail' + ($result.Results['130001'] | Where-Object MethodName -eq 'TestTwo').Outcome | Should -Be 'Pass' + ($result.Results['130002'] | Where-Object MethodName -eq 'TestThree').Outcome | Should -Be 'Skip' + } + + Should -Invoke -ModuleName AlToolTestRunner Write-Host -Times 0 -Exactly -ParameterFilter { + "$Object" -like '::warning::*' + } + Should -Invoke -ModuleName AlToolTestRunner OutputDebug -Times 1 -Exactly -ParameterFilter { + $message -like 'al runtests stderr:*test diagnostics*' + } + } + + It 'Terminates when AlTool exits above one despite complete passing JSON' { + Mock -ModuleName AlToolTestRunner ConvertFrom-AlTestGroupsOutput { + throw 'structured response should not be parsed' + } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $stdout = @{ + succeeded = $false + data = @{ + results = @( + @{ codeunitId = 130001; methodName = 'TestOne'; status = 'passed'; output = ''; durationMs = 1 }, + @{ codeunitId = 130001; methodName = 'TestTwo'; status = 'passed'; output = ''; durationMs = 2 }, + @{ codeunitId = 130002; methodName = 'TestThree'; status = 'skipped'; output = ''; durationMs = 0 } + ) + } + } | ConvertTo-Json -Depth 6 + return [PSCustomObject]@{ + StandardOutput = [string[]]@($stdout) + StandardError = [string[]]@('transport failed') + Output = [string[]]@($stdout, 'transport failed') + ExitCode = [int] 9 + } + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + { + Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + } | Should -Throw '*process failure*unexpected code 9*stderr: transport failed*' + } + Should -Invoke -ModuleName AlToolTestRunner ConvertFrom-AlTestGroupsOutput -Times 0 -Exactly + } + + It 'Surfaces stderr when AlTool fails before serializing a ToolResponse' { + Mock -ModuleName AlToolTestRunner ConvertFrom-AlTestGroupsOutput { + throw 'empty stdout should not be parsed' + } + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@('no response diagnostic') + Output = [string[]]@('no response diagnostic') + ExitCode = [int] 1 + } + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + { + Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + } | Should -Throw '*al runtests failed: no response diagnostic*exit code 1*' + } + Should -Invoke -ModuleName AlToolTestRunner ConvertFrom-AlTestGroupsOutput -Times 0 -Exactly + } + + It 'Reports the exit code when AlTool returns no stdout or stderr' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + return [PSCustomObject]@{ + StandardOutput = [string[]]@() + StandardError = [string[]]@() + Output = [string[]]@() + ExitCode = [int] 1 + } + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + { + Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + } | Should -Throw '*no structured stdout or stderr*exit code 1*' + } + } + + It 'Surfaces the failed-envelope message and stderr when results are absent' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $stdout = @{ + succeeded = $false + message = 'The company could not be opened.' + } | ConvertTo-Json -Depth 5 + return [PSCustomObject]@{ + StandardOutput = [string[]]@($stdout) + StandardError = [string[]]@('The company could not be opened.', 'server refused the connection') + Output = [string[]]@($stdout, 'The company could not be opened.', 'server refused the connection') + ExitCode = [int] 1 + } + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + try { + Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + throw 'Expected Invoke-AlRunTestsBatch to fail.' + } + catch { + $_.Exception.Message | Should -BeLike '*The company could not be opened*' + $_.Exception.Message | Should -BeLike '*server refused the connection*' + } + } + } + + It 'Does not repeat stderr that exactly matches a failed-envelope message' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $stdout = @{ + succeeded = $false + message = 'The company could not be opened.' + } | ConvertTo-Json + return [PSCustomObject]@{ + StandardOutput = [string[]]@($stdout) + StandardError = [string[]]@('The company could not be opened.') + Output = [string[]]@($stdout, 'The company could not be opened.') + ExitCode = [int] 1 + } + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + { + Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + } | Should -Throw 'al runtests failed: The company could not be opened.' + } + } + + It 'Terminates with stdout and stderr when the batch response is malformed' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $testGroupsIndex = [Array]::IndexOf($ArgumentList, '--testgroups') + $script:capturedTestGroupsPath = $ArgumentList[$testGroupsIndex + 1] + return [PSCustomObject]@{ + StandardOutput = [string[]]@('{not-json') + StandardError = [string[]]@('parse diagnostic') + Output = [string[]]@('{not-json', 'parse diagnostic') + ExitCode = [int] 1 + } + } + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + { + Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + } | Should -Throw '*protocol failure*could not be parsed as JSON*stdout: {not-json*stderr: parse diagnostic*' + } + + Test-Path -LiteralPath $script:capturedTestGroupsPath | Should -BeFalse + } + + It 'Removes the testgroups file when native invocation fails' { + Mock -ModuleName AlToolTestRunner Invoke-AlNativeCommand { + $testGroupsIndex = [Array]::IndexOf($ArgumentList, '--testgroups') + $script:capturedTestGroupsPath = $ArgumentList[$testGroupsIndex + 1] + throw 'native invocation failed' + } + + InModuleScope AlToolTestRunner -Parameters @{ + Codeunits = $script:batchCodeunits + Connection = $script:batchConnection + } { + { + Invoke-AlRunTestsBatch -Codeunits $Codeunits -Company 'CRONUS' ` + -Tenant 'default' -Connection $Connection + } | Should -Throw '*native invocation failed*' + } + + Test-Path -LiteralPath $script:capturedTestGroupsPath | Should -BeFalse + } + } + + Context 'Invoke-AlToolTestRun parameter contract' { + BeforeAll { + $script:requiredParameterCredential = New-Object System.Management.Automation.PSCredential( + 'admin', + (ConvertTo-SecureString 'password' -AsPlainText -Force) + ) + $script:requiredParameterExtensionId = [Guid]::NewGuid().ToString() + $script:requiredParameterResultFile = Join-Path $TestDrive 'RequiredParameters.xml' + $script:requiredParameterModulePath = (Resolve-Path ( + Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' + )).Path + $script:requiredParameterPowerShell = (Get-Process -Id $PID).Path + } + + It 'Declares only the explicit runner contract and marks required values mandatory' { + $command = Get-Command Invoke-AlToolTestRun + + $command.Parameters.ContainsKey('Parameters') | Should -BeFalse + foreach ($parameterName in @('ContainerName', 'Credential', 'ExtensionId', 'JUnitResultFileName')) { + $parameterAttribute = $command.Parameters[$parameterName].Attributes | + Where-Object { $_ -is [System.Management.Automation.ParameterAttribute] } | + Select-Object -First 1 + $parameterAttribute.Mandatory | Should -BeTrue + } + $command.Parameters.DisabledTests.ParameterType | Should -Be ([hashtable[]]) + $command.Parameters.TestType.ParameterType | Should -Be ([string]) + } + + It 'Rejects calls that omit required parameter ' -TestCases @( + @{ ParameterName = 'ContainerName' } + @{ ParameterName = 'Credential' } + @{ ParameterName = 'ExtensionId' } + @{ ParameterName = 'JUnitResultFileName' } + ) { + param($ParameterName) + + $parameterExpressions = [ordered]@{ + ContainerName = "-ContainerName 'test'" + Credential = '-Credential $credential' + ExtensionId = "-ExtensionId '$($script:requiredParameterExtensionId)'" + JUnitResultFileName = "-JUnitResultFileName '$($script:requiredParameterResultFile)'" + } + $null = $parameterExpressions.Remove($ParameterName) + $escapedModulePath = $script:requiredParameterModulePath.Replace("'", "''") + $childScript = @" +`$ErrorActionPreference = 'Stop' +Import-Module '$escapedModulePath' -Force +`$credential = New-Object System.Management.Automation.PSCredential( + 'admin', + (ConvertTo-SecureString 'password' -AsPlainText -Force) +) +Invoke-AlToolTestRun $($parameterExpressions.Values -join ' ') +"@ + $encodedScript = [Convert]::ToBase64String([Text.Encoding]::Unicode.GetBytes($childScript)) + $standardOutputPath = Join-Path $TestDrive "$ParameterName-stdout.txt" + $standardErrorPath = Join-Path $TestDrive "$ParameterName-stderr.txt" + + $process = Start-Process -FilePath $script:requiredParameterPowerShell -ArgumentList @( + '-NoLogo', '-NoProfile', '-NonInteractive', '-EncodedCommand', $encodedScript + ) -RedirectStandardOutput $standardOutputPath -RedirectStandardError $standardErrorPath -Wait -PassThru + $bindingOutput = @( + Get-Content -LiteralPath $standardOutputPath -Raw -ErrorAction SilentlyContinue + Get-Content -LiteralPath $standardErrorPath -Raw -ErrorAction SilentlyContinue + ) -join "`n" + + $process.ExitCode | Should -Not -Be 0 + $bindingOutput | Should -Match ([regex]::Escape($ParameterName)) + } + + It 'Rejects a blank JUnitResultFileName' -TestCases @( + @{ Value = $null } + @{ Value = '' } + @{ Value = ' ' } + ) { + param($Value) + + { + Invoke-AlToolTestRun -ContainerName 'test' -Credential $script:requiredParameterCredential ` + -ExtensionId $script:requiredParameterExtensionId -JUnitResultFileName $Value + } | Should -Throw + } + } + + Context 'Invoke-AlToolTestRun batch behavior' { + BeforeEach { + $script:testRunCredential = New-Object System.Management.Automation.PSCredential( + 'admin', + (ConvertTo-SecureString 'password' -AsPlainText -Force) + ) + $script:testRunParameters = @{ + ContainerName = 'test' + Credential = $script:testRunCredential + ExtensionId = [Guid]::NewGuid().ToString() + AppName = 'Test App' + JUnitResultFileName = Join-Path $TestDrive 'TestResults.xml' + } + $script:testRunCodeunit = [PSCustomObject]@{ + Id = 130001 + Name = 'My Tests' + Tests = @('TestOne') + } + Remove-Item -LiteralPath (Join-Path $TestDrive 'TestResults.xml') -Force -ErrorAction SilentlyContinue + + Mock -ModuleName AlToolTestRunner Install-AlTool { return '1.2.3' } + Mock -ModuleName AlToolTestRunner Get-Command { return $null } -ParameterFilter { $Name -eq 'al' } + Mock -ModuleName AlToolTestRunner Get-AlToolConnection { + return @{ Server = 'http://test'; ServerInstance = 'BC'; Port = 7049 } + } + Mock -ModuleName AlToolTestRunner Get-AlToolCompany { return 'CRONUS' } + Mock -ModuleName AlToolTestRunner Get-AlToolTestCodeunits { return @($script:testRunCodeunit) } + Mock -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch { + return @{ + Results = @{ '130001' = @( + @{ MethodName = 'TestOne'; Outcome = 'Pass'; Ms = 1; Message = ''; Stacktrace = '' } + ) } + Succeeded = $true + ElapsedSec = 0.1 + } + } + Mock -ModuleName AlToolTestRunner OutputWarning {} + Mock -ModuleName AlToolTestRunner Write-Host {} + } + + It 'Creates a new testsuites document when the result file does not exist' { + $junitFile = $script:testRunParameters.JUnitResultFileName + Test-Path -LiteralPath $junitFile | Should -BeFalse + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + + [xml] $junit = Get-Content -LiteralPath $junitFile -Raw -Encoding UTF8 + $junit.DocumentElement.LocalName | Should -Be 'testsuites' + $junit.SelectNodes('testsuites/testsuite').Count | Should -Be 1 + $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='TestOne']") | Should -Not -BeNullOrEmpty + } + + It 'Installs AlTool when a direct call cannot find al' { + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + + Should -Invoke -ModuleName AlToolTestRunner Get-Command -Times 1 -Exactly -ParameterFilter { + $Name -eq 'al' -and $ErrorAction -eq 'SilentlyContinue' + } + Should -Invoke -ModuleName AlToolTestRunner Install-AlTool -Times 1 -Exactly + } + + It 'Does not install AlTool when a direct call finds al' { + Mock -ModuleName AlToolTestRunner Get-Command { + return [PSCustomObject]@{ Name = 'al' } + } -ParameterFilter { $Name -eq 'al' } + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + + Should -Invoke -ModuleName AlToolTestRunner Get-Command -Times 1 -Exactly -ParameterFilter { + $Name -eq 'al' -and $ErrorAction -eq 'SilentlyContinue' + } + Should -Invoke -ModuleName AlToolTestRunner Install-AlTool -Times 0 -Exactly + } + + It 'Runs all codeunits for an app in exactly one batch' { + $script:testRunParameters.TestType = 'UnitTest' + $secondCodeunit = [PSCustomObject]@{ Id = 130002; Name = 'Other Tests'; Tests = @('TestTwo') } + Mock -ModuleName AlToolTestRunner Get-AlToolTestCodeunits { + return @($script:testRunCodeunit, $secondCodeunit) + } + Mock -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch { + return @{ + Results = @{ + '130001' = @(@{ MethodName = 'TestOne'; Outcome = 'Pass'; Ms = 1; Message = ''; Stacktrace = '' }) + '130002' = @(@{ MethodName = 'TestTwo'; Outcome = 'Pass'; Ms = 1; Message = ''; Stacktrace = '' }) + } + Succeeded = $true + ElapsedSec = 0.1 + } + } + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + + Should -Invoke -ModuleName AlToolTestRunner Get-AlToolTestCodeunits -Times 1 -Exactly -ParameterFilter { + $ContainerName -eq 'test' -and + $Credential.UserName -eq 'admin' -and + $ExtensionId -eq $script:testRunParameters.ExtensionId -and + $Tenant -eq 'default' -and + $TestType -eq 'UnitTest' -and + $DisabledTests.Count -eq 0 + } + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 1 -Exactly -ParameterFilter { + $Codeunits.Count -eq 2 -and + "$($Codeunits[0].Id)" -eq '130001' -and + "$($Codeunits[1].Id)" -eq '130002' + } + } + + It 'Keeps pass, fail, and skip outcomes from the single batch' { + $script:testRunCodeunit.Tests = @('Passing', 'Failing', 'Skipped') + $junitFile = Join-Path $TestDrive 'TestResults.xml' + $script:testRunParameters.JUnitResultFileName = $junitFile + Mock -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch { + return @{ + Results = @{ '130001' = @( + @{ MethodName = 'Passing'; Outcome = 'Pass'; Ms = 1; Message = ''; Stacktrace = '' }, + @{ MethodName = 'Failing'; Outcome = 'Fail'; Ms = 2; Message = 'primary failure'; Stacktrace = 'stack' }, + @{ MethodName = 'Skipped'; Outcome = 'Skip'; Ms = 0; Message = ''; Stacktrace = '' } + ) } + Succeeded = $false + ElapsedSec = 0.1 + } + } + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeFalse + + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 1 -Exactly + [xml] $junit = Get-Content -Path $junitFile -Raw + $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='Passing']/failure") | Should -BeNullOrEmpty + $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='Failing']/failure").GetAttribute('message') | + Should -Be 'primary failure' + $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='Skipped']/skipped") | Should -Not -BeNullOrEmpty + } + + It 'Turns a partial valid batch into a final missing-result JUnit failure' { + $script:testRunCodeunit.Tests = @('Reported', 'Missing') + $junitFile = Join-Path $TestDrive 'TestResults.xml' + $script:testRunParameters.JUnitResultFileName = $junitFile + Mock -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch { + return @{ + Results = @{ '130001' = @( + @{ MethodName = 'Reported'; Outcome = 'Pass'; Ms = 1; Message = ''; Stacktrace = '' } + ) } + Succeeded = $false + ElapsedSec = 0.1 + } + } + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeFalse + + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 1 -Exactly + [xml] $junit = Get-Content -Path $junitFile -Raw + $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='Reported']/failure") | Should -BeNullOrEmpty + $missingFailure = $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='Missing']/failure") + $missingFailure.GetAttribute('message') | Should -Be 'No result produced by al runtests' + } + + It 'Reports a failed batch envelope after preserving complete JUnit results' { + $junitFile = Join-Path $TestDrive 'TestResults.xml' + $script:testRunParameters.JUnitResultFileName = $junitFile + Mock -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch { + return @{ + Results = @{ '130001' = @( + @{ MethodName = 'TestOne'; Outcome = 'Pass'; Ms = 1; Message = ''; Stacktrace = '' } + ) } + Succeeded = $false + ElapsedSec = 0.1 + } + } + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeFalse + + [xml] $junit = Get-Content -Path $junitFile -Raw + $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='TestOne']") | Should -Not -BeNullOrEmpty + $junit.SelectSingleNode("testsuites/testsuite/testcase[@name='TestOne']/failure") | Should -BeNullOrEmpty + } + + It 'Emits exact duplicate results and includes a duplicate failure in allPassed' { + $script:testRunCodeunit.Tests = @('Duplicate') + $junitFile = Join-Path $TestDrive 'TestResults.xml' + $script:testRunParameters.JUnitResultFileName = $junitFile + Mock -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch { + return @{ + Results = @{ '130001' = @( + @{ MethodName = 'Duplicate'; Outcome = 'Pass'; Ms = 1; Message = ''; Stacktrace = '' }, + @{ MethodName = 'Duplicate'; Outcome = 'Fail'; Ms = 2; Message = 'second occurrence failed'; Stacktrace = 'stack' } + ) } + Succeeded = $false + ElapsedSec = 0.1 + } + } + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeFalse + + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 1 -Exactly + [xml] $junit = Get-Content -Path $junitFile -Raw + $cases = @($junit.SelectNodes("testsuites/testsuite/testcase[@name='Duplicate']")) + $cases.Count | Should -Be 2 + @($cases | Where-Object { $null -ne $_.SelectSingleNode('failure') }).Count | Should -Be 1 + $cases[1].SelectSingleNode('failure').GetAttribute('message') | Should -Be 'second occurrence failed' + $junit.SelectNodes("testsuites/testsuite/testcase/failure[@message='No result produced by al runtests']").Count | + Should -Be 0 + } + + It 'Clears runtime credentials when batch execution fails' { + $previousUserName = $env:BC_SERVER_USERNAME + $previousPassword = $env:BC_SERVER_PASSWORD + Mock -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch { + throw 'batch failed' + } + + try { + { Invoke-AlToolTestRun @script:testRunParameters } | Should -Throw '*batch failed*' + Test-Path Env:\BC_SERVER_USERNAME | Should -BeFalse + Test-Path Env:\BC_SERVER_PASSWORD | Should -BeFalse + } + finally { + if ($null -ne $previousUserName) { + $env:BC_SERVER_USERNAME = $previousUserName + } + if ($null -ne $previousPassword) { + $env:BC_SERVER_PASSWORD = $previousPassword + } + } + } + + It 'Clears runtime credentials after successful batch execution' { + $previousUserName = $env:BC_SERVER_USERNAME + $previousPassword = $env:BC_SERVER_PASSWORD + try { + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + Test-Path Env:\BC_SERVER_USERNAME | Should -BeFalse + Test-Path Env:\BC_SERVER_PASSWORD | Should -BeFalse + } + finally { + if ($null -eq $previousUserName) { + Remove-Item Env:\BC_SERVER_USERNAME -ErrorAction SilentlyContinue + } + else { + $env:BC_SERVER_USERNAME = $previousUserName + } + if ($null -eq $previousPassword) { + Remove-Item Env:\BC_SERVER_PASSWORD -ErrorAction SilentlyContinue + } + else { + $env:BC_SERVER_PASSWORD = $previousPassword + } + } + } + + It 'Skips AlTool when enumeration finds no enabled codeunits' { + Mock -ModuleName AlToolTestRunner Get-AlToolTestCodeunits { return @() } + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + Should -Invoke -ModuleName AlToolTestRunner Install-AlTool -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 0 -Exactly + } + + It 'Preserves a valid existing testsuites document and appends the next app suite' { + $junitFile = Join-Path $TestDrive 'TestResults.xml' + $script:testRunParameters.JUnitResultFileName = $junitFile + + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + [xml] $firstDocument = Get-Content -LiteralPath $junitFile -Raw -Encoding UTF8 + $firstSuiteXml = $firstDocument.SelectSingleNode('testsuites/testsuite').OuterXml + + $script:testRunParameters.AppName = 'Second Test App' + Invoke-AlToolTestRun @script:testRunParameters | Should -BeTrue + + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 2 -Exactly + [xml] $junit = Get-Content -LiteralPath $junitFile -Raw -Encoding UTF8 + $suites = @($junit.SelectNodes('testsuites/testsuite')) + $suites.Count | Should -Be 2 + $suites[0].OuterXml | Should -Be $firstSuiteXml + $junit.SelectNodes('testsuites/testsuite/testcase').Count | Should -Be 2 + } + + It 'Fails on malformed accumulated JUnit without replacing the file' { + $junitFile = $script:testRunParameters.JUnitResultFileName + $originalContent = '' + Set-Content -LiteralPath $junitFile -Value $originalContent -Encoding UTF8 -NoNewline + + $loadMessage = try { + Invoke-AlToolTestRun @script:testRunParameters + '' + } + catch { + $_.Exception.Message + } + + $loadMessage | Should -Match 'Could not load existing JUnit file' + $loadMessage | Should -Match ([regex]::Escape($junitFile)) + Get-Content -LiteralPath $junitFile -Raw -Encoding UTF8 | Should -Be $originalContent + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner OutputWarning -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner Write-Host -Times 0 -Exactly -ParameterFilter { + "$Object" -like '*starting fresh*' + } + } + + It 'Fails on an unexpected accumulated JUnit root without replacing the file' { + $junitFile = $script:testRunParameters.JUnitResultFileName + $originalContent = '' + Set-Content -LiteralPath $junitFile -Value $originalContent -Encoding UTF8 -NoNewline + + $loadMessage = try { + Invoke-AlToolTestRun @script:testRunParameters + '' + } + catch { + $_.Exception.Message + } + + $loadMessage | Should -Match 'Existing JUnit file' + $loadMessage | Should -Match ([regex]::Escape($junitFile)) + $loadMessage | Should -Match "expected a 'testsuites' root element" + Get-Content -LiteralPath $junitFile -Raw -Encoding UTF8 | Should -Be $originalContent + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner OutputWarning -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner Write-Host -Times 0 -Exactly -ParameterFilter { + "$Object" -like '*starting fresh*' + } + } + + It 'Fails with file context when the accumulated JUnit path cannot be read as a file' { + $junitPath = Join-Path $TestDrive 'UnreadableResults' + New-Item -ItemType Directory -Path $junitPath | Out-Null + $script:testRunParameters.JUnitResultFileName = $junitPath + + $loadMessage = try { + Invoke-AlToolTestRun @script:testRunParameters + '' + } + catch { + $_.Exception.Message + } + + $loadMessage | Should -Match 'Could not load existing JUnit file' + $loadMessage | Should -Match ([regex]::Escape($junitPath)) + Test-Path -LiteralPath $junitPath -PathType Container | Should -BeTrue + Should -Invoke -ModuleName AlToolTestRunner Invoke-AlRunTestsBatch -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner OutputWarning -Times 0 -Exactly + Should -Invoke -ModuleName AlToolTestRunner Write-Host -Times 0 -Exactly -ParameterFilter { + "$Object" -like '*starting fresh*' + } + } + } + + Context 'Add-JUnitTestSuite' { + InModuleScope AlToolTestRunner { + BeforeEach { + $script:doc = New-Object System.Xml.XmlDocument + $script:doc.AppendChild($script:doc.CreateXmlDeclaration("1.0", "UTF-8", $null)) | Out-Null + $script:suites = $script:doc.CreateElement("testsuites") + $script:doc.AppendChild($script:suites) | Out-Null + } + + It 'Writes a passing suite with correct counts and no failure nodes' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @( + @{ MethodName = 'TestA'; Outcome = 'Pass'; Ms = 10; Message = ''; Stacktrace = '' } + ) + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA') -MethodResults $methodResults -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' + + $failed | Should -Be 0 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('name') | Should -Be '130001 My Tests' + $suite.GetAttribute('tests') | Should -Be '1' + $suite.GetAttribute('failures') | Should -Be '0' + $suite.GetAttribute('time') | Should -Be '0.01' + $suite.SelectNodes('testcase/failure').Count | Should -Be 0 + } + + It 'Marks a requested method with no result as a failure' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('MissingTest') -MethodResults @() -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' + + $failed | Should -Be 1 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('failures') | Should -Be '1' + $suite.SelectSingleNode('testcase/failure').GetAttribute('message') | Should -Match 'No result produced' + } + + It 'Records a failing method with its message and stacktrace' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @( + @{ MethodName = 'TestA'; Outcome = 'Fail'; Ms = 4; Message = 'boom'; Stacktrace = 'line1;line2' } + ) + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA') -MethodResults $methodResults -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' + + $failed | Should -Be 1 + $failureNode = $script:suites.SelectSingleNode('testsuite/testcase/failure') + $failureNode.GetAttribute('message') | Should -Be 'boom' + $failureNode.InnerText | Should -Match 'line1' + $failureNode.InnerText | Should -Match 'line2' + } + + It 'Records skipped methods in the suite count' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @( + @{ MethodName = 'TestA'; Outcome = 'Skip'; Ms = 0; Message = ''; Stacktrace = '' } + ) + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA') -MethodResults $methodResults -ExtensionId 'ext-id' -AppName 'MyApp' ` + -Hostname 'host' + + $failed | Should -Be 0 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('tests') | Should -Be '1' + $suite.GetAttribute('skipped') | Should -Be '1' + $suite.SelectNodes('testcase/skipped').Count | Should -Be 1 + } + + It 'Sets suite time to the sum of available requested method durations' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @( + @{ MethodName = 'TestA'; Outcome = 'Pass'; Ms = 1250; Message = ''; Stacktrace = '' }, + @{ MethodName = 'TestB'; Outcome = 'Skip'; Ms = 250; Message = ''; Stacktrace = '' } + ) + + Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('TestA', 'TestB', 'MissingTest') -MethodResults $methodResults ` + -ExtensionId 'ext-id' -AppName 'MyApp' -Hostname 'host' | Out-Null + + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('time') | Should -Be '1.5' + $suite.SelectSingleNode("testcase[@name='MissingTest']").GetAttribute('time') | Should -Be '0' + } + + It 'Emits every decorated data-driven case and preserves nested brackets' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @( + @{ MethodName = 'DataTest[Case A]'; Outcome = 'Pass'; Ms = 100; Message = ''; Stacktrace = '' }, + @{ MethodName = 'DataTest[Outer[Inner]]'; Outcome = 'Skip'; Ms = 250; Message = ''; Stacktrace = '' } + ) + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('DataTest') -MethodResults $methodResults -ExtensionId 'ext-id' ` + -AppName 'MyApp' -Hostname 'host' + + $failed | Should -Be 0 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('tests') | Should -Be '2' + $suite.GetAttribute('skipped') | Should -Be '1' + $suite.GetAttribute('time') | Should -Be '0.35' + $suite.SelectSingleNode("testcase[@name='DataTest[Case A]']") | Should -Not -BeNullOrEmpty + $suite.SelectSingleNode("testcase[@name='DataTest[Outer[Inner]]']") | Should -Not -BeNullOrEmpty + $suite.SelectNodes("testcase/failure[@message='No result produced by al runtests']").Count | Should -Be 0 + } + + It 'Does not let an unrelated decorated result satisfy a requested base method' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @( + @{ MethodName = 'OtherTest[Case A]'; Outcome = 'Pass'; Ms = 100; Message = ''; Stacktrace = '' }, + @{ MethodName = 'DataTest[]'; Outcome = 'Pass'; Ms = 100; Message = ''; Stacktrace = '' } + ) + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('DataTest') -MethodResults $methodResults -ExtensionId 'ext-id' ` + -AppName 'MyApp' -Hostname 'host' + + $failed | Should -Be 1 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('tests') | Should -Be '1' + $suite.SelectSingleNode("testcase[@name='DataTest']/failure").GetAttribute('message') | + Should -Be 'No result produced by al runtests' + $suite.SelectSingleNode("testcase[@name='OtherTest[Case A]']") | Should -BeNullOrEmpty + $suite.SelectSingleNode("testcase[@name='DataTest[]']") | Should -BeNullOrEmpty + } + + It 'Emits every exact duplicate occurrence including mixed outcomes' { + $codeunit = [PSCustomObject]@{ Id = 130001; Name = 'My Tests' } + $methodResults = @( + @{ MethodName = 'Duplicate'; Outcome = 'Pass'; Ms = 100; Message = ''; Stacktrace = '' }, + @{ MethodName = 'Duplicate'; Outcome = 'Fail'; Ms = 200; Message = 'failed duplicate'; Stacktrace = 'line' } + ) + + $failed = Add-JUnitTestSuite -Doc $script:doc -TestSuitesNode $script:suites -Codeunit $codeunit ` + -RequestedMethods @('Duplicate') -MethodResults $methodResults -ExtensionId 'ext-id' ` + -AppName 'MyApp' -Hostname 'host' + + $failed | Should -Be 1 + $suite = $script:suites.SelectSingleNode('testsuite') + $suite.GetAttribute('tests') | Should -Be '2' + $suite.GetAttribute('failures') | Should -Be '1' + $suite.GetAttribute('time') | Should -Be '0.3' + $suite.SelectNodes("testcase[@name='Duplicate']").Count | Should -Be 2 + $suite.SelectNodes("testcase/failure[@message='No result produced by al runtests']").Count | Should -Be 0 + } + } + } +} diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index f53005c5f6..005d8aa483 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -56,6 +56,9 @@ Describe "AnalyzeTests Action Tests" { [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'thresholdsFile', Justification = 'False positive.')] $thresholdsFile = Join-Path ([System.IO.Path]::GetTempPath()) "$([GUID]::NewGuid().ToString()).json" @{ "NumberOfSqlStmtsThresholdWarning" = 1; "NumberOfSqlStmtsThresholdError" = 2 } | ConvertTo-Json | Set-Content -Path $thresholdsFile -Encoding UTF8 + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'alToolTestRunnerModule', Justification = 'Used to generate production JUnit in a test.')] + $alToolTestRunnerModule = Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/AlToolTestRunner.psm1' -Resolve) ` + -DisableNameChecking -Force -PassThru } It 'Compile Action' { @@ -68,6 +71,41 @@ Describe "AnalyzeTests Action Tests" { YamlTest -scriptRoot $scriptRoot -actionName $actionName -actionScript $actionScript -outputs $outputs } + It 'Analyzes mixed normal JUnit produced by Add-JUnitTestSuite' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + $junitFile = Join-Path $TestDrive 'TestResults.xml' + + & $alToolTestRunnerModule { + param($Path) + + $doc = [xml] '' + $codeunit = [PSCustomObject]@{ + Id = 130001 + Name = 'Mixed Tests' + Tests = @('Passes', 'Fails', 'Skips') + } + $results = @( + @{ MethodName = 'Passes'; Outcome = 'Pass'; Ms = 10; Message = ''; Stacktrace = '' }, + @{ MethodName = 'Fails'; Outcome = 'Fail'; Ms = 20; Message = 'assertion failed'; Stacktrace = 'line one;line two' }, + @{ MethodName = 'Skips'; Outcome = 'Skip'; Ms = 0; Message = ''; Stacktrace = '' } + ) + Add-JUnitTestSuite -Doc $doc -TestSuitesNode $doc.DocumentElement ` + -Codeunit $codeunit -RequestedMethods $codeunit.Tests -MethodResults $results ` + -ExtensionId '11111111-1111-1111-1111-111111111111' -AppName 'Test App' -Hostname 'host' | Out-Null + $doc.Save($Path) + } $junitFile + + $summary, $failures, $failureSummary = GetTestResultSummaryMD -testResultsFile $junitFile + + $summary | Should -Match '\|Test App\|3\|1[^|]*\|1[^|]*\|1[^|]*\|0\.03\|' + $failures | Should -Match 'Fails, Failure' + $failures | Should -Match 'assertion failed' + $failures | Should -Match 'line one' + $failures | Should -Not -Match 'Skips, Failure' + $failureSummary | Should -Be '1 failing tests, download test results to see details' + } + It 'Test ReadBcptFile' { . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') @@ -177,5 +215,7 @@ Describe "AnalyzeTests Action Tests" { Remove-Item -Path $bcptFilename -Force -ErrorAction SilentlyContinue Remove-Item -Path $bcptBaseLine1 -Force -ErrorAction SilentlyContinue Remove-Item -Path $bcptBaseLine2 -Force -ErrorAction SilentlyContinue + Remove-Item -Path $thresholdsFile -Force -ErrorAction SilentlyContinue + Remove-Module -ModuleInfo $alToolTestRunnerModule -Force -ErrorAction SilentlyContinue } } diff --git a/Tests/RunTests.Action.Test.ps1 b/Tests/RunTests.Action.Test.ps1 new file mode 100644 index 0000000000..eeb22d1abe --- /dev/null +++ b/Tests/RunTests.Action.Test.ps1 @@ -0,0 +1,267 @@ +Get-Module TestActionsHelper | Remove-Module -Force +Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +Describe "RunTests Action Tests" { + BeforeAll { + $actionName = "RunTests" + $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve + $scriptName = "$actionName.ps1" + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'scriptPath', Justification = 'False positive.')] + $scriptPath = Join-Path $scriptRoot $scriptName + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'actionScript', Justification = 'False positive.')] + $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName + + $tokens = $null + $parseErrors = $null + $actionAst = [System.Management.Automation.Language.Parser]::ParseFile($scriptPath, [ref] $tokens, [ref] $parseErrors) + $parseErrors | Should -BeNullOrEmpty + foreach ($functionName in @('Get-TestRunnerCredential', 'Get-TestRunnerContainerName')) { + $functionAst = $actionAst.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $functionName + }, $true) + . ([ScriptBlock]::Create($functionAst.Extent.Text)) + } + + $helperPath = Join-Path $scriptRoot '..\AL-Go-Helper.ps1' -Resolve + $helperAst = [System.Management.Automation.Language.Parser]::ParseFile($helperPath, [ref] $tokens, [ref] $parseErrors) + $parseErrors | Should -BeNullOrEmpty + $convertToHashTableAst = $helperAst.Find({ + param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'ConvertTo-HashTable' + }, $true) + . ([ScriptBlock]::Create($convertToHashTableAst.Extent.Text)) + + . $helperPath + Import-Module (Join-Path $scriptRoot 'RunTests.psm1' -Resolve) -DisableNameChecking -Force + + } + + BeforeEach { + $script:previousContainerCredential = $ENV:containerCredential + $script:previousContainerName = $ENV:containerName + $script:previousRunTestsToken = $ENV:_token + $script:previousTokenObservationPath = $ENV:_runTestsTokenObservationPath + $script:previousGitHubWorkspace = $ENV:GITHUB_WORKSPACE + $script:previousSettings = $ENV:Settings + } + + AfterEach { + if ($null -eq $script:previousContainerCredential) { + Remove-Item Env:\containerCredential -ErrorAction SilentlyContinue + } + else { + $ENV:containerCredential = $script:previousContainerCredential + } + if ($null -eq $script:previousContainerName) { + Remove-Item Env:\containerName -ErrorAction SilentlyContinue + } + else { + $ENV:containerName = $script:previousContainerName + } + if ($null -eq $script:previousRunTestsToken) { + Remove-Item Env:\_token -ErrorAction SilentlyContinue + } + else { + $ENV:_token = $script:previousRunTestsToken + } + if ($null -eq $script:previousTokenObservationPath) { + Remove-Item Env:\_runTestsTokenObservationPath -ErrorAction SilentlyContinue + } + else { + $ENV:_runTestsTokenObservationPath = $script:previousTokenObservationPath + } + if ($null -eq $script:previousGitHubWorkspace) { + Remove-Item Env:\GITHUB_WORKSPACE -ErrorAction SilentlyContinue + } + else { + $ENV:GITHUB_WORKSPACE = $script:previousGitHubWorkspace + } + if ($null -eq $script:previousSettings) { + Remove-Item Env:\Settings -ErrorAction SilentlyContinue + } + else { + $ENV:Settings = $script:previousSettings + } + } + + It 'Compile Action' { + Invoke-Expression $actionScript + } + + It 'Test action.yaml matches script' { + $outputs = [ordered]@{ + } + YamlTest -scriptRoot $scriptRoot -actionName $actionName -actionScript $actionScript -outputs $outputs + } + + It 'Loads only the runner override and delegates event log capture to the module' { + $overrideCommands = @($actionAst.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.CommandAst] -and + $node.GetCommandName() -eq 'Get-ScriptOverrides' + }, $true)) + $overrideCommands.Count | Should -Be 1 + $overrideParameter = @($overrideCommands[0].CommandElements | + Where-Object { + $_ -is [System.Management.Automation.Language.CommandParameterAst] -and + $_.ParameterName -eq 'OverrideScriptNames' + })[0] + $overrideParameterIndex = [Array]::IndexOf($overrideCommands[0].CommandElements, $overrideParameter) + $overrideArgument = $overrideCommands[0].CommandElements[$overrideParameterIndex + 1] + @($overrideArgument.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.StringConstantExpressionAst] + }, $true).Value) | Should -Be @('RunTestsInBcContainer') + + $actionEventLogCalls = @($actionAst.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.CommandAst] -and + $node.GetCommandName() -eq 'Export-AlGoContainerEventLog' + }, $true)) + $actionEventLogCalls.Count | Should -Be 0 + + $moduleTokens = $null + $moduleParseErrors = $null + $moduleAst = [System.Management.Automation.Language.Parser]::ParseFile( + (Join-Path $scriptRoot 'RunTests.psm1'), + [ref] $moduleTokens, + [ref] $moduleParseErrors + ) + $moduleParseErrors | Should -BeNullOrEmpty + @($moduleAst.FindAll({ + param($node) + $node -is [System.Management.Automation.Language.CommandAst] -and + $node.GetCommandName() -eq 'Export-AlGoContainerEventLog' + }, $true)).Count | Should -Be 1 + } + + It 'Refreshes the override token from a direct script parameter' { + $providedToken = "provided-$([Guid]::NewGuid())" + $ENV:_token = 'stale-token' + $ENV:GITHUB_WORKSPACE = $TestDrive + $ENV:Settings = '{}' + $ENV:containerName = 'test-container' + $ENV:_runTestsTokenObservationPath = Join-Path $TestDrive 'observed-token.txt' + $ENV:containerCredential = [Convert]::ToBase64String( + [Text.Encoding]::UTF8.GetBytes('{"username":"test-user","password":"test-password"}') + ) + + Mock DownloadAndImportBcContainerHelper {} + Mock AnalyzeRepo { return @{ testFolders = @() } } + Mock Get-ScriptOverrides { + return @{ + RunTestsInBcContainer = { + param([hashtable] $parameters) + $null = $parameters + Set-Content -Path $ENV:_runTestsTokenObservationPath -Value $ENV:_token -Encoding UTF8 + return $true + } + } + } + Mock Invoke-AlGoTestRun { + param($runTestsOverride) + return (& $runTestsOverride -parameters @{}) + } + + & $scriptPath -token $providedToken + + (Get-Content -Path $ENV:_runTestsTokenObservationPath -Raw).Trim() | Should -Be $providedToken + $ENV:_token | Should -Be $providedToken + } + + It 'Passes nested settings to AnalyzeRepo as recursive hashtables' { + $ENV:GITHUB_WORKSPACE = $TestDrive + $ENV:Settings = @{ + workspaceCompilation = @{ + enabled = $true + options = @(@{ name = 'nested-entry' }) + } + } | ConvertTo-Json -Depth 4 + $ENV:containerName = 'test-container' + $ENV:containerCredential = [Convert]::ToBase64String( + [Text.Encoding]::UTF8.GetBytes('{"username":"test-user","password":"test-password"}') + ) + + Mock DownloadAndImportBcContainerHelper {} + Mock AnalyzeRepo { + param($settings) + $settings | Should -BeOfType System.Collections.Hashtable + $settings.workspaceCompilation | Should -BeOfType System.Collections.Hashtable + $settings.workspaceCompilation.options[0] | Should -BeOfType System.Collections.Hashtable + $settings.workspaceCompilation.options[0].name | Should -Be 'nested-entry' + return @{ testFolders = @() } + } + Mock Get-ScriptOverrides { return @{} } + Mock Invoke-AlGoTestRun {} + + & $scriptPath + + Should -Invoke AnalyzeRepo -Times 1 -Exactly + } + + Context 'RunPipeline wiring' { + It 'Rejects a missing or blank kept container credential' -TestCases @( + @{ Value = $null } + @{ Value = '' } + @{ Value = ' ' } + ) { + param($Value) + $ENV:containerCredential = $Value + + { Get-TestRunnerCredential } | + Should -Throw '*RunPipeline-to-RunTests wiring error*container credential*not provided*' + } + + It 'Rejects malformed kept container credentials without exposing their value' -TestCases @( + @{ Value = 'credential-secret-value' } + @{ Value = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes('{invalid')) } + @{ Value = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes('{"username":"admin","password":""}')) } + ) { + param($Value) + $ENV:containerCredential = $Value + + try { + Get-TestRunnerCredential + throw 'Expected credential validation to fail.' + } + catch { + $_.Exception.Message | Should -BeLike '*RunPipeline-to-RunTests wiring error*container credential*expected format*' + $_.Exception.Message | Should -Not -Match [regex]::Escape($Value) + } + } + + It 'Creates the credential supplied by RunPipeline' { + $ENV:containerCredential = [Convert]::ToBase64String( + [Text.Encoding]::UTF8.GetBytes('{"username":"pipeline-user","password":"pipeline-password"}') + ) + + $credential = Get-TestRunnerCredential + + $credential.UserName | Should -Be 'pipeline-user' + $credential.GetNetworkCredential().Password | Should -Be 'pipeline-password' + } + + It 'Rejects a missing or blank kept container name' -TestCases @( + @{ Value = $null } + @{ Value = '' } + @{ Value = ' ' } + ) { + param($Value) + $ENV:containerName = $Value + + { Get-TestRunnerContainerName } | + Should -Throw '*RunPipeline-to-RunTests wiring error*container name*not provided*' + } + + It 'Uses the kept container name supplied by RunPipeline' { + $ENV:containerName = 'pipeline-container' + + Get-TestRunnerContainerName | Should -Be 'pipeline-container' + } + } + + # Call action + +} diff --git a/Tests/RunTests.Test.ps1 b/Tests/RunTests.Test.ps1 new file mode 100644 index 0000000000..725c7057db --- /dev/null +++ b/Tests/RunTests.Test.ps1 @@ -0,0 +1,1046 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'Mock/callback parameters must match function signatures')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', Justification = 'Test-only credential')] +param() + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 + +. (Join-Path -Path $PSScriptRoot -ChildPath "../Actions/AL-Go-Helper.ps1" -Resolve) + +# Stub for the BcContainerHelper function so it can be mocked within the module scope +function Get-AppJsonFromAppFile { param($appFile) } +function Get-BcContainerEventLog { param($containerName, [switch] $doNotOpen) } + +Import-Module (Join-Path $PSScriptRoot '../Actions/RunTests/RunTests.psm1' -Resolve) -DisableNameChecking -Force + +Describe 'RunTests.psm1 Tests' { + BeforeAll { + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'testCredential', Justification = 'Used in tests')] + $testCredential = New-Object System.Management.Automation.PSCredential("admin", (ConvertTo-SecureString "password" -AsPlainText -Force)) + $script:compiledAppMetadataByPath = @{} + $script:testFoldersByProject = @{} + + function New-TestProject { + Param( + [string[]] $CompiledTestApps = @(), + [hashtable] $CompiledAppIds = @{} + ) + $projectPath = Join-Path ([System.IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + $testAppsFolder = Join-Path (Join-Path $projectPath ".buildartifacts") "TestApps" + New-Item -Path $testAppsFolder -ItemType Directory -Force | Out-Null + $testFolders = @() + $index = 0 + foreach ($app in $CompiledTestApps) { + $index++ + $appPath = Join-Path $testAppsFolder $app + New-Item -Path $appPath -ItemType File -Force | Out-Null + $appId = if ($CompiledAppIds.ContainsKey($app)) { "$($CompiledAppIds[$app])" } else { [Guid]::NewGuid().ToString() } + $appName = [System.IO.Path]::GetFileNameWithoutExtension($app) + $script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($appPath)] = [PSCustomObject]@{ + id = $appId + name = $appName + } + + $testFolder = "TestApp$index" + $testFolderPath = Join-Path $projectPath $testFolder + New-Item -Path $testFolderPath -ItemType Directory -Force | Out-Null + @{ id = $appId; name = $appName } | ConvertTo-Json | + Set-Content -Path (Join-Path $testFolderPath "app.json") -Encoding UTF8 + $testFolders += $testFolder + } + $script:testFoldersByProject[$projectPath] = @($testFolders) + return $projectPath + } + + function Get-TestFoldersForProject { + Param( + [string] $ProjectPath + ) + + return @($script:testFoldersByProject[$ProjectPath]) + } + } + + BeforeEach { + $script:eventLogSource = Join-Path $TestDrive "$([Guid]::NewGuid()).evtx" + Set-Content -Path $script:eventLogSource -Value 'post-test-events' -Encoding UTF8 + Mock -ModuleName RunTests Get-AppJsonFromAppFile { + $fullPath = [System.IO.Path]::GetFullPath("$appFile") + if (-not $script:compiledAppMetadataByPath.ContainsKey($fullPath)) { + throw "No test metadata configured for '$appFile'." + } + return $script:compiledAppMetadataByPath[$fullPath] + } + Mock -ModuleName RunTests Get-BcContainerEventLog { return $script:eventLogSource } + Mock -ModuleName RunTests Install-AlTool { return '1.2.3' } + } + + Context 'Get-TestAppsToRun' { + It 'Collects compiled test apps from the build artifacts folder' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $settings = @{ + runTestsInAllInstalledTestApps = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + $testApps = @(Get-TestAppsToRun -settings $settings -projectPath $projectPath) + + $testApps.Count | Should -Be 2 + $testApps | ForEach-Object { $_ | Should -BeOfType System.Management.Automation.PSCustomObject } + @($testApps.Path | ForEach-Object { [System.IO.Path]::GetFileName($_) }) | + Should -Be @('App1.Test.app', 'App2.Test.app') + @($testApps.Name) | Should -Be @('App1.Test', 'App2.Test') + @($testApps.Id) | Should -Be @( + "$($script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($testApps[0].Path)].id)", + "$($script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($testApps[1].Path)].id)" + ) + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Includes installed test apps (unwrapping parentheses) when runTestsInAllInstalledTestApps is set' { + $projectPath = New-TestProject + $installedApp1 = Join-Path $projectPath 'Installed1.app' + $installedApp2 = Join-Path $projectPath 'Installed2.app' + New-Item -Path $installedApp1 -ItemType File -Force | Out-Null + New-Item -Path $installedApp2 -ItemType File -Force | Out-Null + $installedApp1Id = [Guid]::NewGuid().ToString() + $installedApp2Id = [Guid]::NewGuid().ToString() + $script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($installedApp1)] = [PSCustomObject]@{ + id = $installedApp1Id; name = 'Installed1' + } + $script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($installedApp2)] = [PSCustomObject]@{ + id = $installedApp2Id; name = 'Installed2' + } + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($installedApp1, "($installedApp2)") | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true; testFolders = @() } + $testApps = @(Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson) + + $testApps.Count | Should -Be 2 + @($testApps.Path) | Should -Be @($installedApp1, $installedApp2) + @($testApps.Id) | Should -Be @($installedApp1Id, $installedApp2Id) + @($testApps.Name) | Should -Be @('Installed1', 'Installed2') + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Ignores installed test apps when runTestsInAllInstalledTestApps is not set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $installedApp = Join-Path $projectPath 'Installed1.app' + New-Item -Path $installedApp -ItemType File -Force | Out-Null + $script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($installedApp)] = [PSCustomObject]@{ + id = [Guid]::NewGuid().ToString(); name = 'Installed1' + } + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($installedApp) | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ + runTestsInAllInstalledTestApps = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + $testApps = @(Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson) + + $testApps.Count | Should -Be 1 + [System.IO.Path]::GetFileName($testApps[0].Path) | Should -Be 'App1.Test.app' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not throw and returns compiled test apps when installTestAppsJson is an empty array' { + # Regression: in Windows PowerShell 5.1 ConvertFrom-Json emits a JSON array as a single + # object, so an empty '[]' previously surfaced as a one-element System.Object[] and threw + # "does not contain a method named 'TrimStart'". A test project with no installed test apps + # (the common case) must still return its compiled test apps without throwing. + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @() | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ + runTestsInAllInstalledTestApps = $true + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + $testApps = @(Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson) + + $testApps.Count | Should -Be 1 + [System.IO.Path]::GetFileName($testApps[0].Path) | Should -Be 'App1.Test.app' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Includes a single installed test app when runTestsInAllInstalledTestApps is set' { + # Regression: a single-element JSON array must enumerate to the string element (not the + # whole array) on both Windows PowerShell 5.1 and PowerShell 7. + $projectPath = New-TestProject + $installedApp = Join-Path $projectPath 'Installed1.app' + New-Item -Path $installedApp -ItemType File -Force | Out-Null + $installedAppId = [Guid]::NewGuid().ToString() + $script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($installedApp)] = [PSCustomObject]@{ + id = $installedAppId; name = 'Installed1' + } + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @("($installedApp)") | Set-Content -Path $installJson -Encoding UTF8 + + $settings = @{ runTestsInAllInstalledTestApps = $true; testFolders = @() } + $testApps = @(Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson) + + $testApps.Count | Should -Be 1 + $testApps[0].Path | Should -Be $installedApp + $testApps[0].Id | Should -Be $installedAppId + $testApps[0].Name | Should -Be 'Installed1' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Fails when an installed test app listed by RunPipeline is missing' { + $projectPath = New-TestProject + $missingApp = Join-Path $projectPath 'Missing.Test.app' + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($missingApp) | Set-Content -Path $installJson -Encoding UTF8 + $settings = @{ runTestsInAllInstalledTestApps = $true; testFolders = @() } + + { Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson } | + Should -Throw "*Failed to read installed test app metadata*$missingApp*" + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Reports malformed installed test app JSON with handoff context' { + $projectPath = New-TestProject + $installJson = Join-Path $projectPath 'installTestApps.json' + Set-Content -Path $installJson -Value '{invalid' -Encoding UTF8 + $settings = @{ runTestsInAllInstalledTestApps = $true; testFolders = @() } + + { + Get-TestAppsToRun -settings $settings -projectPath $projectPath ` + -installTestAppsJson $installJson + } | Should -Throw "*Failed to parse JSON file*$installJson*" + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Selects a compiled app only once when the installed list contains the same path' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $compiledAppPath = Join-Path (Join-Path (Join-Path $projectPath '.buildartifacts') 'TestApps') 'App1.Test.app' + $installJson = Join-Path $projectPath 'installTestApps.json' + ConvertTo-Json @($compiledAppPath) | Set-Content -Path $installJson -Encoding UTF8 + $settings = @{ + runTestsInAllInstalledTestApps = $true + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + $testApps = @(Get-TestAppsToRun -settings $settings -projectPath $projectPath -installTestAppsJson $installJson) + + $testApps.Count | Should -Be 1 + $testApps[0].Path | Should -Be $compiledAppPath + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Selects only compiled apps whose IDs belong to normal test folders' { + $projectPath = New-TestProject -CompiledTestApps @('Normal.Test.app', 'Performance.Test.app') + $testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + $settings = @{ + runTestsInAllInstalledTestApps = $false + testFolders = @($testFolders[0]) + bcptTestFolders = @($testFolders[1]) + } + + $testApps = @(Get-TestAppsToRun -settings $settings -projectPath $projectPath) + + $testApps.Count | Should -Be 1 + [System.IO.Path]::GetFileName($testApps[0].Path) | Should -Be 'Normal.Test.app' + $testApps[0].Name | Should -Be 'Normal.Test' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Reports corrupt normal source app metadata clearly' { + $projectPath = New-TestProject + $testFolder = Join-Path $projectPath 'BrokenTestApp' + New-Item -Path $testFolder -ItemType Directory -Force | Out-Null + Set-Content -Path (Join-Path $testFolder 'app.json') -Value '{invalid' -Encoding UTF8 + $settings = @{ runTestsInAllInstalledTestApps = $false; testFolders = @('BrokenTestApp') } + + { Get-TestAppsToRun -settings $settings -projectPath $projectPath } | + Should -Throw "*Failed to read normal test app metadata*BrokenTestApp*app.json*" + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Reports unreadable compiled app metadata clearly' { + $projectPath = New-TestProject -CompiledTestApps @('Broken.Test.app') + Mock -ModuleName RunTests Get-AppJsonFromAppFile { throw 'corrupt package' } + $settings = @{ + runTestsInAllInstalledTestApps = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Get-TestAppsToRun -settings $settings -projectPath $projectPath } | + Should -Throw "*Failed to read compiled test app metadata*Broken.Test.app*corrupt package*" + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Reports invalid selected compiled app metadata clearly' { + $projectPath = New-TestProject -CompiledTestApps @('Broken.Test.app') + $compiledAppPath = Join-Path (Join-Path (Join-Path $projectPath '.buildartifacts') 'TestApps') 'Broken.Test.app' + $script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($compiledAppPath)].name = '' + $settings = @{ + runTestsInAllInstalledTestApps = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Get-TestAppsToRun -settings $settings -projectPath $projectPath } | + Should -Throw "*Failed to read compiled test app metadata*Broken.Test.app*app name*" + + Remove-Item -Path $projectPath -Recurse -Force + } + } + + Context 'Invoke-AlGoTestRun' { + It 'Does not run tests when there are no test apps' { + $projectPath = New-TestProject + $script:runnerCalls = 0 + Mock -ModuleName RunTests Invoke-AlToolTestRun { $script:runnerCalls++; return $true } + $settings = @{ doNotRunTests = $false; runTestsInAllInstalledTestApps = $false; companyName = ''; treatTestFailuresAsWarnings = $false; testFolders = @() } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential + + $script:runnerCalls | Should -Be 0 + Should -Invoke -ModuleName RunTests Install-AlTool -Times 0 -Exactly + Test-Path (Join-Path $projectPath 'TestResults.xml') | Should -BeFalse + Test-Path (Join-Path (Join-Path $projectPath '.buildartifacts') 'TestResults.xml') | Should -BeFalse + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Copies passing test results to build artifacts and preserves the root result' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $script:runnerCalls = 0 + $script:capturedOverrideKeys = @() + $script:resultContent = '' + $override = { + param($parameters) + $script:runnerCalls++ + $script:capturedOverrideKeys = @($parameters.Keys) + Set-Content -Path $parameters.JUnitResultFileName -Value $script:resultContent -Encoding UTF8 + return $true + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Not -Throw + + $script:runnerCalls | Should -Be 2 + Should -Invoke -ModuleName RunTests Install-AlTool -Times 0 -Exactly + @($script:capturedOverrideKeys | Sort-Object) | Should -Be @( + 'AppendToJUnitResultFile', + 'appName', + 'companyName', + 'containerName', + 'credential', + 'detailed', + 'disabledTests', + 'extensionId', + 'GitHubActions', + 'JUnitResultFileName', + 'returnTrueIfAllPassed' + ) + $rootResult = Join-Path $projectPath 'TestResults.xml' + $artifactResult = Join-Path (Join-Path $projectPath '.buildartifacts') 'TestResults.xml' + (Get-Content -Path $rootResult -Raw -Encoding UTF8).Trim() | Should -Be $script:resultContent + (Get-Content -Path $artifactResult -Raw -Encoding UTF8).Trim() | Should -Be $script:resultContent + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes an override-specific testType without built-in validation' { + Mock -ModuleName RunTests Invoke-AlToolTestRun { + throw 'The built-in runner should not be called' + } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedTestType = $null + $override = { + param($parameters) + $script:capturedTestType = $parameters.testType + return $true + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + testType = 'Legacy' + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' ` + -credential $testCredential -runTestsOverride $override + + $script:capturedTestType | Should -Be 'Legacy' + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 0 -Exactly + Should -Invoke -ModuleName RunTests Install-AlTool -Times 0 -Exactly + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not create an artifact when test execution produces no result file' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $artifactResult = Join-Path (Join-Path $projectPath '.buildartifacts') 'TestResults.xml' + Set-Content -Path $artifactResult -Value '' -Encoding UTF8 + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride { return $true } + + Test-Path (Join-Path $projectPath 'TestResults.xml') | Should -BeFalse + Test-Path $artifactResult | Should -BeFalse + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Invokes only the normal test app when compiled artifacts also contain a BCPT app' { + $projectPath = New-TestProject -CompiledTestApps @('Normal.Test.app', 'Performance.Test.app') + $testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + $script:invokedExtensionIds = @() + $override = { + param($parameters) + $script:invokedExtensionIds += "$($parameters.extensionId)" + return $true + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @($testFolders[0]) + bcptTestFolders = @($testFolders[1]) + } + $normalAppPath = Join-Path (Join-Path (Join-Path $projectPath '.buildartifacts') 'TestApps') 'Normal.Test.app' + $normalAppId = "$($script:compiledAppMetadataByPath[[System.IO.Path]::GetFullPath($normalAppPath)].id)" + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:invokedExtensionIds | Should -Be @($normalAppId) + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Uses selected app metadata without reading the compiled app again during execution' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' ` + -credential $testCredential -runTestsOverride { return $true } + + Should -Invoke -ModuleName RunTests Get-AppJsonFromAppFile -Times 1 -Exactly + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes nested and project-wide disabled tests to an override as recursive hashtables' { + $appId = [Guid]::NewGuid().ToString() + $otherAppId = [Guid]::NewGuid().ToString() + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') -CompiledAppIds @{ 'App1.Test.app' = $appId } + + $testFolder = Join-Path $projectPath 'TestApp' + $nestedFolder = Join-Path $testFolder 'Nested' + New-Item -Path $nestedFolder -ItemType Directory -Force | Out-Null + @{ id = $appId } | ConvertTo-Json | Set-Content -Path (Join-Path $testFolder 'app.json') -Encoding UTF8 + @( + @{ + codeunitName = 'Nested Tests' + method = @('TestOne') + metadata = @{ source = 'nested'; issue = @{ id = 123 } } + } + ) | ConvertTo-Json -Depth 5 | Set-Content -Path (Join-Path $nestedFolder 'disabledTests.json') -Encoding UTF8 + + $otherTestFolder = Join-Path $projectPath 'OtherTestApp' + New-Item -Path $otherTestFolder -ItemType Directory -Force | Out-Null + @{ id = $otherAppId } | ConvertTo-Json | Set-Content -Path (Join-Path $otherTestFolder 'app.json') -Encoding UTF8 + @(@{ codeunitName = 'Other Tests'; method = 'Ignored' }) | ConvertTo-Json | Set-Content -Path (Join-Path $otherTestFolder 'disabledTests.json') -Encoding UTF8 + + $projectSettingsFolder = Join-Path $projectPath '.AL-Go' + New-Item -Path $projectSettingsFolder -ItemType Directory -Force | Out-Null + @( + @{ codeunitName = 'Project Tests'; method = 'TestTwo'; metadata = @{ source = 'project-wide' } } + ) | ConvertTo-Json -Depth 3 | Set-Content -Path (Join-Path $projectSettingsFolder "$appId.disabledTests.json") -Encoding UTF8 + + $script:capturedDisabledTests = @() + $override = { param($parameters) $script:capturedDisabledTests = @($parameters.disabledTests); return $true } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @('TestApp', 'OtherTestApp') + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedDisabledTests.Count | Should -Be 2 + @($script:capturedDisabledTests.codeunitName | Sort-Object) | + Should -Be @('Nested Tests', 'Project Tests') + $nestedDisabledTest = $script:capturedDisabledTests | Where-Object { $_.codeunitName -eq 'Nested Tests' } + $nestedDisabledTest | Should -BeOfType System.Collections.Hashtable + $nestedDisabledTest.metadata | Should -BeOfType System.Collections.Hashtable + $nestedDisabledTest.metadata.issue | Should -BeOfType System.Collections.Hashtable + $nestedDisabledTest.metadata.issue.id | Should -Be 123 + $nestedDisabledTest.metadata.source | Should -Be 'nested' + @($nestedDisabledTest.method) | Should -Be @('TestOne') + $projectDisabledTest = $script:capturedDisabledTests | Where-Object { $_.codeunitName -eq 'Project Tests' } + $projectDisabledTest.metadata.source | Should -Be 'project-wide' + $projectDisabledTest.method | Should -Be 'TestTwo' + $script:capturedDisabledTests.codeunitName | Should -Not -Contain 'Other Tests' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Fails with the disabled tests file path when its JSON is invalid' { + $appId = [Guid]::NewGuid().ToString() + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') -CompiledAppIds @{ 'App1.Test.app' = $appId } + $testFolder = Join-Path $projectPath 'TestApp' + New-Item -Path $testFolder -ItemType Directory -Force | Out-Null + @{ id = $appId } | ConvertTo-Json | Set-Content -Path (Join-Path $testFolder 'app.json') -Encoding UTF8 + Set-Content -Path (Join-Path $testFolder 'disabledTests.json') -Value '{invalid' -Encoding UTF8 + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @('TestApp') + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride { return $true } } | + Should -Throw '*disabledTests.json*' + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Copies failed test results before throwing when treatTestFailuresAsWarnings is not set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:resultContent = '' + $override = { + param($parameters) + Set-Content -Path $parameters.JUnitResultFileName -Value $script:resultContent -Encoding UTF8 + return $false + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | + Should -Throw '*There are test failures*' + + $rootResult = Join-Path $projectPath 'TestResults.xml' + $artifactResult = Join-Path (Join-Path $projectPath '.buildartifacts') 'TestResults.xml' + (Get-Content -Path $rootResult -Raw -Encoding UTF8).Trim() | Should -Be $script:resultContent + (Get-Content -Path $artifactResult -Raw -Encoding UTF8).Trim() | Should -Be $script:resultContent + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Copies failed test results when treatTestFailuresAsWarnings is set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:resultContent = '' + $override = { + param($parameters) + Set-Content -Path $parameters.JUnitResultFileName -Value $script:resultContent -Encoding UTF8 + return $false + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $true + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | Should -Not -Throw + + $rootResult = Join-Path $projectPath 'TestResults.xml' + $artifactResult = Join-Path (Join-Path $projectPath '.buildartifacts') 'TestResults.xml' + (Get-Content -Path $rootResult -Raw -Encoding UTF8).Trim() | Should -Be $script:resultContent + (Get-Content -Path $artifactResult -Raw -Encoding UTF8).Trim() | Should -Be $script:resultContent + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Copies partial test results before rethrowing a later runner error' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $script:runnerCalls = 0 + $script:resultContent = '' + $override = { + param($parameters) + $script:runnerCalls++ + if ($script:runnerCalls -eq 1) { + Set-Content -Path $parameters.JUnitResultFileName -Value $script:resultContent -Encoding UTF8 + return $true + } + throw 'second app runner failed' + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | + Should -Throw '*second app runner failed*' + + $artifactResult = Join-Path (Join-Path $projectPath '.buildartifacts') 'TestResults.xml' + (Get-Content -Path $artifactResult -Raw -Encoding UTF8).Trim() | Should -Be $script:resultContent + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Preserves a later runner error when copying partial results also fails' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $script:runnerCalls = 0 + $script:resultContent = '' + $script:partialArtifactResult = Join-Path (Join-Path $projectPath '.buildartifacts') 'TestResults.xml' + $override = { + param($parameters) + $script:runnerCalls++ + if ($script:runnerCalls -eq 1) { + Set-Content -Path $parameters.JUnitResultFileName -Value $script:resultContent -Encoding UTF8 + return $true + } + throw 'second app runner failed' + } + Mock -ModuleName RunTests Copy-Item { throw 'partial copy blocked' } -ParameterFilter { + $Destination -eq $script:partialArtifactResult + } + Mock -ModuleName RunTests OutputWarning {} + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | + Should -Throw '*second app runner failed*' + + Test-Path $script:partialArtifactResult | Should -BeFalse + Should -Invoke -ModuleName RunTests OutputWarning -Times 1 -Exactly -ParameterFilter { + $message -like '*test results could not be copied to build artifacts*partial copy blocked*' + } + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Surfaces artifact copy errors before a test failure message' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $override = { + param($parameters) + Set-Content -Path $parameters.JUnitResultFileName -Value '' -Encoding UTF8 + return $false + } + Mock -ModuleName RunTests Copy-Item { throw 'copy blocked' } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override } | + Should -Throw '*Failed to copy test results*copy blocked*' + + Should -Invoke -ModuleName RunTests Get-BcContainerEventLog -Times 1 -Exactly + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes GitHubActions severity error when treatTestFailuresAsWarnings is not set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedSeverity = $null + $override = { param($parameters) $script:capturedSeverity = $parameters.GitHubActions; return $true } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedSeverity | Should -Be 'error' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes GitHubActions severity warning when treatTestFailuresAsWarnings is set' { + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedSeverity = $null + $override = { param($parameters) $script:capturedSeverity = $parameters.GitHubActions; return $true } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $true + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedSeverity | Should -Be 'warning' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Builds a parameter set that is valid for the real Run-TestsInBcContainer cmdlet' { + # Guard against parameter drift: every key/value passed to the BcContainerHelper test + # runner is validated against the real cmdlet signature (parameter names and ValidateSet + # values). This catches invalid parameter names and out-of-set values locally instead of + # only surfacing them in CI, where the real cmdlet is actually invoked. + $compatibleModule = Get-Module -ListAvailable -Name BcContainerHelper | + Where-Object { $_.Version -ge [version] '6.1.9' } | + Sort-Object Version -Descending | + Select-Object -First 1 + if (-not $compatibleModule) { + Set-ItResult -Skipped -Because 'BcContainerHelper 6.1.9 or later is not available in this environment' + return + } + + $compatibleModulePath = Join-Path $compatibleModule.ModuleBase 'BcContainerHelper.psd1' + if (-not (Test-Path -LiteralPath $compatibleModulePath -PathType Leaf)) { + $compatibleModulePath = $compatibleModule.Path + } + $previousModulePaths = @( + foreach ($previousModule in @(Get-Module -Name BcContainerHelper)) { + $manifestPath = Join-Path $previousModule.ModuleBase "$($previousModule.Name).psd1" + if (Test-Path -LiteralPath $manifestPath -PathType Leaf) { + $manifestPath + } + else { + $previousModule.Path + } + } + ) + $loadedModule = Import-Module $compatibleModulePath -DisableNameChecking -Force -PassThru + try { + $command = Get-Command -Name 'Run-TestsInBcContainer' -Module $loadedModule.Name -ErrorAction Stop + if (($command -is [System.Management.Automation.AliasInfo]) -and $command.ResolvedCommand) { + $command = $command.ResolvedCommand + } + + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:capturedParams = $null + $override = { param($parameters) $script:capturedParams = $parameters; return $true } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential -runTestsOverride $override + + $script:capturedParams | Should -Not -BeNullOrEmpty + foreach ($key in $script:capturedParams.Keys) { + $parameter = $command.Parameters[$key] + $parameter | Should -Not -BeNullOrEmpty -Because "'$key' must be a real parameter of Run-TestsInBcContainer" + + $validateSet = $parameter.Attributes | + Where-Object { $_ -is [System.Management.Automation.ValidateSetAttribute] } | + Select-Object -First 1 + if ($validateSet) { + $validateSet.ValidValues | Should -Contain $script:capturedParams[$key] -Because "the value for '$key' must be one of its allowed ValidateSet values" + } + } + + Remove-Item -Path $projectPath -Recurse -Force + } + finally { + Remove-Module -ModuleInfo $loadedModule -Force -ErrorAction SilentlyContinue + foreach ($previousModulePath in $previousModulePaths) { + Import-Module $previousModulePath -DisableNameChecking -Force + } + } + } + } + + Context 'Invoke-AlGoTestRun (default AlTool runner)' { + It 'Runs the AlTool runner for every test app when no override is supplied' { + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $true } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app', 'App2.Test.app') + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Not -Throw + + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 2 -Exactly + Should -Invoke -ModuleName RunTests Install-AlTool -Times 1 -Exactly + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Maps the testType setting and other built-in runner parameters' { + $appId = [Guid]::NewGuid().ToString() + $script:capturedAlToolParams = $null + Mock -ModuleName RunTests Invoke-AlToolTestRun { + param( + $ContainerName, + [System.Management.Automation.PSCredential] $Credential, + $ExtensionId, + $AppName, + $CompanyName, + $Tenant, + $TestType, + [hashtable[]] $DisabledTests, + $JUnitResultFileName + ) + $script:capturedAlToolParams = @{ + Keys = @($PSBoundParameters.Keys) + ContainerName = $ContainerName + Credential = $Credential + ExtensionId = $ExtensionId + AppName = $AppName + CompanyName = $CompanyName + Tenant = $Tenant + TestType = $TestType + DisabledTests = @($DisabledTests) + JUnitResultFileName = $JUnitResultFileName + } + return $true + } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') -CompiledAppIds @{ 'App1.Test.app' = $appId } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = 'CRONUS' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + testType = 'IntegrationTest' + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'mycontainer' -credential $testCredential + + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 1 -Exactly + Should -Invoke -ModuleName RunTests Install-AlTool -Times 1 -Exactly + @($script:capturedAlToolParams.Keys | Sort-Object) | Should -Be @( + 'AppName', + 'CompanyName', + 'ContainerName', + 'Credential', + 'DisabledTests', + 'ExtensionId', + 'JUnitResultFileName', + 'Tenant', + 'TestType' + ) + $script:capturedAlToolParams.ContainerName | Should -Be 'mycontainer' + $script:capturedAlToolParams.Credential | Should -BeOfType [System.Management.Automation.PSCredential] + $script:capturedAlToolParams.ExtensionId | Should -Be $appId + $script:capturedAlToolParams.AppName | Should -Be 'App1.Test' + $script:capturedAlToolParams.CompanyName | Should -Be 'CRONUS' + $script:capturedAlToolParams.Tenant | Should -Be 'default' + $script:capturedAlToolParams.TestType | Should -Be 'IntegrationTest' + $script:capturedAlToolParams.DisabledTests.Count | Should -Be 0 + $script:capturedAlToolParams.JUnitResultFileName | Should -Be (Join-Path $projectPath 'TestResults.xml') + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Passes project-wide disabled tests to the AlTool runner' { + $appId = [Guid]::NewGuid().ToString() + $script:capturedAlToolParams = $null + Mock -ModuleName RunTests Invoke-AlToolTestRun { + param([hashtable[]] $DisabledTests) + $script:capturedAlToolParams = @($DisabledTests) + return $true + } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') -CompiledAppIds @{ 'App1.Test.app' = $appId } + @(@{ codeunitName = 'Project Tests'; method = 'TestOne' }) | ConvertTo-Json | + Set-Content -Path (Join-Path $projectPath "$appId.disabledTests.json") -Encoding UTF8 + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential + + Should -Invoke -ModuleName RunTests Invoke-AlToolTestRun -Times 1 -Exactly + $script:capturedAlToolParams.Count | Should -Be 1 + $script:capturedAlToolParams[0] | Should -BeOfType System.Collections.Hashtable + $script:capturedAlToolParams[0].codeunitName | Should -Be 'Project Tests' + $script:capturedAlToolParams[0].method | Should -Be 'TestOne' + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Throws when the AlTool runner reports failure and treatTestFailuresAsWarnings is not set' { + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $false } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Does not throw when the AlTool runner reports failure but treatTestFailuresAsWarnings is set' { + Mock -ModuleName RunTests Invoke-AlToolTestRun { return $false } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $true + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'test' -credential $testCredential } | Should -Not -Throw + + Remove-Item -Path $projectPath -Recurse -Force + } + } + + Context 'Invoke-AlGoTestRun event log lifecycle' { + It 'Captures after ' -TestCases @( + @{ Case = 'passing tests'; HasTestApp = $true; RunnerBehavior = 'Pass'; TreatAsWarning = $false; ExpectedError = $null; ExpectedRunnerCalls = 1 } + @{ Case = 'a hard test failure'; HasTestApp = $true; RunnerBehavior = 'Fail'; TreatAsWarning = $false; ExpectedError = 'There are test failures'; ExpectedRunnerCalls = 1 } + @{ Case = 'a warning-mode test failure'; HasTestApp = $true; RunnerBehavior = 'Fail'; TreatAsWarning = $true; ExpectedError = $null; ExpectedRunnerCalls = 1 } + @{ Case = 'a no-test run'; HasTestApp = $false; RunnerBehavior = 'Pass'; TreatAsWarning = $false; ExpectedError = $null; ExpectedRunnerCalls = 0 } + @{ Case = 'a runner exception'; HasTestApp = $true; RunnerBehavior = 'Throw'; TreatAsWarning = $false; ExpectedError = 'runner failed'; ExpectedRunnerCalls = 1 } + ) { + param($HasTestApp, $RunnerBehavior, $TreatAsWarning, $ExpectedError, $ExpectedRunnerCalls) + + Mock -ModuleName RunTests OutputWarning {} + $compiledTestApps = if ($HasTestApp) { @('App1.Test.app') } else { @() } + $projectPath = New-TestProject -CompiledTestApps $compiledTestApps + $eventLogDestination = Join-Path $projectPath 'ContainerEventLog.evtx' + Set-Content -Path $eventLogDestination -Value 'pre-test-events' -Encoding UTF8 + $script:invocationOrder = @() + $script:runnerCalls = 0 + $script:runnerBehavior = $RunnerBehavior + $runTestsOverride = { + param($parameters) + $null = $parameters + $script:runnerCalls++ + $script:invocationOrder += 'run' + switch ($script:runnerBehavior) { + 'Pass' { return $true } + 'Fail' { return $false } + 'Throw' { throw 'runner failed' } + } + } + Mock -ModuleName RunTests Get-BcContainerEventLog { + $script:invocationOrder += 'capture' + return $script:eventLogSource + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $TreatAsWarning + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + $actualError = $null + try { + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'kept-container' -credential $testCredential -runTestsOverride $runTestsOverride + } + catch { + $actualError = $_.Exception.Message + } + + if ($ExpectedError) { + $actualError | Should -BeLike "*$ExpectedError*" + } + else { + $actualError | Should -BeNullOrEmpty + } + $expectedOrder = if ($HasTestApp) { @('run', 'capture') } else { @('capture') } + $script:invocationOrder | Should -Be $expectedOrder + $script:runnerCalls | Should -Be $ExpectedRunnerCalls + Should -Invoke -ModuleName RunTests Get-BcContainerEventLog -Times 1 -Exactly -ParameterFilter { + $containerName -eq 'kept-container' -and $doNotOpen + } + (Get-Content -Path $eventLogDestination -Raw -Encoding UTF8).Trim() | Should -Be 'post-test-events' + Test-Path (Join-Path $projectPath 'TestResults.xml') | Should -BeFalse + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Warns and succeeds when event log capture fails after passing tests' { + Mock -ModuleName RunTests OutputWarning {} + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $eventLogDestination = Join-Path $projectPath 'ContainerEventLog.evtx' + Set-Content -Path $eventLogDestination -Value 'pre-test-events' -Encoding UTF8 + Mock -ModuleName RunTests Get-BcContainerEventLog { return (Join-Path $TestDrive 'missing.evtx') } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + { Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'kept-container' -credential $testCredential -runTestsOverride { return $true } } | + Should -Not -Throw + + (Get-Content -Path $eventLogDestination -Raw -Encoding UTF8).Trim() | Should -Be 'pre-test-events' + Should -Invoke -ModuleName RunTests OutputWarning -Times 1 -Exactly -ParameterFilter { + $message -like '*post-test container event log could not be captured*did not return a readable event log file*' + } + Remove-Item -Path $projectPath -Recurse -Force + } + + It 'Preserves when event log capture also fails' -TestCases @( + @{ RunnerBehavior = 'Fail'; ExpectedError = 'There are test failures.' } + @{ RunnerBehavior = 'Throw'; ExpectedError = 'runner failed' } + ) { + param($RunnerBehavior, $ExpectedError) + + Mock -ModuleName RunTests OutputWarning {} + Mock -ModuleName RunTests Get-BcContainerEventLog { throw 'event export failed' } + $projectPath = New-TestProject -CompiledTestApps @('App1.Test.app') + $script:runnerBehavior = $RunnerBehavior + $runTestsOverride = { + if ($script:runnerBehavior -eq 'Throw') { throw 'runner failed' } + return $false + } + $settings = @{ + doNotRunTests = $false + runTestsInAllInstalledTestApps = $false + companyName = '' + treatTestFailuresAsWarnings = $false + testFolders = @(Get-TestFoldersForProject -ProjectPath $projectPath) + } + + $actualError = $null + try { + Invoke-AlGoTestRun -settings $settings -projectPath $projectPath -containerName 'kept-container' ` + -credential $testCredential -runTestsOverride $runTestsOverride + } + catch { + $actualError = $_.Exception.Message + } + + $actualError | Should -Be $ExpectedError + Should -Invoke -ModuleName RunTests OutputWarning -Times 1 -Exactly -ParameterFilter { + $message -like '*post-test container event log could not be captured*event export failed*' + } + Remove-Item -Path $projectPath -Recurse -Force + } + } +}