From 4270d24ac85138f8ca87fc805bdf0fc43fc0e090 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sun, 11 Oct 2026 17:18:38 +0200 Subject: [PATCH 01/15] New-DbaAvailabilityGroup - Check the last progress record of each Id in the pipeline stop test The test counted an Id as completed when any completed record of it existed. Windows PowerShell writes a completed record with Id 0 of its own while it loads modules, so an open bar of the command could pass the check. The test now waits for a record of the command itself, checks the last record of each Id and imports the manifest instead of the psm1, as the progress tests of #10773-#10775 do. Red on the command before #10764 and green on development, on 5.1 and 7.6. Refs #10776 (do New-DbaAvailabilityGroup) Co-Authored-By: Claude Opus 5.5 --- tests/New-DbaAvailabilityGroup.Tests.ps1 | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/New-DbaAvailabilityGroup.Tests.ps1 b/tests/New-DbaAvailabilityGroup.Tests.ps1 index 0f61cc03dea..6a3d87c2915 100644 --- a/tests/New-DbaAvailabilityGroup.Tests.ps1 +++ b/tests/New-DbaAvailabilityGroup.Tests.ps1 @@ -159,6 +159,9 @@ Describe $CommandName -Tag IntegrationTests { # Write-Progress puts every record into Streams.Progress, so a bar that was never completed shows # as an Id with records but without a completed one. The pipeline is stopped as soon as the first # record arrives, which is what Ctrl+C does. WhatIf keeps the instance unchanged. + # It waits for a record of the command itself: Windows PowerShell writes progress records of its own + # while it loads modules. The runspace imports the manifest: an import of the psm1 without a command + # line skips the type data. $splatStopAg = @{ Primary = $TestConfig.InstanceHadr Name = $agName @@ -174,7 +177,8 @@ Describe $CommandName -Tag IntegrationTests { $stopRunspace.Open() $importShell = [powershell]::Create() $importShell.Runspace = $stopRunspace - $null = $importShell.AddCommand("Import-Module").AddParameter("Name", (Get-Module -Name $ModuleName | Select-Object -First 1).Path).Invoke() + $manifestPath = Join-Path -Path (Get-Module -Name $ModuleName | Select-Object -First 1).ModuleBase -ChildPath "$ModuleName.psd1" + $null = $importShell.AddCommand("Import-Module").AddParameter("Name", $manifestPath).Invoke() $importShell.Dispose() $stopShell = [powershell]::Create() @@ -182,7 +186,7 @@ Describe $CommandName -Tag IntegrationTests { $null = $stopShell.AddCommand("New-DbaAvailabilityGroup").AddParameters($splatStopAg) $stopAsync = $stopShell.BeginInvoke() $stopWatch = [System.Diagnostics.Stopwatch]::StartNew() - while ($stopShell.Streams.Progress.Count -eq 0 -and -not $stopAsync.IsCompleted -and $stopWatch.Elapsed.TotalSeconds -lt 60) { + while (-not ($stopShell.Streams.Progress | Where-Object Activity -eq "Adding new availability group") -and -not $stopAsync.IsCompleted -and $stopWatch.Elapsed.TotalSeconds -lt 60) { Start-Sleep -Milliseconds 10 } $stopShell.Stop() @@ -197,9 +201,10 @@ Describe $CommandName -Tag IntegrationTests { } It "Completes its progress bar" { - $completedIds = @($stopRecords | Where-Object RecordType -eq "Completed" | Select-Object -ExpandProperty ActivityId -Unique) - $openIds = $stopRecords | Where-Object RecordType -eq "Processing" | Select-Object -ExpandProperty ActivityId -Unique | Where-Object { $PSItem -notin $completedIds } - $stopRecords | Should -Not -BeNullOrEmpty + # An Id stays on screen when its last record is not a completed one. Windows PowerShell completes its + # own bar for loading modules with Id 0 as well, so a completed record somewhere is not enough. + $openIds = $stopRecords | Group-Object -Property ActivityId | Where-Object { @($PSItem.Group)[-1].RecordType -ne "Completed" } | Select-Object -ExpandProperty Name + $stopRecords | Where-Object Activity -eq "Adding new availability group" | Should -Not -BeNullOrEmpty $openIds | Should -BeNullOrEmpty } } From e2b5e3d80b913e7d26df70ad23f2c7f53dd45333 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:16:14 +0200 Subject: [PATCH 02/15] Sync-DbaAvailabilityGroup - Close the dedicated admin connection also when a copy command stops The dedicated admin connection opened for Copy-DbaCredential, Copy-DbaDbMail and Copy-DbaLinkedServer was closed only after the three commands. A throw, Ctrl+C or Select-Object -First in between left it open, and SQL Server allows only one dedicated admin connection per instance. The finally that completes the progress bar now closes it too, when the command opened it itself. Part of #10776. (do Sync-DbaAvailabilityGroup) Co-Authored-By: Claude Opus 5.5 --- public/Sync-DbaAvailabilityGroup.ps1 | 9 ++- tests/Sync-DbaAvailabilityGroup.Tests.ps1 | 68 +++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/public/Sync-DbaAvailabilityGroup.ps1 b/public/Sync-DbaAvailabilityGroup.ps1 index 62dca910b30..9bf06af575c 100644 --- a/public/Sync-DbaAvailabilityGroup.ps1 +++ b/public/Sync-DbaAvailabilityGroup.ps1 @@ -298,6 +298,8 @@ function Sync-DbaAvailabilityGroup { $primaryserver = $server.Name $secondaryservers = $secondaries.Name -join ", " + # Set before the try, so the finally knows whether a dedicated admin connection of ours is still open. + $dacOpened = $false try { if ($Exclude -notcontains "SpConfigure") { Write-ProgressHelper -Activity $activity -StepNumber ($stepCounter++) -Message "Syncing SQL Server Configuration" @@ -328,7 +330,6 @@ function Sync-DbaAvailabilityGroup { $dacNeeded = -not $ExcludePassword -and ($Exclude -notcontains "Credentials" -or $Exclude -notcontains "DatabaseMail" -or $Exclude -notcontains "LinkedServers") # We open the DAC as late as possible and close it as early as possible, because SQL Server only allows one DAC per instance. - $dacOpened = $false if ($dacNeeded -and -not $serverDac) { Write-Message -Level Verbose -Message "Opening dedicated admin connection for password retrieval." $serverDac = Connect-DbaInstance -SqlInstance $server -SqlCredential $PrimarySqlCredential -DedicatedAdminConnection -WarningAction SilentlyContinue @@ -364,6 +365,7 @@ function Sync-DbaAvailabilityGroup { # Disconnect is important because it is a DAC, but only if we opened it ourselves. if ($dacOpened) { $null = $serverDac | Disconnect-DbaInstance -WhatIf:$false + $dacOpened = $false } if ($Exclude -notcontains "SystemTriggers") { @@ -450,6 +452,11 @@ function Sync-DbaAvailabilityGroup { } } finally { Write-ProgressHelper -Activity $activity -Completed + # A throw or a stopped pipeline in one of the commands that copy passwords skips the disconnect above. + # SQL Server allows only one dedicated admin connection per instance, so it must not stay open. + if ($dacOpened) { + $null = $serverDac | Disconnect-DbaInstance -WhatIf:$false + } } } } diff --git a/tests/Sync-DbaAvailabilityGroup.Tests.ps1 b/tests/Sync-DbaAvailabilityGroup.Tests.ps1 index 3250572a56c..871802322b2 100644 --- a/tests/Sync-DbaAvailabilityGroup.Tests.ps1 +++ b/tests/Sync-DbaAvailabilityGroup.Tests.ps1 @@ -583,4 +583,72 @@ WHERE j.name = @jobName AND s.step_id = 1" $openIds | Should -BeNullOrEmpty } } + + Context "When the pipeline stops while the dedicated admin connection is open" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $dacCredentialName = "dbatoolsci_syncdac_$(Get-Random)" + $splatDacCredential = @{ + SqlInstance = $TestConfig.InstanceCopy1 + Name = $dacCredentialName + Identity = $dacCredentialName + SecurePassword = ConvertTo-SecureString "dbatools.IO" -AsPlainText -Force + } + $null = New-DbaCredential @splatDacCredential + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + # Only the credentials are synced, so the dedicated admin connection is opened for Copy-DbaCredential, and + # its first result is the first output of the command. Select-Object -First 1 stops the pipeline right + # there, while the connection is open. The disconnect used to run only after the three commands that copy + # passwords, so it stayed open, and SQL Server allows only one dedicated admin connection per instance. + $splatDacSync = @{ + Primary = $TestConfig.InstanceCopy1 + Secondary = $TestConfig.InstanceCopy2 + Exclude = @( + "AgentAlert", + "AgentCategory", + "AgentJob", + "AgentOperator", + "AgentProxy", + "AgentSchedule", + "CustomErrors", + "DatabaseMail", + "DatabaseOwner", + "LinkedServers", + "LoginPermissions", + "Logins", + "SpConfigure", + "SystemTriggers" + ) + } + $dacSessionQuery = @" +SELECT COUNT(*) AS DacSessions +FROM sys.dm_exec_sessions AS s +INNER JOIN sys.endpoints AS e ON e.endpoint_id = s.endpoint_id +WHERE e.is_admin_endpoint = 1 +"@ + } + + AfterAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $null = Get-DbaCredential -SqlInstance $TestConfig.InstanceCopy1, $TestConfig.InstanceCopy2 -Credential $dacCredentialName | Remove-DbaCredential + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + It "Closes the dedicated admin connection" { + $firstResult = Sync-DbaAvailabilityGroup @splatDacSync | Select-Object -First 1 + $firstResult.Name | Should -Be $dacCredentialName + $WarnVar | Should -BeNullOrEmpty + + # The session can take a moment to end on the server after the client closed the connection. + $dacWatch = [System.Diagnostics.Stopwatch]::StartNew() + do { + $dacSessions = (Invoke-DbaQuery -SqlInstance $TestConfig.InstanceCopy1 -Query $dacSessionQuery).DacSessions + if ($dacSessions -gt 0) { + Start-Sleep -Milliseconds 200 + } + } while ($dacSessions -gt 0 -and $dacWatch.Elapsed.TotalSeconds -lt 5) + $dacSessions | Should -Be 0 + } + } } From 0b8ffb9b6cd7175007c3603830340b90ebdc7cd0 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:19:27 +0200 Subject: [PATCH 03/15] Invoke-DbaDbDataGenerator - Close the connection for the inserts also when the command stops early The copy of the connection that the inserts use was closed after the try/finally around the database loop. The returns inside it (failed read of the existing unique values, no unique value found after 100 tries, no unique values in the dictionary), a throw under EnableException and a stopped pipeline skipped the close and left one session open per call. The close moved into the finally; a failing close warns there instead of calling Stop-Function, because a throw from a finally would replace the error on its way out. Part of #10776. (do Invoke-DbaDbDataGenerator) Co-Authored-By: Claude Opus 5.5 --- public/Invoke-DbaDbDataGenerator.ps1 | 14 ++-- tests/Invoke-DbaDbDataGenerator.Tests.ps1 | 80 +++++++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/public/Invoke-DbaDbDataGenerator.ps1 b/public/Invoke-DbaDbDataGenerator.ps1 index 295e4d0c285..14f8a070196 100644 --- a/public/Invoke-DbaDbDataGenerator.ps1 +++ b/public/Invoke-DbaDbDataGenerator.ps1 @@ -595,12 +595,14 @@ function Invoke-DbaDbDataGenerator { } } finally { Write-ProgressHelper -Completed - } - - try { - $sqlconn.Close() - } catch { - Stop-Function -Message "Failure" -Continue -ErrorRecord $_ + # The returns above, a throw under EnableException and a stopped pipeline skip everything after this + # block, so the connection is closed here. A warning instead of Stop-Function, because a throw from a + # finally would replace the error that is on its way out. + try { + $sqlconn.Close() + } catch { + Write-Message -Level Warning -Message "Failure closing the connection to $instance" -ErrorRecord $_ + } } } } diff --git a/tests/Invoke-DbaDbDataGenerator.Tests.ps1 b/tests/Invoke-DbaDbDataGenerator.Tests.ps1 index 68c4f5820ac..60a5b7f926b 100644 --- a/tests/Invoke-DbaDbDataGenerator.Tests.ps1 +++ b/tests/Invoke-DbaDbDataGenerator.Tests.ps1 @@ -429,6 +429,86 @@ Describe $CommandName -Tag IntegrationTests { } } + Context "Config that asks for more unique values than the column can hold" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + # Five rows from a domain of two values: the third row collides on every try, and the command stops with + # a return. That return skipped the close of the connection it opened for the inserts, so every such call + # left one session open. The sessions are counted through a server object opened once, because a + # counting command would open connections of its own. + $null = Invoke-DbaQuery -SqlInstance $TestConfig.InstanceSingle -Database $generatorDb -Query "CREATE TABLE [dbo].[fewnumbers]([Nr] [int] NOT NULL); + CREATE UNIQUE INDEX [ix_fewnumbers_nr] ON [dbo].[fewnumbers]([Nr]);" + + $fewNumbersConfig = @" +{ + "Name": "$generatorDb", + "Type": "DataGenerationConfiguration", + "Tables": [ + { + "Name": "fewnumbers", + "Schema": "dbo", + "Columns": [ + { + "Name": "Nr", + "ColumnType": "int", + "CharacterString": null, + "MinValue": 1, + "MaxValue": 2, + "MaskingType": "Random", + "SubType": "Number", + "Identity": false, + "ForeignKey": false, + "Composite": false, + "Nullable": false + } + ], + "ResetIdentity": false, + "TruncateTable": false, + "HasUniqueIndex": true, + "Rows": 5 + } + ] +} +"@ + $fewNumbersConfigPath = "$backupPath\fewnumbers.json" + Set-Content -Path $fewNumbersConfigPath -Value $fewNumbersConfig + + $countServer = Connect-DbaInstance -SqlInstance $TestConfig.InstanceSingle -NonPooledConnection + $countQuery = "SELECT COUNT(*) FROM sys.dm_exec_sessions WHERE host_process_id = $PID" + + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + $splatFewNumbers = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $generatorDb + FilePath = $fewNumbersConfigPath + WarningAction = "SilentlyContinue" + } + # One warm-up call, so the pooled connections of a normal call exist before the baseline is taken. + $null = Invoke-DbaDbDataGenerator @splatFewNumbers + $sessionsBefore = $countServer.ConnectionContext.ExecuteScalar($countQuery) + foreach ($i in 1..3) { + $fewNumbersResult = Invoke-DbaDbDataGenerator @splatFewNumbers + } + $fewNumbersWarning = $WarnVar + $sessionsAfter = $countServer.ConnectionContext.ExecuteScalar($countQuery) + } + + AfterAll { + $countServer.ConnectionContext.Disconnect() + } + + It "Warns that it cannot generate the unique values and returns nothing" { + $fewNumbersResult | Should -BeNullOrEmpty + ($fewNumbersWarning -join " ") | Should -BeLike "*Could not generate a unique value*" + } + + It "Leaves no session behind" { + $sessionsAfter | Should -Be $sessionsBefore + } + } + Context "Config for a table with an identity primary key" { BeforeAll { $PSDefaultParameterValues["*-Dba*:EnableException"] = $true From 52c952f0aeabd80795f15c7637da021b261ab331 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:29:52 +0200 Subject: [PATCH 04/15] Invoke-DbaDbDataMasking - Clean up in finally blocks and stop writing SQL and $null to the pipeline - The identity column, the masking index and the unique values table that the command adds per table were removed only at the end of the table loop. Every Stop-Function -Continue after the identity column was added (failed action or composite update, failed index, -ExcludeColumn on a unique index), the "no columns to process" continue, a throw under EnableException and a stopped pipeline left them in the user's table. They are removed in a finally now, and a failed removal warns instead of continuing, so a failed index drop no longer skips the column drop. - tempdb.dbo.DeterministicValues is dropped in a finally around the work of the instance. - "You cannot use both composites and actions" was followed by masking the table anyway. The table is skipped now. - The UPDATE statements of the actions (on failure) and of the composites (on every run) were written to the pipeline, and the drop of a leftover DeterministicValues table wrote a $null. Part of #10776. (do Invoke-DbaDbDataMasking) Co-Authored-By: Claude Opus 5.5 --- public/Invoke-DbaDbDataMasking.ps1 | 1810 ++++++++++++----------- tests/Invoke-DbaDbDataMasking.Tests.ps1 | 222 +++ 2 files changed, 1133 insertions(+), 899 deletions(-) diff --git a/public/Invoke-DbaDbDataMasking.ps1 b/public/Invoke-DbaDbDataMasking.ps1 index 145c1a59d76..d3329e226bc 100644 --- a/public/Invoke-DbaDbDataMasking.ps1 +++ b/public/Invoke-DbaDbDataMasking.ps1 @@ -278,394 +278,666 @@ function Invoke-DbaDbDataMasking { Stop-Function -Message "Failure" -Category ConnectionError -ErrorRecord $_ -Target $instance -Continue } - # Check if the deterministic values table is already present - if ($server.Databases['tempdb'].Tables.Name -contains 'DeterministicValues') { - Write-Message -Level Verbose -Message "Deterministic values table already exists. Dropping it...." - $query = "DROP TABLE [dbo].[DeterministicValues];" - $server.Databases['tempdb'].Query($query) - } + # The finally drops the deterministic values table also when the command throws under EnableException or + # the pipeline stops early (Ctrl+C, Select-Object -First). + try { + # Check if the deterministic values table is already present + if ($server.Databases["tempdb"].Tables.Name -contains "DeterministicValues") { + Write-Message -Level Verbose -Message "Deterministic values table already exists. Dropping it...." + $query = "DROP TABLE [dbo].[DeterministicValues];" + # Query() returns a $null for a statement without a result set. + $null = $server.Databases["tempdb"].Query($query) + } - # Create the deterministic value table - $query = " - CREATE TABLE dbo.DeterministicValues - ( - [ValueKey] VARCHAR(900), - [NewValue] VARCHAR(900) - ) - - CREATE UNIQUE NONCLUSTERED INDEX UNX__DeterministicValues_ValueKey - ON dbo.DeterministicValues ( ValueKey ) - " - - $null = $server.Databases['tempdb'].Query($query) - - # Import the dictionary files - if ($DictionaryFilePath.Count -ge 1) { - foreach ($file in $DictionaryFilePath) { - Write-Message -Level Verbose -Message "Importing dictionary file '$file'" - if (Test-Path -Path $file) { - try { - # Import the keys and values - Import-DbaCsv -Path $file -SqlInstance $server -Database tempdb -Schema dbo -Table DeterministicValues - } catch { - Stop-Function -Message "Could not import csv data from file '$file'" -ErrorRecord $_ -Target $file + # Create the deterministic value table + $query = " + CREATE TABLE dbo.DeterministicValues + ( + [ValueKey] VARCHAR(900), + [NewValue] VARCHAR(900) + ) + + CREATE UNIQUE NONCLUSTERED INDEX UNX__DeterministicValues_ValueKey + ON dbo.DeterministicValues ( ValueKey ) + " + + $null = $server.Databases['tempdb'].Query($query) + + # Import the dictionary files + if ($DictionaryFilePath.Count -ge 1) { + foreach ($file in $DictionaryFilePath) { + Write-Message -Level Verbose -Message "Importing dictionary file '$file'" + if (Test-Path -Path $file) { + try { + # Import the keys and values + Import-DbaCsv -Path $file -SqlInstance $server -Database tempdb -Schema dbo -Table DeterministicValues + } catch { + Stop-Function -Message "Could not import csv data from file '$file'" -ErrorRecord $_ -Target $file + } + } else { + Stop-Function -Message "Could not import dictionary file '$file'" -Target $file } - } else { - Stop-Function -Message "Could not import dictionary file '$file'" -Target $file } } - } - - # Get the database name - if (-not $Database) { - $Database = $tables.Name - } - # Loop through the databases - foreach ($dbName in $Database) { - if ($server.VersionMajor -lt 9) { - Stop-Function -Message "SQL Server version must be 2005 or greater" -Continue + # Get the database name + if (-not $Database) { + $Database = $tables.Name } - $db = $server.Databases[$($dbName)] + # Loop through the databases + foreach ($dbName in $Database) { + if ($server.VersionMajor -lt 9) { + Stop-Function -Message "SQL Server version must be 2005 or greater" -Continue + } - $nullmod = 0 + $db = $server.Databases[$($dbName)] - #region for each table - foreach ($tableobject in $tables.Tables) { - $elapsed = [System.Diagnostics.Stopwatch]::StartNew() + $nullmod = 0 - $uniqueDataTableName = $null - $uniqueValueColumns = @() - $stringBuilder = [System.Text.StringBuilder]'' + #region for each table + foreach ($tableobject in $tables.Tables) { + $elapsed = [System.Diagnostics.Stopwatch]::StartNew() - if ($tableobject.Name -in $ExcludeTable) { - Write-Message -Level Verbose -Message "Skipping $($tableobject.Name) because it is explicitly excluded" - continue - } + $uniqueDataTableName = $null + $uniqueValueColumns = @() + $stringBuilder = [System.Text.StringBuilder]'' - if ($tableobject.Name -notin $db.Tables.Name) { - Stop-Function -Message "Table $($tableobject.Name) is not present in $db" -Target $db -Continue - } + if ($tableobject.Name -in $ExcludeTable) { + Write-Message -Level Verbose -Message "Skipping $($tableobject.Name) because it is explicitly excluded" + continue + } - $dbTable = $db.Tables | Where-Object { $_.Schema -eq $tableobject.Schema -and $_.Name -eq $tableobject.Name } + if ($tableobject.Name -notin $db.Tables.Name) { + Stop-Function -Message "Table $($tableobject.Name) is not present in $db" -Target $db -Continue + } - [bool]$cleanupIdentityColumn = $false - [bool]$cleanupMaskingIndex = $false + $dbTable = $db.Tables | Where-Object { $_.Schema -eq $tableobject.Schema -and $_.Name -eq $tableobject.Name } - # The masking index name used for cleanup checks - $maskingIndexName = "NIX__$($dbTable.Schema)_$($dbTable.Name)_Masking" + [bool]$cleanupIdentityColumn = $false + [bool]$cleanupMaskingIndex = $false - # Make sure there is an identity column present to speed things up - # Skip column and index creation when -WhatIf is active to avoid leaving behind schema changes - if (-not $WhatIfPreference) { - if (-not ($dbTable.Columns | Where-Object { $_.Identity -eq $true })) { - Write-Message -Level Verbose -Message "Adding identity column to table [$($dbTable.Schema)].[$($dbTable.Name)]" - $query = "ALTER TABLE [$($dbTable.Schema)].[$($dbTable.Name)] ADD MaskingID BIGINT IDENTITY(1, 1) NOT NULL;" + # The masking index name used for cleanup checks + $maskingIndexName = "NIX__$($dbTable.Schema)_$($dbTable.Name)_Masking" - try { - Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database $db.Name -Query $query - } catch { - Stop-Function -Message "Could not alter the table to add the masking id" -Target $db -Continue - } + # The finally removes the masking index, the identity column and the unique values table also + # when a Stop-Function -Continue skips the rest of the table, when the command throws under + # EnableException and when the pipeline stops early (Ctrl+C, Select-Object -First). + try { + # Make sure there is an identity column present to speed things up + # Skip column and index creation when -WhatIf is active to avoid leaving behind schema changes + if (-not $WhatIfPreference) { + if (-not ($dbTable.Columns | Where-Object { $_.Identity -eq $true })) { + Write-Message -Level Verbose -Message "Adding identity column to table [$($dbTable.Schema)].[$($dbTable.Name)]" + $query = "ALTER TABLE [$($dbTable.Schema)].[$($dbTable.Name)] ADD MaskingID BIGINT IDENTITY(1, 1) NOT NULL;" - $cleanupIdentityColumn = $true + try { + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database $db.Name -Query $query + } catch { + Stop-Function -Message "Could not alter the table to add the masking id" -Target $db -Continue + } - $identityColumn = "MaskingID" + $cleanupIdentityColumn = $true - $dbTable.Columns.Refresh() - } else { - $identityColumn = $dbTable.Columns | Where-Object { $_.Identity } | Select-Object -ExpandProperty Name - } + $identityColumn = "MaskingID" - # Check if the index for the identity column is already present - try { - if ($dbTable.Indexes.Name -contains $maskingIndexName) { - Write-Message -Level Verbose -Message "Masking index already exists in table [$($dbTable.Schema)].[$($dbTable.Name)]. Dropping it..." - $dbTable.Indexes[$($maskingIndexName)].Drop() - } - } catch { - Stop-Function -Message "Could not remove identity index to table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue - } + $dbTable.Columns.Refresh() + } else { + $identityColumn = $dbTable.Columns | Where-Object { $_.Identity } | Select-Object -ExpandProperty Name + } - # Create the index for the identity column - try { - Write-Message -Level Verbose -Message "Adding index on identity column [$($identityColumn)] in table [$($dbTable.Schema)].[$($dbTable.Name)]" + # Check if the index for the identity column is already present + try { + if ($dbTable.Indexes.Name -contains $maskingIndexName) { + Write-Message -Level Verbose -Message "Masking index already exists in table [$($dbTable.Schema)].[$($dbTable.Name)]. Dropping it..." + $dbTable.Indexes[$($maskingIndexName)].Drop() + } + } catch { + Stop-Function -Message "Could not remove identity index to table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue + } + + # Create the index for the identity column + try { + Write-Message -Level Verbose -Message "Adding index on identity column [$($identityColumn)] in table [$($dbTable.Schema)].[$($dbTable.Name)]" + + $query = "CREATE NONCLUSTERED INDEX [$($maskingIndexName)] ON [$($dbTable.Schema)].[$($dbTable.Name)]([$($identityColumn)])" - $query = "CREATE NONCLUSTERED INDEX [$($maskingIndexName)] ON [$($dbTable.Schema)].[$($dbTable.Name)]([$($identityColumn)])" + $queryParams = @{ + SqlInstance = $server + SqlCredential = $SqlCredential + Database = $db.Name + Query = $query + QueryTimeout = $CommandTimeout + } - $queryParams = @{ - SqlInstance = $server - SqlCredential = $SqlCredential - Database = $db.Name - Query = $query - QueryTimeout = $CommandTimeout + Invoke-DbaQuery @queryParams + $cleanupMaskingIndex = $true + } catch { + Stop-Function -Message "Could not add identity index to table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue + } } - Invoke-DbaQuery @queryParams - $cleanupMaskingIndex = $true - } catch { - Stop-Function -Message "Could not add identity index to table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue - } - } + $actionIdentityValues = @() - $actionIdentityValues = @() + try { + if ($WhatIfPreference) { + # In WhatIf mode, only get the row count without modifying the table structure + if ($tableobject.FilterQuery) { + $trimmedFilterQuery = ($tableobject.FilterQuery).Trim() - try { - if ($WhatIfPreference) { - # In WhatIf mode, only get the row count without modifying the table structure - if ($tableobject.FilterQuery) { - $trimmedFilterQuery = ($tableobject.FilterQuery).Trim() + if ($trimmedFilterQuery.EndsWith(";")) { + $trimmedFilterQuery = $trimmedFilterQuery.Substring(0, $trimmedFilterQuery.Length - 1) + } - if ($trimmedFilterQuery.EndsWith(";")) { - $trimmedFilterQuery = $trimmedFilterQuery.Substring(0, $trimmedFilterQuery.Length - 1) - } + $query = "SELECT COUNT(*) AS RowCount FROM ($trimmedFilterQuery) AS [dbatools_masking_source]" + } else { + $query = "SELECT COUNT(*) AS RowCount FROM [$($tableobject.Schema)].[$($tableobject.Name)]" + } - $query = "SELECT COUNT(*) AS RowCount FROM ($trimmedFilterQuery) AS [dbatools_masking_source]" - } else { - $query = "SELECT COUNT(*) AS RowCount FROM [$($tableobject.Schema)].[$($tableobject.Name)]" - } + $rowCount = ($db.Query($query)).RowCount + $data = New-Object object[] $rowCount + } elseif (-not $tableobject.FilterQuery) { + # Get all the columns from the table + $columnString = "[" + (($dbTable.Columns | Where-Object { $_.DataType -in $supportedDataTypes } | Select-Object Name -ExpandProperty Name) -join "],[") + "]" - $rowCount = ($db.Query($query)).RowCount - $data = New-Object object[] $rowCount - } elseif (-not $tableobject.FilterQuery) { - # Get all the columns from the table - $columnString = "[" + (($dbTable.Columns | Where-Object { $_.DataType -in $supportedDataTypes } | Select-Object Name -ExpandProperty Name) -join "],[") + "]" + # Add the identifier column + $columnString += ",[$($identityColumn)]" - # Add the identifier column - $columnString += ",[$($identityColumn)]" + # Put it all together + $query = "SELECT $($columnString) FROM [$($tableobject.Schema)].[$($tableobject.Name)]" - # Put it all together - $query = "SELECT $($columnString) FROM [$($tableobject.Schema)].[$($tableobject.Name)]" + # Get the data + [array]$data = $db.Query($query) + } else { + # Get the query from the table objects + $query = ($tableobject.FilterQuery).ToLower() - # Get the data - [array]$data = $db.Query($query) - } else { - # Get the query from the table objects - $query = ($tableobject.FilterQuery).ToLower() + # Check if the query already contains the identifier column + if (-not ($query | Select-String -Pattern $identityColumn)) { + # Split up the query from the first "FROM" + $queryParts = $query -split "FROM", 2 + + # Put it all together again with the identifier + $query = "$($queryParts[0].Trim()), $($identityColumn) FROM $($queryParts[1].Trim())" + } - # Check if the query already contains the identifier column - if (-not ($query | Select-String -Pattern $identityColumn)) { - # Split up the query from the first "FROM" - $queryParts = $query -split "FROM", 2 + # Get the data + [array]$data = $db.Query($query) - # Put it all together again with the identifier - $query = "$($queryParts[0].Trim()), $($identityColumn) FROM $($queryParts[1].Trim())" + $actionIdentityValues = @($data | ForEach-Object { $PSItem.$identityColumn } | Where-Object { $null -ne $PSItem } | Select-Object -Unique) + } + } catch { + Stop-Function -Message "Failure retrieving the data from table [$($tableobject.Schema)].[$($tableobject.Name)]" -Target $Database -ErrorRecord $_ -Continue } - # Get the data - [array]$data = $db.Query($query) + #region unique indexes + # Check if the table contains unique indexes + if ($WhatIfPreference -and $tableobject.HasUniqueIndex) { + Write-Message -Level Verbose -Message "Skipping unique value preparation for [$($tableobject.Schema)].[$($tableobject.Name)] because -WhatIf is active" + } elseif ($tableobject.HasUniqueIndex) { + + # Loop through the rows and generate a unique value for each row + Write-Message -Level Verbose -Message "Generating unique values for [$($tableobject.Schema)].[$($tableobject.Name)]" + + $params = @{ + SqlInstance = $server + SqlCredential = $SqlCredential + Database = $db.name + Schema = $tableobject.Schema + Table = $tableobject.Name + } - $actionIdentityValues = @($data | ForEach-Object { $PSItem.$identityColumn } | Where-Object { $null -ne $PSItem } | Select-Object -Unique) - } - } catch { - Stop-Function -Message "Failure retrieving the data from table [$($tableobject.Schema)].[$($tableobject.Name)]" -Target $Database -ErrorRecord $_ -Continue - } + $indexToTable = Convert-DbaIndexToTable @params - #region unique indexes - # Check if the table contains unique indexes - if ($WhatIfPreference -and $tableobject.HasUniqueIndex) { - Write-Message -Level Verbose -Message "Skipping unique value preparation for [$($tableobject.Schema)].[$($tableobject.Name)] because -WhatIf is active" - } elseif ($tableobject.HasUniqueIndex) { - - # Loop through the rows and generate a unique value for each row - Write-Message -Level Verbose -Message "Generating unique values for [$($tableobject.Schema)].[$($tableobject.Name)]" - - $params = @{ - SqlInstance = $server - SqlCredential = $SqlCredential - Database = $db.name - Schema = $tableobject.Schema - Table = $tableobject.Name - } + if ($indexToTable) { + # compare the index columns to the column in the json table object + $compareParams = @{ + ReferenceObject = $indexToTable.Columns + DifferenceObject = $tableobject.Columns.Name + IncludeEqual = $true + } + $maskingColumnIndexCount = (Compare-Object @compareParams | Where-Object { $_.SideIndicator -eq "==" }).Count - $indexToTable = Convert-DbaIndexToTable @params + # Check if there is any need to generate unique values + if ($maskingColumnIndexCount -ge 1) { - if ($indexToTable) { - # compare the index columns to the column in the json table object - $compareParams = @{ - ReferenceObject = $indexToTable.Columns - DifferenceObject = $tableobject.Columns.Name - IncludeEqual = $true - } - $maskingColumnIndexCount = (Compare-Object @compareParams | Where-Object { $_.SideIndicator -eq "==" }).Count + # Check if the temporary table already exists + $server.Databases['tempdb'].Tables.Refresh() + $uniqueDataTableName = $indexToTable.TempTableName - # Check if there is any need to generate unique values - if ($maskingColumnIndexCount -ge 1) { + if ($server.Databases['tempdb'].Tables.Name -contains $indexToTable.TempTableName) { + Write-Message -Level Verbose -Message "Table '$($indexToTable.TempTableName)' already exists. Dropping it.." + try { + $query = "DROP TABLE $($indexToTable.TempTableName)" + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $query + } catch { + Stop-Function -Message "Could not drop temporary table" + } + } - # Check if the temporary table already exists - $server.Databases['tempdb'].Tables.Refresh() - $uniqueDataTableName = $indexToTable.TempTableName + # Create the temporary table + try { + Write-Message -Level Verbose -Message "Creating temporary table '$($indexToTable.TempTableName)'" + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $indexToTable.CreateStatement + } catch { + Stop-Function -Message "Could not create temporary table #[$($tableobject.Schema)].[$($tableobject.Name)]" + } - if ($server.Databases['tempdb'].Tables.Name -contains $indexToTable.TempTableName) { - Write-Message -Level Verbose -Message "Table '$($indexToTable.TempTableName)' already exists. Dropping it.." - try { - $query = "DROP TABLE $($indexToTable.TempTableName)" - Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $query - } catch { - Stop-Function -Message "Could not drop temporary table" - } - } + # Create the unique index table + try { + Write-Message -Level Verbose -Message "Creating the unique index for temporary table '$($indexToTable.TempTableName)'" + Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $indexToTable.UniqueIndexStatement + } catch { + Stop-Function -Message "Could not create temporary table #[$($tableobject.Schema)].[$($tableobject.Name)]" + } - # Create the temporary table - try { - Write-Message -Level Verbose -Message "Creating temporary table '$($indexToTable.TempTableName)'" - Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $indexToTable.CreateStatement - } catch { - Stop-Function -Message "Could not create temporary table #[$($tableobject.Schema)].[$($tableobject.Name)]" - } + # Create a unique row + $retryCount = 0 + for ($i = 0; $i -lt $data.Count; $i++) { + $insertQuery = "INSERT INTO [$($indexToTable.TempTableName)]([$($indexToTable.Columns -join '],[')]) VALUES(" + $insertFailed = $false + $insertValues = @() - # Create the unique index table - try { - Write-Message -Level Verbose -Message "Creating the unique index for temporary table '$($indexToTable.TempTableName)'" - Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $indexToTable.UniqueIndexStatement - } catch { - Stop-Function -Message "Could not create temporary table #[$($tableobject.Schema)].[$($tableobject.Name)]" - } + foreach ($indexColumn in $indexToTable.Columns) { + $columnMaskInfo = $tableobject.Columns | Where-Object { $_.Name -eq $indexColumn } - # Create a unique row - $retryCount = 0 - for ($i = 0; $i -lt $data.Count; $i++) { - $insertQuery = "INSERT INTO [$($indexToTable.TempTableName)]([$($indexToTable.Columns -join '],[')]) VALUES(" - $insertFailed = $false - $insertValues = @() - - foreach ($indexColumn in $indexToTable.Columns) { - $columnMaskInfo = $tableobject.Columns | Where-Object { $_.Name -eq $indexColumn } - - if ($indexColumn -eq "RowNr") { - $newValue = $i + 1 - } elseif ($columnMaskInfo) { - # make sure min is good - if ($columnMaskInfo.MinValue) { - $min = $columnMaskInfo.MinValue - } else { - if ($columnMaskInfo.CharacterString) { - $min = 1 + if ($indexColumn -eq "RowNr") { + $newValue = $i + 1 + } elseif ($columnMaskInfo) { + # make sure min is good + if ($columnMaskInfo.MinValue) { + $min = $columnMaskInfo.MinValue + } else { + if ($columnMaskInfo.CharacterString) { + $min = 1 + } else { + $min = 0 + } + } + + # make sure max is good + if ($MaxValue) { + if ($columnMaskInfo.MaxValue -le $MaxValue) { + $max = $columnMaskInfo.MaxValue + } else { + $max = $MaxValue + } + } else { + $max = $columnMaskInfo.MaxValue + } + + if (-not $columnMaskInfo.MaxValue -and -not (Test-Bound -ParameterName MaxValue)) { + $max = 10 + } + + if ((-not $columnMaskInfo.MinValue -or -not $columnMaskInfo.MaxValue) -and ($columnMaskInfo.ColumnType -match 'date')) { + if (-not $columnMaskInfo.MinValue) { + $min = (Get-Date).AddDays(-365) + } + if (-not $columnMaskInfo.MaxValue) { + $max = (Get-Date).AddDays(365) + } + } + + if ($columnMaskInfo.CharacterString) { + $charstring = $columnMaskInfo.CharacterString + } else { + $charstring = $CharacterString + } + + # Generate a new value + $newValue = $null + + $newValueParams = $null + + try { + $newValueParams = $null + if (-not $columnobject.SubType -and $columnobject.ColumnType -in $supportedDataTypes) { + $newValueParams = @{ + DataType = $columnMaskInfo.SubType + Min = $columnMaskInfo.MinValue + Max = $columnMaskInfo.MaxValue + Locale = $Locale + } + } else { + $newValueParams = @{ + RandomizerType = $columnMaskInfo.MaskingType + RandomizerSubtype = $columnMaskInfo.SubType + Min = $min + Max = $max + CharacterString = $charstring + Format = $columnMaskInfo.Format + Separator = $columnMaskInfo.Separator + Locale = $Locale + } + } + + $newValue = Get-DbaRandomizedValue @newValueParams + } catch { + Stop-Function -Message "Failure" -Target $columnMaskInfo -Continue -ErrorRecord $_ + } } else { - $min = 0 + $newValue = $null } - } - # make sure max is good - if ($MaxValue) { - if ($columnMaskInfo.MaxValue -le $MaxValue) { - $max = $columnMaskInfo.MaxValue + if ($columnMaskInfo) { + try { + $insertValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnMaskInfo.ColumnType -Nullable:$columnMaskInfo.Nullable -EnableException + + if ($convertedValue.ErrorMessage) { + $maskingErrorFlag = $true + Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue + } + } catch { + Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $newValue + } + + $insertValues += $insertValue.NewValue + } elseif ($indexColumn -eq "RowNr") { + $insertValues += $newValue } else { - $max = $MaxValue + $insertValues += "NULL" } - } else { - $max = $columnMaskInfo.MaxValue + + $uniqueValueColumns += $columnMaskInfo.Name } - if (-not $columnMaskInfo.MaxValue -and -not (Test-Bound -ParameterName MaxValue)) { - $max = 10 + # Join all the values to the insert query + $insertQuery += "$($insertValues -join ','));" + + # Try inserting the value + try { + $null = $server.Databases['tempdb'].Query($insertQuery) + $insertFailed = $false + } catch { + Write-Message -Level Verbose -Message "Could not insert value" + $insertFailed = $true } - if ((-not $columnMaskInfo.MinValue -or -not $columnMaskInfo.MaxValue) -and ($columnMaskInfo.ColumnType -match 'date')) { - if (-not $columnMaskInfo.MinValue) { - $min = (Get-Date).AddDays(-365) + # Try to insert the value as long it's failed + while ($insertFailed) { + if ($retryCount -eq $Retry) { + Stop-Function -Message "Could not create a unique row after $retryCount tries. Stopping..." + return } - if (-not $columnMaskInfo.MaxValue) { - $max = (Get-Date).AddDays(365) - } - } - if ($columnMaskInfo.CharacterString) { - $charstring = $columnMaskInfo.CharacterString - } else { - $charstring = $CharacterString - } + $insertQuery = "INSERT INTO [$($indexToTable.TempTableName)]([$($indexToTable.Columns -join '],[')]) VALUES(" + + foreach ($indexColumn in $indexToTable.Columns) { + $columnMaskInfo = $tableobject.Columns | Where-Object { $_.Name -eq $indexColumn } + + if ($indexColumn -eq "RowNr") { + $newValue = $i + 1 + } elseif ($columnMaskInfo) { + # make sure min is good + if ($columnMaskInfo.MinValue) { + $min = $columnMaskInfo.MinValue + } else { + if ($columnMaskInfo.CharacterString) { + $min = 1 + } else { + $min = 0 + } + } - # Generate a new value - $newValue = $null + # make sure max is good + if ($MaxValue) { + if ($columnMaskInfo.MaxValue -le $MaxValue) { + $max = $columnMaskInfo.MaxValue + } else { + $max = $MaxValue + } + } else { + $max = $columnMaskInfo.MaxValue + } - $newValueParams = $null + if (-not $columnMaskInfo.MaxValue -and -not (Test-Bound -ParameterName MaxValue)) { + $max = 10 + } - try { - $newValueParams = $null - if (-not $columnobject.SubType -and $columnobject.ColumnType -in $supportedDataTypes) { - $newValueParams = @{ - DataType = $columnMaskInfo.SubType - Min = $columnMaskInfo.MinValue - Max = $columnMaskInfo.MaxValue - Locale = $Locale + if ((-not $columnMaskInfo.MinValue -or -not $columnMaskInfo.MaxValue) -and ($columnMaskInfo.ColumnType -match 'date')) { + if (-not $columnMaskInfo.MinValue) { + $min = (Get-Date).AddDays(-365) + } + if (-not $columnMaskInfo.MaxValue) { + $max = (Get-Date).AddDays(365) + } + } + + if ($columnMaskInfo.CharacterString) { + $charstring = $columnMaskInfo.CharacterString + } else { + $charstring = $CharacterString + } + + # Generate a new value + $newValue = $null + + $newValueParams = $null + + try { + $newValueParams = $null + if (-not $columnobject.SubType -and $columnobject.ColumnType -in $supportedDataTypes) { + $newValueParams = @{ + DataType = $columnMaskInfo.SubType + Min = $columnMaskInfo.MinValue + Max = $columnMaskInfo.MaxValue + Locale = $Locale + } + } else { + $newValueParams = @{ + RandomizerType = $columnMaskInfo.MaskingType + RandomizerSubtype = $columnMaskInfo.SubType + Min = $min + Max = $max + CharacterString = $charstring + Format = $columnMaskInfo.Format + Separator = $columnMaskInfo.Separator + Locale = $Locale + } + } + + $newValue = Get-DbaRandomizedValue @newValueParams + } catch { + Stop-Function -Message "Failure" -Target $columnMaskInfo -Continue -ErrorRecord $_ + } + } else { + $newValue = $null } - } else { - $newValueParams = @{ - RandomizerType = $columnMaskInfo.MaskingType - RandomizerSubtype = $columnMaskInfo.SubType - Min = $min - Max = $max - CharacterString = $charstring - Format = $columnMaskInfo.Format - Separator = $columnMaskInfo.Separator - Locale = $Locale + + if ($columnMaskInfo) { + try { + $insertValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnMaskInfo.ColumnType -Nullable:$columnMaskInfo.Nullable -EnableException + + if ($convertedValue.ErrorMessage) { + $maskingErrorFlag = $true + Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue + } + } catch { + Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $newValue + } + + $insertValues += $insertValue.NewValue + } elseif ($indexColumn -eq "RowNr") { + $insertValues += $newValue + } else { + $insertValues += "NULL" } } - $newValue = Get-DbaRandomizedValue @newValueParams - } catch { - Stop-Function -Message "Failure" -Target $columnMaskInfo -Continue -ErrorRecord $_ + # Join all the values to the insert query + $insertQuery += "$($insertValues -join ','));" + + # Try inserting the value + try { + $null = $server.Databases['tempdb'].Query($insertQuery) + $insertFailed = $false + } catch { + Write-Message -Level Verbose -Message "Could not insert value" + $insertFailed = $true + $retryCount++ + } } - } else { - $newValue = $null } - if ($columnMaskInfo) { - try { - $insertValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnMaskInfo.ColumnType -Nullable:$columnMaskInfo.Nullable -EnableException + try { + Write-Message -Level Verbose -Message "Creating masking index for [$($indexToTable.TempTableName)]" + $query = "CREATE NONCLUSTERED INDEX [NIX_$($indexToTable.TempTableName)_MaskID] ON [$($indexToTable.TempTableName)]([RowNr])" + $null = $server.Databases['tempdb'].Query($query) + } catch { + Stop-Function -Message "Could not add masking index for [$($indexToTable.TempTableName)]" -ErrorRecord $_ + } + } else { + Write-Message -Level Verbose -Message "Table [$($tableobject.Schema)].[$($tableobject.Name)] does not contain any masking index columns to process" + } + } else { + Stop-Function -Message "The table does not have any indexes" + } + } - if ($convertedValue.ErrorMessage) { - $maskingErrorFlag = $true - Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue - } - } catch { - Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $newValue + #endregion unique indexes + + $tablecolumns = $tableobject.Columns + + if ($Column) { + $tablecolumns = $tablecolumns | Where-Object { $_.Name -in $Column } + } + + if ($ExcludeColumn) { + if ([string]$uniqueIndex.Columns -match ($ExcludeColumn -join "|")) { + Stop-Function -Message "Column present in -ExcludeColumn cannot be excluded because it's part of an unique index" -Target $ExcludeColumn -Continue + } + + $tablecolumns = $tablecolumns | Where-Object { $_.Name -notin $ExcludeColumn } + } + + if (-not $tablecolumns) { + Write-Message -Level Verbose "No columns to process in [$($dbName)].[$($tableobject.Schema)].[$($tableobject.Name)], moving on" + continue + } + + # Figure out if the columns has actions + $columnsWithActions = @() + $columnsWithActions += $tableobject.Columns | Where-Object { $null -ne $_.Action } + + # Figure out if the columns has composites + $columnsWithComposites = @() + $columnsWithComposites += $tableobject.Columns | Where-Object { $null -ne $_.Composite } + + # Check for both special actions + if (($columnsWithComposites.Count -ge 1) -and ($columnsWithActions.Count -ge 1)) { + Stop-Function -Message "You cannot use both composites and actions" -Target $tableobject -Continue + } + + # Filter out columns with actions or composites for separate processing + $standardColumns = $tablecolumns | Where-Object { + ($_.Name -notin $columnsWithActions.Name) -and + ($_.Name -notin $columnsWithComposites.Name) + } + + if ($Pscmdlet.ShouldProcess($instance, "Masking $($data.Count) row(s) for column [$($tablecolumns.Name -join ', ')] in $($dbName).$($tableobject.Schema).$($tableobject.Name)")) { + $totalBatches = [System.Math]::Ceiling($data.Count / $BatchSize) + [bool]$maskingErrorFlag = $false + + # OPTIMIZED SECTION: Process rows in batches, updating all columns for each row at once + $batchNr = 0 + $batchRowNr = 0 + $rowNumber = 0 + + # Process rows in batches + try { + for ($rowIndex = 0; $rowIndex -lt $data.Count; $rowIndex++) { + $row = $data[$rowIndex] + $rowNumber++ + $batchRowNr++ + + if ((($batchRowNr - 1) % 100) -eq 0) { + $progressParams = @{ + StepNumber = $batchNr + TotalSteps = $totalBatches + Activity = "Masking $($data.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($dbName) on $instance" + Message = "Generating Updates" } - $insertValues += $insertValue.NewValue - } elseif ($indexColumn -eq "RowNr") { - $insertValues += $newValue - } else { - $insertValues += "NULL" + Write-ProgressHelper @progressParams } - $uniqueValueColumns += $columnMaskInfo.Name - } + # Create array to hold all column updates for this row + $updates = @() - # Join all the values to the insert query - $insertQuery += "$($insertValues -join ','));" + # Process all standard columns for this row + foreach ($columnobject in $standardColumns) { + $newValue = $null - # Try inserting the value - try { - $null = $server.Databases['tempdb'].Query($insertQuery) - $insertFailed = $false - } catch { - Write-Message -Level Verbose -Message "Could not insert value" - $insertFailed = $true - } + # Handle static values + if ($null -ne $columnobject.StaticValue) { + $newValue = $columnobject.StaticValue - # Try to insert the value as long it's failed - while ($insertFailed) { - if ($retryCount -eq $Retry) { - Stop-Function -Message "Could not create a unique row after $retryCount tries. Stopping..." - return - } + if ($null -eq $newValue -and -not $columnobject.Nullable) { + Write-Message -Message "Column '$($columnobject.Name)' static value cannot be null when column is set not to be nullable." -Level Warning + continue + } + } + # Check for various conditions to determine the new value + elseif ($columnobject.KeepNull -and $columnobject.Nullable -and + (($row.($columnobject.Name)).GetType().Name -eq 'DBNull') -or + ($row.($columnobject.Name) -eq '')) { + $newValue = $null + } elseif (-not $columnobject.KeepNull -and $columnobject.Nullable -and + (($nullmod++) % $ModulusFactor -eq 0)) { + $newValue = $null + } elseif ($tableobject.HasUniqueIndex -and $columnobject.Name -in $uniqueValueColumns) { + # Get value from unique data table + $query = "SELECT $($columnobject.Name) FROM $($uniqueDataTableName) WHERE [RowNr] = $rowNumber" + + try { + $uniqueData = Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database tempdb -Query $query + } catch { + Stop-Function -Message "Something went wrong getting the unique data" -Target $query -ErrorRecord $_ -continue + } - $insertQuery = "INSERT INTO [$($indexToTable.TempTableName)]([$($indexToTable.Columns -join '],[')]) VALUES(" + if ($null -eq $uniqueData) { + Stop-Function -Message "Could not find any unique values" -Target $tableobject + return + } - foreach ($indexColumn in $indexToTable.Columns) { - $columnMaskInfo = $tableobject.Columns | Where-Object { $_.Name -eq $indexColumn } + $newValue = $uniqueData.$($columnobject.Name) + } elseif ($columnobject.Deterministic) { + # Check for deterministic value + if (($null -ne $row.($columnobject.Name)) -and ($row.($columnobject.Name) -ne '')) { + try { + $lookupValue = Convert-DbaMaskingValue -Value $row.($columnobject.Name) -DataType varchar -Nullable:$columnobject.Nullable -EnableException - if ($indexColumn -eq "RowNr") { - $newValue = $i + 1 - } elseif ($columnMaskInfo) { + if ($convertedValue.ErrorMessage) { + $maskingErrorFlag = $true + Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue -continue + } + } catch { + Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $row.($columnobject.Name) -continue + } + + $query = "SELECT [NewValue] FROM dbo.DeterministicValues WHERE [ValueKey] = $($lookupValue.NewValue)" + + try { + $lookupResult = $null + $lookupResult = $server.Databases['tempdb'].Query($query) + + if ($lookupResult.NewValue) { + $newValue = $lookupResult.NewValue + } + } catch { + Stop-Function -Message "Something went wrong retrieving the deterministic values" -Target $query -ErrorRecord $_ -continue + } + } + } + + # If we haven't determined a value yet, generate one + if ($null -eq $newValue -and $null -eq $columnobject.StaticValue) { # make sure min is good - if ($columnMaskInfo.MinValue) { - $min = $columnMaskInfo.MinValue + if ($columnobject.MinValue) { + $min = $columnobject.MinValue } else { - if ($columnMaskInfo.CharacterString) { + if ($columnobject.CharacterString) { $min = 1 } else { $min = 0 @@ -674,722 +946,462 @@ function Invoke-DbaDbDataMasking { # make sure max is good if ($MaxValue) { - if ($columnMaskInfo.MaxValue -le $MaxValue) { - $max = $columnMaskInfo.MaxValue + if ($columnobject.MaxValue -le $MaxValue) { + $max = $columnobject.MaxValue } else { $max = $MaxValue } } else { - $max = $columnMaskInfo.MaxValue + $max = $columnobject.MaxValue } - if (-not $columnMaskInfo.MaxValue -and -not (Test-Bound -ParameterName MaxValue)) { + if (-not $columnobject.MaxValue -and -not (Test-Bound -ParameterName MaxValue)) { $max = 10 } - if ((-not $columnMaskInfo.MinValue -or -not $columnMaskInfo.MaxValue) -and ($columnMaskInfo.ColumnType -match 'date')) { - if (-not $columnMaskInfo.MinValue) { + if ((-not $columnobject.MinValue -or -not $columnobject.MaxValue) -and ($columnobject.ColumnType -match 'date')) { + if (-not $columnobject.MinValue) { $min = (Get-Date).AddDays(-365) } - if (-not $columnMaskInfo.MaxValue) { + if (-not $columnobject.MaxValue) { $max = (Get-Date).AddDays(365) } } - if ($columnMaskInfo.CharacterString) { - $charstring = $columnMaskInfo.CharacterString + if ($columnobject.CharacterString) { + $charstring = $columnobject.CharacterString } else { $charstring = $CharacterString } - # Generate a new value - $newValue = $null - + # Setup the new value parameters $newValueParams = $null - try { - $newValueParams = $null - if (-not $columnobject.SubType -and $columnobject.ColumnType -in $supportedDataTypes) { + if ($null -eq $columnobject.SubType) { + $newValueParams = @{ + DataType = $columnobject.ColumnType + Min = $min + Max = $max + CharacterString = $charstring + Format = $columnobject.Format + Locale = $Locale + } + } elseif ($columnobject.SubType.ToLowerInvariant() -in 'shuffle', 'string2', 'string') { + if ($columnobject.ColumnType -in 'bigint', 'char', 'int', 'nchar', 'nvarchar', 'smallint', 'tinyint', 'varchar') { $newValueParams = @{ - DataType = $columnMaskInfo.SubType - Min = $columnMaskInfo.MinValue - Max = $columnMaskInfo.MaxValue - Locale = $Locale + RandomizerType = "Random" + RandomizerSubtype = "Shuffle" + Value = ($row.$($columnobject.Name)) + Locale = $Locale } - } else { + } elseif ($columnobject.ColumnType -in 'decimal', 'numeric', 'float', 'money', 'smallmoney', 'real') { $newValueParams = @{ - RandomizerType = $columnMaskInfo.MaskingType - RandomizerSubtype = $columnMaskInfo.SubType - Min = $min - Max = $max - CharacterString = $charstring - Format = $columnMaskInfo.Format - Separator = $columnMaskInfo.Separator + RandomizerType = "Random" + RandomizerSubtype = "Shuffle" + Value = ($row.$($columnobject.Name)) Locale = $Locale } } + } else { + $newValueParams = @{ + RandomizerType = $columnobject.MaskingType + RandomizerSubtype = $columnobject.SubType + Min = $min + Max = $max + CharacterString = $charstring + Format = $columnobject.Format + Separator = $columnobject.Separator + Locale = $Locale + } + } + # Generate the new value + try { $newValue = Get-DbaRandomizedValue @newValueParams } catch { - Stop-Function -Message "Failure" -Target $columnMaskInfo -Continue -ErrorRecord $_ + $maskingErrorFlag = $true + Stop-Function -Message "Failure" -Target $columnobject -Continue -ErrorRecord $_ } - } else { - $newValue = $null } - if ($columnMaskInfo) { + # Convert the value for SQL + try { + if ($row.($columnobject.Name) -eq '' -and $columnobject.ColumnType -in 'decimal') { + $newvalue = "0.00" + } + $convertedValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnobject.ColumnType -Nullable:$columnobject.Nullable -EnableException + + if ($convertedValue.ErrorMessage) { + $maskingErrorFlag = $true + Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue -continue + } + } catch { + Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $newValue -continue + } + + # Add to the updates + $updates += "[$($columnobject.Name)] = $($convertedValue.NewValue)" + + # Handle deterministic values storage + if ($columnobject.Deterministic -and ($null -ne $row.($columnobject.Name)) -and + ($row.($columnobject.Name) -ne '') -and ($null -eq $lookupResult.NewValue)) { try { - $insertValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnMaskInfo.ColumnType -Nullable:$columnMaskInfo.Nullable -EnableException + $previous = Convert-DbaMaskingValue -Value $row.($columnobject.Name) -DataType $columnobject.ColumnType -Nullable:$columnobject.Nullable -EnableException if ($convertedValue.ErrorMessage) { $maskingErrorFlag = $true Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue + continue } + + $query = "INSERT INTO dbo.DeterministicValues (ValueKey, NewValue) VALUES ($($previous.NewValue), $($convertedValue.NewValue));" + $null = $server.Databases['tempdb'].Query($query) } catch { - Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $newValue + Stop-Function -Message "Could not save deterministic value.`n$_" -Target $query -ErrorRecord $_ + continue } - - $insertValues += $insertValue.NewValue - } elseif ($indexColumn -eq "RowNr") { - $insertValues += $newValue - } else { - $insertValues += "NULL" } } - # Join all the values to the insert query - $insertQuery += "$($insertValues -join ','));" - - # Try inserting the value - try { - $null = $server.Databases['tempdb'].Query($insertQuery) - $insertFailed = $false - } catch { - Write-Message -Level Verbose -Message "Could not insert value" - $insertFailed = $true - $retryCount++ + # Only create an update if we have columns to update + if ($updates.Count -gt 0) { + # Create one UPDATE statement for all columns in this row + $updateQuery = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($updates -join ', ') WHERE [$($identityColumn)] = $($row.$($identityColumn)); " + $null = $stringBuilder.AppendLine($updateQuery) } - } - } - try { - Write-Message -Level Verbose -Message "Creating masking index for [$($indexToTable.TempTableName)]" - $query = "CREATE NONCLUSTERED INDEX [NIX_$($indexToTable.TempTableName)_MaskID] ON [$($indexToTable.TempTableName)]([RowNr])" - $null = $server.Databases['tempdb'].Query($query) - } catch { - Stop-Function -Message "Could not add masking index for [$($indexToTable.TempTableName)]" -ErrorRecord $_ - } - } else { - Write-Message -Level Verbose -Message "Table [$($tableobject.Schema)].[$($tableobject.Name)] does not contain any masking index columns to process" - } - } else { - Stop-Function -Message "The table does not have any indexes" - } - } - - #endregion unique indexes - - $tablecolumns = $tableobject.Columns - - if ($Column) { - $tablecolumns = $tablecolumns | Where-Object { $_.Name -in $Column } - } - - if ($ExcludeColumn) { - if ([string]$uniqueIndex.Columns -match ($ExcludeColumn -join "|")) { - Stop-Function -Message "Column present in -ExcludeColumn cannot be excluded because it's part of an unique index" -Target $ExcludeColumn -Continue - } + # If we've reached the batch size or this is the last row, execute the batch + if ($batchRowNr -eq $BatchSize -or $rowIndex -eq ($data.Count - 1)) { + # Increase the batch counter + $batchNr++ - $tablecolumns = $tablecolumns | Where-Object { $_.Name -notin $ExcludeColumn } - } - - if (-not $tablecolumns) { - Write-Message -Level Verbose "No columns to process in [$($dbName)].[$($tableobject.Schema)].[$($tableobject.Name)], moving on" - continue - } - - # Figure out if the columns has actions - $columnsWithActions = @() - $columnsWithActions += $tableobject.Columns | Where-Object { $null -ne $_.Action } + # Execute the batch if we have updates + if ($stringBuilder.Length -gt 0) { + try { + $progressParams = @{ + StepNumber = $batchNr + TotalSteps = $totalBatches + Activity = "Masking $($data.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($dbName) on $instance" + Message = "Executing Batch $batchNr/$totalBatches" + } - # Figure out if the columns has composites - $columnsWithComposites = @() - $columnsWithComposites += $tableobject.Columns | Where-Object { $null -ne $_.Composite } + Write-ProgressHelper @progressParams - # Check for both special actions - if (($columnsWithComposites.Count -ge 1) -and ($columnsWithActions.Count -ge 1)) { - Stop-Function -Message "You cannot use both composites and actions" - } + Write-Message -Level Verbose -Message "Executing batch $batchNr/$totalBatches" - # Filter out columns with actions or composites for separate processing - $standardColumns = $tablecolumns | Where-Object { - ($_.Name -notin $columnsWithActions.Name) -and - ($_.Name -notin $columnsWithComposites.Name) - } + $queryParams = @{ + SqlInstance = $instance + SqlCredential = $SqlCredential + Database = $db.Name + Query = $stringBuilder.ToString() + EnableException = $EnableException + QueryTimeout = $CommandTimeout + } - if ($Pscmdlet.ShouldProcess($instance, "Masking $($data.Count) row(s) for column [$($tablecolumns.Name -join ', ')] in $($dbName).$($tableobject.Schema).$($tableobject.Name)")) { - $totalBatches = [System.Math]::Ceiling($data.Count / $BatchSize) - [bool]$maskingErrorFlag = $false + Invoke-DbaQuery @queryParams + } catch { + $maskingErrorFlag = $true + Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $_ `n$($stringBuilder.ToString())" -Target $stringBuilder.ToString() -Continue -ErrorRecord $_ + } - # OPTIMIZED SECTION: Process rows in batches, updating all columns for each row at once - $batchNr = 0 - $batchRowNr = 0 - $rowNumber = 0 + # Clear the string builder for the next batch + $null = $stringBuilder.Clear() + } - # Process rows in batches - try { - for ($rowIndex = 0; $rowIndex -lt $data.Count; $rowIndex++) { - $row = $data[$rowIndex] - $rowNumber++ - $batchRowNr++ - - if ((($batchRowNr - 1) % 100) -eq 0) { - $progressParams = @{ - StepNumber = $batchNr - TotalSteps = $totalBatches - Activity = "Masking $($data.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($dbName) on $instance" - Message = "Generating Updates" + # Reset batch row counter + $batchRowNr = 0 + } } - - Write-ProgressHelper @progressParams + } finally { + Write-ProgressHelper -Completed } - # Create array to hold all column updates for this row - $updates = @() - - # Process all standard columns for this row - foreach ($columnobject in $standardColumns) { - $newValue = $null + # Process Actions separately + if ($columnsWithActions.Count -ge 1) { + foreach ($columnObject in $columnsWithActions) { + Write-Message -Level Verbose -Message "Processing action for [$($columnObject.Name)]" - # Handle static values - if ($null -ne $columnobject.StaticValue) { - $newValue = $columnobject.StaticValue + [bool]$validAction = $true + $columnAction = $columnobject.Action + $query = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET [$($columnObject.Name)] = " - if ($null -eq $newValue -and -not $columnobject.Nullable) { - Write-Message -Message "Column '$($columnobject.Name)' static value cannot be null when column is set not to be nullable." -Level Warning - continue - } - } - # Check for various conditions to determine the new value - elseif ($columnobject.KeepNull -and $columnobject.Nullable -and - (($row.($columnobject.Name)).GetType().Name -eq 'DBNull') -or - ($row.($columnobject.Name) -eq '')) { - $newValue = $null - } elseif (-not $columnobject.KeepNull -and $columnobject.Nullable -and - (($nullmod++) % $ModulusFactor -eq 0)) { - $newValue = $null - } elseif ($tableobject.HasUniqueIndex -and $columnobject.Name -in $uniqueValueColumns) { - # Get value from unique data table - $query = "SELECT $($columnobject.Name) FROM $($uniqueDataTableName) WHERE [RowNr] = $rowNumber" - try { - $uniqueData = Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database tempdb -Query $query - } catch { - Stop-Function -Message "Something went wrong getting the unique data" -Target $query -ErrorRecord $_ -continue - } - if ($null -eq $uniqueData) { - Stop-Function -Message "Could not find any unique values" -Target $tableobject - return - } - - $newValue = $uniqueData.$($columnobject.Name) - } elseif ($columnobject.Deterministic) { - # Check for deterministic value - if (($null -ne $row.($columnobject.Name)) -and ($row.($columnobject.Name) -ne '')) { - try { - $lookupValue = Convert-DbaMaskingValue -Value $row.($columnobject.Name) -DataType varchar -Nullable:$columnobject.Nullable -EnableException - - if ($convertedValue.ErrorMessage) { - $maskingErrorFlag = $true - Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue -continue + if ($columnAction.Category -eq 'DateTime') { + switch ($columnAction.Type) { + "Add" { + $query += "DATEADD($($columnAction.SubCategory), $($columnAction.Value), [$($columnObject.Name)]);" } - } catch { - Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $row.($columnobject.Name) -continue - } - - $query = "SELECT [NewValue] FROM dbo.DeterministicValues WHERE [ValueKey] = $($lookupValue.NewValue)" - - try { - $lookupResult = $null - $lookupResult = $server.Databases['tempdb'].Query($query) - - if ($lookupResult.NewValue) { - $newValue = $lookupResult.NewValue + "Subtract" { + $query += "DATEADD($($columnAction.SubCategory), - $($columnAction.Value), [$($columnObject.Name)]);" + } + default { + $validAction = $false } - } catch { - Stop-Function -Message "Something went wrong retrieving the deterministic values" -Target $query -ErrorRecord $_ -continue - } - } - } - - # If we haven't determined a value yet, generate one - if ($null -eq $newValue -and $null -eq $columnobject.StaticValue) { - # make sure min is good - if ($columnobject.MinValue) { - $min = $columnobject.MinValue - } else { - if ($columnobject.CharacterString) { - $min = 1 - } else { - $min = 0 - } - } - - # make sure max is good - if ($MaxValue) { - if ($columnobject.MaxValue -le $MaxValue) { - $max = $columnobject.MaxValue - } else { - $max = $MaxValue - } - } else { - $max = $columnobject.MaxValue - } - - if (-not $columnobject.MaxValue -and -not (Test-Bound -ParameterName MaxValue)) { - $max = 10 - } - - if ((-not $columnobject.MinValue -or -not $columnobject.MaxValue) -and ($columnobject.ColumnType -match 'date')) { - if (-not $columnobject.MinValue) { - $min = (Get-Date).AddDays(-365) - } - if (-not $columnobject.MaxValue) { - $max = (Get-Date).AddDays(365) } - } - - if ($columnobject.CharacterString) { - $charstring = $columnobject.CharacterString - } else { - $charstring = $CharacterString - } - - # Setup the new value parameters - $newValueParams = $null - - if ($null -eq $columnobject.SubType) { - $newValueParams = @{ - DataType = $columnobject.ColumnType - Min = $min - Max = $max - CharacterString = $charstring - Format = $columnobject.Format - Locale = $Locale + } elseif ($columnAction.Category -eq 'Number') { + switch ($columnAction.Type) { + "Add" { + $query += "[$($columnObject.Name)] + $($columnAction.Value);" + } + "Divide" { + $query += "[$($columnObject.Name)] / $($columnAction.Value);" + } + "Multiply" { + $query += "[$($columnObject.Name)] * $($columnAction.Value);" + } + "Subtract" { + $query += "[$($columnObject.Name)] - $($columnAction.Value);" + } + default { + $validAction = $false + } } - } elseif ($columnobject.SubType.ToLowerInvariant() -in 'shuffle', 'string2', 'string') { - if ($columnobject.ColumnType -in 'bigint', 'char', 'int', 'nchar', 'nvarchar', 'smallint', 'tinyint', 'varchar') { - $newValueParams = @{ - RandomizerType = "Random" - RandomizerSubtype = "Shuffle" - Value = ($row.$($columnobject.Name)) - Locale = $Locale + } elseif ($columnAction.Category -eq 'Column') { + switch ($columnAction.Type) { + "Set" { + if ($columnobject.ColumnType -like '*int*' -or $columnobject.ColumnType -in 'bit', 'bool', 'decimal', 'numeric', 'float', 'money', 'smallmoney', 'real') { + $query += "$($columnAction.Value)" + } elseif ($columnobject.ColumnType -in '*date*', 'time', 'uniqueidentifier') { + $query += "'$($columnAction.Value)'" + } else { + $query += "'$($columnAction.Value)'" + } + } + "Nullify" { + if ($columnobject.Nullable) { + $query += "NULL" + } else { + $validAction = $false + } } - } elseif ($columnobject.ColumnType -in 'decimal', 'numeric', 'float', 'money', 'smallmoney', 'real') { - $newValueParams = @{ - RandomizerType = "Random" - RandomizerSubtype = "Shuffle" - Value = ($row.$($columnobject.Name)) - Locale = $Locale + default { + $validAction = $false } } - } else { - $newValueParams = @{ - RandomizerType = $columnobject.MaskingType - RandomizerSubtype = $columnobject.SubType - Min = $min - Max = $max - CharacterString = $charstring - Format = $columnobject.Format - Separator = $columnobject.Separator - Locale = $Locale + } + # Apply actions only to the rows returned by FilterQuery + if ($validAction -and $tableobject.FilterQuery -and $actionIdentityValues.Count -ge 1) { + for ($batchStart = 0; $batchStart -lt $actionIdentityValues.Count; $batchStart += $BatchSize) { + $batchEnd = [System.Math]::Min($batchStart + $BatchSize - 1, $actionIdentityValues.Count - 1) + $identityBatch = $actionIdentityValues[$batchStart .. $batchEnd] -join ", " + $null = $stringBuilder.AppendLine($query.TrimEnd(";") + " WHERE [$identityColumn] IN ($identityBatch);") } } - # Generate the new value - try { - $newValue = Get-DbaRandomizedValue @newValueParams - } catch { - $maskingErrorFlag = $true - Stop-Function -Message "Failure" -Target $columnobject -Continue -ErrorRecord $_ + # Add the query to the rest + if ($validAction -and -not $tableobject.FilterQuery) { + $null = $stringBuilder.AppendLine($query) } } - # Convert the value for SQL try { - if ($row.($columnobject.Name) -eq '' -and $columnobject.ColumnType -in 'decimal') { - $newvalue = "0.00" - } - $convertedValue = Convert-DbaMaskingValue -Value $newValue -DataType $columnobject.ColumnType -Nullable:$columnobject.Nullable -EnableException - - if ($convertedValue.ErrorMessage) { - $maskingErrorFlag = $true - Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue -continue + if ($stringBuilder.Length -ge 1) { + Invoke-DbaQuery -SqlInstance $instance -SqlCredential $SqlCredential -Database $db.Name -Query $stringBuilder.ToString() -EnableException } } catch { - Stop-Function -Message "Could not convert value" -ErrorRecord $_ -Target $newValue -continue - } - - # Add to the updates - $updates += "[$($columnobject.Name)] = $($convertedValue.NewValue)" - - # Handle deterministic values storage - if ($columnobject.Deterministic -and ($null -ne $row.($columnobject.Name)) -and - ($row.($columnobject.Name) -ne '') -and ($null -eq $lookupResult.NewValue)) { - try { - $previous = Convert-DbaMaskingValue -Value $row.($columnobject.Name) -DataType $columnobject.ColumnType -Nullable:$columnobject.Nullable -EnableException - - if ($convertedValue.ErrorMessage) { - $maskingErrorFlag = $true - Stop-Function "Could not convert the value. $($convertedValue.ErrorMessage)" -Target $convertedValue - continue - } - - $query = "INSERT INTO dbo.DeterministicValues (ValueKey, NewValue) VALUES ($($previous.NewValue), $($convertedValue.NewValue));" - $null = $server.Databases['tempdb'].Query($query) - } catch { - Stop-Function -Message "Could not save deterministic value.`n$_" -Target $query -ErrorRecord $_ - continue - } + Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $_" -Target $stringBuilder -Continue -ErrorRecord $_ } - } - # Only create an update if we have columns to update - if ($updates.Count -gt 0) { - # Create one UPDATE statement for all columns in this row - $updateQuery = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($updates -join ', ') WHERE [$($identityColumn)] = $($row.$($identityColumn)); " - $null = $stringBuilder.AppendLine($updateQuery) + $null = $stringBuilder.Clear() } - # If we've reached the batch size or this is the last row, execute the batch - if ($batchRowNr -eq $BatchSize -or $rowIndex -eq ($data.Count - 1)) { - # Increase the batch counter - $batchNr++ - - # Execute the batch if we have updates - if ($stringBuilder.Length -gt 0) { - try { - $progressParams = @{ - StepNumber = $batchNr - TotalSteps = $totalBatches - Activity = "Masking $($data.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($dbName) on $instance" - Message = "Executing Batch $batchNr/$totalBatches" - } - - Write-ProgressHelper @progressParams - - Write-Message -Level Verbose -Message "Executing batch $batchNr/$totalBatches" - - $queryParams = @{ - SqlInstance = $instance - SqlCredential = $SqlCredential - Database = $db.Name - Query = $stringBuilder.ToString() - EnableException = $EnableException - QueryTimeout = $CommandTimeout - } - - Invoke-DbaQuery @queryParams - } catch { - $maskingErrorFlag = $true - Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $_ `n$($stringBuilder.ToString())" -Target $stringBuilder.ToString() -Continue -ErrorRecord $_ - } - - # Clear the string builder for the next batch - $null = $stringBuilder.Clear() - } + # Process Composites separately + if ($columnsWithComposites.Count -ge 1) { + foreach ($columnObject in $columnsWithComposites) { + Write-Message -Level Verbose -Message "Processing composite for [$($columnObject.Name)]" - # Reset batch row counter - $batchRowNr = 0 - } - } - } finally { - Write-ProgressHelper -Completed - } + $compositeItems = @() - # Process Actions separately - if ($columnsWithActions.Count -ge 1) { - foreach ($columnObject in $columnsWithActions) { - Write-Message -Level Verbose -Message "Processing action for [$($columnObject.Name)]" + foreach ($columnComposite in $columnObject.Composite) { + if ($columnComposite.Type -eq 'Column') { + $compositeItems += "[$($columnComposite.Value)]" + } elseif ($columnComposite.Type -eq 'Static') { + $compositeItems += "'$($columnComposite.Value)'" + } elseif ($columnComposite.Type -in $supportedFakerMaskingTypes) { + try { + $newValue = $null - [bool]$validAction = $true - $columnAction = $columnobject.Action - $query = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET [$($columnObject.Name)] = " + if ($columnobject.SubType -in $supportedDataTypes) { + $newValueParams = @{ + DataType = $columnobject.SubType + CharacterString = $charstring + Min = $columnComposite.Min + Max = $columnComposite.Max + Locale = $Locale + } + $newValue = Get-DbaRandomizedValue @newValueParams + } else { + $newValueParams = @{ + RandomizerType = $columnobject.MaskingType + RandomizerSubtype = $columnobject.SubType + Min = $min + Max = $max + CharacterString = $charstring + Format = $columnobject.Format + Separator = $columnobject.Separator + Locale = $Locale + } + $newValue = Get-DbaRandomizedValue @newValueParams + } + } catch { + Stop-Function -Message "Failure" -Target $faker -Continue -ErrorRecord $_ + } - if ($columnAction.Category -eq 'DateTime') { - switch ($columnAction.Type) { - "Add" { - $query += "DATEADD($($columnAction.SubCategory), $($columnAction.Value), [$($columnObject.Name)]);" - } - "Subtract" { - $query += "DATEADD($($columnAction.SubCategory), - $($columnAction.Value), [$($columnObject.Name)]);" - } - default { - $validAction = $false - } - } - } elseif ($columnAction.Category -eq 'Number') { - switch ($columnAction.Type) { - "Add" { - $query += "[$($columnObject.Name)] + $($columnAction.Value);" - } - "Divide" { - $query += "[$($columnObject.Name)] / $($columnAction.Value);" - } - "Multiply" { - $query += "[$($columnObject.Name)] * $($columnAction.Value);" - } - "Subtract" { - $query += "[$($columnObject.Name)] - $($columnAction.Value);" - } - default { - $validAction = $false - } - } - } elseif ($columnAction.Category -eq 'Column') { - switch ($columnAction.Type) { - "Set" { - if ($columnobject.ColumnType -like '*int*' -or $columnobject.ColumnType -in 'bit', 'bool', 'decimal', 'numeric', 'float', 'money', 'smallmoney', 'real') { - $query += "$($columnAction.Value)" - } elseif ($columnobject.ColumnType -in '*date*', 'time', 'uniqueidentifier') { - $query += "'$($columnAction.Value)'" - } else { - $query += "'$($columnAction.Value)'" - } - } - "Nullify" { - if ($columnobject.Nullable) { - $query += "NULL" + if ($columnobject.ColumnType -match 'int') { + $compositeItems += " $newValue" + } elseif ($columnobject.ColumnType -in 'bit', 'bool') { + if ($columnValue) { + $compositeItems += "1" + } else { + $compositeItems += "0" + } + } else { + $newValue = ($newValue).Tostring().Replace("'", "''") + $compositeItems += "'$newValue'" + } } else { - $validAction = $false + $compositeItems += "" } } - default { - $validAction = $false - } + + $compositeItemsUpdated = $compositeItems | ForEach-Object { $_ = "ISNULL($($_), '')"; $_ } + + $null = $stringBuilder.AppendLine("UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET [$($columnObject.Name)] = $($compositeItemsUpdated -join ' + ')") } - } - # Apply actions only to the rows returned by FilterQuery - if ($validAction -and $tableobject.FilterQuery -and $actionIdentityValues.Count -ge 1) { - for ($batchStart = 0; $batchStart -lt $actionIdentityValues.Count; $batchStart += $BatchSize) { - $batchEnd = [System.Math]::Min($batchStart + $BatchSize - 1, $actionIdentityValues.Count - 1) - $identityBatch = $actionIdentityValues[$batchStart .. $batchEnd] -join ", " - $null = $stringBuilder.AppendLine($query.TrimEnd(";") + " WHERE [$identityColumn] IN ($identityBatch);") + + try { + Invoke-DbaQuery -SqlInstance $instance -SqlCredential $SqlCredential -Database $db.Name -Query $stringBuilder.ToString() -EnableException + } catch { + Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $_" -Target $stringBuilder -Continue -ErrorRecord $_ } - } - # Add the query to the rest - if ($validAction -and -not $tableobject.FilterQuery) { - $null = $stringBuilder.AppendLine($query) + $null = $stringBuilder.Clear() } - } - try { - if ($stringBuilder.Length -ge 1) { - Invoke-DbaQuery -SqlInstance $instance -SqlCredential $SqlCredential -Database $db.Name -Query $stringBuilder.ToString() -EnableException + # Return the masking results + if ($maskingErrorFlag) { + $maskingStatus = "Failed" + } else { + $maskingStatus = "Successful" } - } catch { - $stringBuilder.ToString() - Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $_" -Target $stringBuilder -Continue -ErrorRecord $_ - } - - $null = $stringBuilder.Clear() - } - - # Process Composites separately - if ($columnsWithComposites.Count -ge 1) { - foreach ($columnObject in $columnsWithComposites) { - Write-Message -Level Verbose -Message "Processing composite for [$($columnObject.Name)]" - - $compositeItems = @() - - foreach ($columnComposite in $columnObject.Composite) { - if ($columnComposite.Type -eq 'Column') { - $compositeItems += "[$($columnComposite.Value)]" - } elseif ($columnComposite.Type -eq 'Static') { - $compositeItems += "'$($columnComposite.Value)'" - } elseif ($columnComposite.Type -in $supportedFakerMaskingTypes) { - try { - $newValue = $null - if ($columnobject.SubType -in $supportedDataTypes) { - $newValueParams = @{ - DataType = $columnobject.SubType - CharacterString = $charstring - Min = $columnComposite.Min - Max = $columnComposite.Max - Locale = $Locale - } - - $newValue = Get-DbaRandomizedValue @newValueParams - } else { - $newValueParams = @{ - RandomizerType = $columnobject.MaskingType - RandomizerSubtype = $columnobject.SubType - Min = $min - Max = $max - CharacterString = $charstring - Format = $columnobject.Format - Separator = $columnobject.Separator - Locale = $Locale - } - - $newValue = Get-DbaRandomizedValue @newValueParams - } - } catch { - Stop-Function -Message "Failure" -Target $faker -Continue -ErrorRecord $_ - } - - if ($columnobject.ColumnType -match 'int') { - $compositeItems += " $newValue" - } elseif ($columnobject.ColumnType -in 'bit', 'bool') { - if ($columnValue) { - $compositeItems += "1" - } else { - $compositeItems += "0" - } - } else { - $newValue = ($newValue).Tostring().Replace("'", "''") - $compositeItems += "'$newValue'" - } - } else { - $compositeItems += "" - } + [PSCustomObject]@{ + ComputerName = $db.Parent.ComputerName + InstanceName = $db.Parent.ServiceName + SqlInstance = $db.Parent.DomainInstanceName + Database = $dbName + Schema = $tableobject.Schema + Table = $tableobject.Name + Columns = $tableobject.Columns.Name + Rows = $($data.Count) + Elapsed = [prettytimespan]$elapsed.Elapsed + Status = $maskingStatus } - $compositeItemsUpdated = $compositeItems | ForEach-Object { $_ = "ISNULL($($_), '')"; $_ } - $null = $stringBuilder.AppendLine("UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET [$($columnObject.Name)] = $($compositeItemsUpdated -join ' + ')") + # Reset time + $null = $elapsed.Reset() } + } finally { + # The cleanups warn instead of calling Stop-Function: a -Continue would skip the cleanups + # after it, and a throw from a finally would replace the error that is on its way out. + # Clean up the masking index created for this masking run + if ($cleanupMaskingIndex) { + try { + # Refresh the indexes to make sure to have the latest list + $dbTable.Indexes.Refresh() - try { - $stringBuilder.ToString() - Invoke-DbaQuery -SqlInstance $instance -SqlCredential $SqlCredential -Database $db.Name -Query $stringBuilder.ToString() -EnableException - } catch { - Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $_" -Target $stringBuilder -Continue -ErrorRecord $_ + # Check if the index is there + if ($dbTable.Indexes.Name -contains $maskingIndexName) { + Write-Message -Level verbose -Message "Removing identity index from table [$($dbTable.Schema)].[$($dbTable.Name)]" + $dbTable.Indexes[$($maskingIndexName)].Drop() + } + } catch { + Write-Message -Level Warning -Message "Could not remove identity index from table [$($dbTable.Schema)].[$($dbTable.Name)]" -Target $dbTable -ErrorRecord $_ + } } - $null = $stringBuilder.Clear() - } - - # Return the masking results - if ($maskingErrorFlag) { - $maskingStatus = "Failed" - } else { - $maskingStatus = "Successful" - } - - [PSCustomObject]@{ - ComputerName = $db.Parent.ComputerName - InstanceName = $db.Parent.ServiceName - SqlInstance = $db.Parent.DomainInstanceName - Database = $dbName - Schema = $tableobject.Schema - Table = $tableobject.Name - Columns = $tableobject.Columns.Name - Rows = $($data.Count) - Elapsed = [prettytimespan]$elapsed.Elapsed - Status = $maskingStatus - } - + # Clean up the identity column (always runs, regardless of -WhatIf or errors during masking) + if ($cleanupIdentityColumn) { + try { + Write-Message -Level Verbose -Message "Removing identity column [$($identityColumn)] from table [$($dbTable.Schema)].[$($dbTable.Name)]" - # Reset time - $null = $elapsed.Reset() - } + $query = "ALTER TABLE [$($dbTable.Schema)].[$($dbTable.Name)] DROP COLUMN [$($identityColumn)]" - # Clean up the masking index created for this masking run - if ($cleanupMaskingIndex) { - try { - # Refresh the indexes to make sure to have the latest list - $dbTable.Indexes.Refresh() + Invoke-DbaQuery -SqlInstance $instance -SqlCredential $SqlCredential -Database $db.Name -Query $query -EnableException + } catch { + Write-Message -Level Warning -Message "Could not remove identity column from table [$($dbTable.Schema)].[$($dbTable.Name)]" -Target $dbTable -ErrorRecord $_ + } + } - # Check if the index is there - if ($dbTable.Indexes.Name -contains $maskingIndexName) { - Write-Message -Level verbose -Message "Removing identity index from table [$($dbTable.Schema)].[$($dbTable.Name)]" - $dbTable.Indexes[$($maskingIndexName)].Drop() + # Cleanup + if ($uniqueDataTableName) { + Write-Message -Message "Cleaning up unique temporary table '$uniqueDataTableName'" -Level verbose + $query = "DROP TABLE [$($uniqueDataTableName)];" + try { + $null = Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $query -EnableException + } catch { + Write-Message -Level Warning -Message "Could not clean up unique values table '$uniqueDataTableName'" -Target $uniqueDataTableName -ErrorRecord $_ + } } - } catch { - Stop-Function -Message "Could not remove identity index from table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue } } + #endregion for each table - # Clean up the identity column (always runs, regardless of -WhatIf or errors during masking) - if ($cleanupIdentityColumn) { + # Export the dictionary when needed + if ($DictionaryExportPath) { try { - Write-Message -Level Verbose -Message "Removing identity column [$($identityColumn)] from table [$($dbTable.Schema)].[$($dbTable.Name)]" + # Handle dictionary + $query = "SELECT [ValueKey], [NewValue] FROM dbo.DeterministicValues" + [array]$dictResult = $server.Databases['tempdb'].Query($query) - $query = "ALTER TABLE [$($dbTable.Schema)].[$($dbTable.Name)] DROP COLUMN [$($identityColumn)]" + if ($dictResult.Count -ge 1) { + Write-Message -Message "Writing dictionary for $($db.Name)" -Level Verbose - Invoke-DbaQuery -SqlInstance $instance -SqlCredential $SqlCredential -Database $db.Name -Query $query -EnableException - } catch { - Stop-Function -Message "Could not remove identity column from table [$($dbTable.Schema)].[$($dbTable.Name)]" -Continue - } - } - - # Cleanup - if ($uniqueDataTableName) { - Write-Message -Message "Cleaning up unique temporary table '$uniqueDataTableName'" -Level verbose - $query = "DROP TABLE [$($uniqueDataTableName)];" - try { - $null = Invoke-DbaQuery -SqlInstance $server -SqlCredential $SqlCredential -Database 'tempdb' -Query $query -EnableException - } catch { - Stop-Function -Message "Could not clean up unique values table '$uniqueDataTableName'" -Target $uniqueDataTableName -ErrorRecord $_ - } - } - } - #endregion for each table + # Check if the output directory already exists + if (-not (Test-Path -Path $DictionaryExportPath)) { + $null = New-Item -Path $DictionaryExportPath -ItemType Directory + } - # Export the dictionary when needed - if ($DictionaryExportPath) { - try { - # Handle dictionary - $query = "SELECT [ValueKey], [NewValue] FROM dbo.DeterministicValues" - [array]$dictResult = $server.Databases['tempdb'].Query($query) + # Of course with Linux we need to change the slashes + if (-not $script:isWindows) { + $dictionaryFileName = $dictionaryFileName.Replace("\", "/") + } - if ($dictResult.Count -ge 1) { - Write-Message -Message "Writing dictionary for $($db.Name)" -Level Verbose + # Setup the file paths + $filenamepart = $server.Name.Replace('\', '$').Replace('TCP:', '').Replace(',', '.') + $dictionaryFileName = "$DictionaryExportPath\$($filenamepart).$($db.Name).Dictionary.csv" - # Check if the output directory already exists - if (-not (Test-Path -Path $DictionaryExportPath)) { - $null = New-Item -Path $DictionaryExportPath -ItemType Directory - } + # Export dictionary + $null = $dictResult | Export-Csv -Path $dictionaryFileName -NoTypeInformation - # Of course with Linux we need to change the slashes - if (-not $script:isWindows) { - $dictionaryFileName = $dictionaryFileName.Replace("\", "/") + Get-ChildItem -Path $dictionaryFileName + } else { + Write-Message -Level Verbose -Message "No values to export as a dictionary" } - - # Setup the file paths - $filenamepart = $server.Name.Replace('\', '$').Replace('TCP:', '').Replace(',', '.') - $dictionaryFileName = "$DictionaryExportPath\$($filenamepart).$($db.Name).Dictionary.csv" - - # Export dictionary - $null = $dictResult | Export-Csv -Path $dictionaryFileName -NoTypeInformation - - Get-ChildItem -Path $dictionaryFileName - } else { - Write-Message -Level Verbose -Message "No values to export as a dictionary" + } catch { + Stop-Function -Message "Something went wrong writing the dictionary to the $DictionaryExportPath" -Target $DictionaryExportPath -Continue -ErrorRecord $_ } - } catch { - Stop-Function -Message "Something went wrong writing the dictionary to the $DictionaryExportPath" -Target $DictionaryExportPath -Continue -ErrorRecord $_ } - } - } # End foreach database - - # Do some cleanup - $null = $server.Databases['tempdb'].Tables.Refresh() + } # End foreach database + } finally { + # Do some cleanup + # A warning instead of Stop-Function, because a throw from a finally would replace the error that is on + # its way out. The refresh is inside the try for the same reason. + try { + $null = $server.Databases["tempdb"].Tables.Refresh() - if ($server.Databases['tempdb'].Tables.Name -contains 'DeterministicValues') { - $query = "DROP TABLE dbo.DeterministicValues" + if ($server.Databases["tempdb"].Tables.Name -contains "DeterministicValues") { + $query = "DROP TABLE dbo.DeterministicValues" - try { - Write-Message -Level Verbose -Message "Cleaning up deterministic values table" - $null = $server.Databases['tempdb'].Query($query) + Write-Message -Level Verbose -Message "Cleaning up deterministic values table" + $null = $server.Databases["tempdb"].Query($query) + } } catch { - Stop-Function -Message "Could not remove deterministic value table" -ErrorRecord $_ + Write-Message -Level Warning -Message "Could not remove deterministic value table" -ErrorRecord $_ } } diff --git a/tests/Invoke-DbaDbDataMasking.Tests.ps1 b/tests/Invoke-DbaDbDataMasking.Tests.ps1 index 93a725808e4..5af77c480fe 100644 --- a/tests/Invoke-DbaDbDataMasking.Tests.ps1 +++ b/tests/Invoke-DbaDbDataMasking.Tests.ps1 @@ -447,6 +447,209 @@ Describe $CommandName -Tag IntegrationTests { } + Context "When a table of deterministic values is left over" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $leftoverPath = "$tempPath\leftover" + $null = New-Item -Path $leftoverPath -ItemType Directory + $splatLeftoverConfig = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Path = $leftoverPath + } + $leftoverConfigFile = New-DbaDbMaskingConfig @splatLeftoverConfig + # A run that ended early left its table behind. The command drops it first, and the drop used to put a $null + # into the output before the results. + $splatLeftoverTable = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = "tempdb" + Query = "IF OBJECT_ID(N'dbo.DeterministicValues') IS NULL CREATE TABLE dbo.DeterministicValues ([ValueKey] VARCHAR(900), [NewValue] VARCHAR(900))" + } + $null = Invoke-DbaQuery @splatLeftoverTable + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + $splatLeftoverMasking = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + FilePath = $leftoverConfigFile.FullName + } + $leftoverResults = @(Invoke-DbaDbDataMasking @splatLeftoverMasking) + $leftoverWarning = $WarnVar + } + + It "Returns one result per table and nothing else" { + $leftoverWarning | Should -BeNullOrEmpty + $leftoverResults.Count | Should -Be 2 + $leftoverResults.Table | Should -Be @("people", "people2") + } + } + + Context "When a column of the config has a composite" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $compositePath = "$tempPath\composite" + $null = New-Item -Path $compositePath -ItemType Directory + $splatCompositeConfig = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Table = "people" + Path = $compositePath + } + $compositeConfigFile = New-DbaDbMaskingConfig @splatCompositeConfig + $compositeConfig = Get-Content -Path $compositeConfigFile.FullName -Raw | ConvertFrom-Json + $compositeColumn = $compositeConfig.Tables[0].Columns | Where-Object Name -eq "lname" + $compositeColumn.Composite = @( + [PSCustomObject]@{ Type = "Column"; Value = "fname" }, + [PSCustomObject]@{ Type = "Static"; Value = "-composite" } + ) + $compositeConfigPath = "$compositePath\composite.json" + $compositeConfig | ConvertTo-Json -Depth 10 | Set-Content -Path $compositeConfigPath + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + # The UPDATE statements of the composites used to be written to the pipeline before they were run. + $splatCompositeMasking = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + FilePath = $compositeConfigPath + } + $compositeResults = @(Invoke-DbaDbDataMasking @splatCompositeMasking) + $compositeWarning = $WarnVar + } + + It "Returns only the result of the table" { + $compositeWarning | Should -BeNullOrEmpty + $compositeResults.Count | Should -Be 1 + $compositeResults[0].Table | Should -Be "people" + } + + It "Fills the column from the composite" { + $splatCompositeData = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Query = "SELECT fname, lname FROM dbo.people" + } + foreach ($row in (Invoke-DbaQuery @splatCompositeData)) { + $row.lname | Should -Be "$($row.fname)-composite" + } + } + } + + Context "When an action of the config fails" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $actionPath = "$tempPath\action" + $null = New-Item -Path $actionPath -ItemType Directory + $splatActionConfig = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Table = "people" + Path = $actionPath + } + $actionConfigFile = New-DbaDbMaskingConfig @splatActionConfig + $actionConfig = Get-Content -Path $actionConfigFile.FullName -Raw | ConvertFrom-Json + # A division by zero makes the UPDATE of the action fail on the server. + $actionColumn = $actionConfig.Tables[0].Columns | Where-Object Name -eq "percenttest" + $actionColumn.Action = [PSCustomObject]@{ + Category = "Number" + Type = "Divide" + Value = 0 + } + $actionConfigPath = "$actionPath\action.json" + $actionConfig | ConvertTo-Json -Depth 10 | Set-Content -Path $actionConfigPath + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + # The failed UPDATE statements used to be written to the pipeline, and the -Continue after the warning skipped + # the removal of the identity column and the index the command had added to the table. + $splatActionMasking = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + FilePath = $actionConfigPath + WarningAction = "SilentlyContinue" + } + $actionResults = @(Invoke-DbaDbDataMasking @splatActionMasking) + $actionWarning = $WarnVar + } + + It "Warns about the failed update and returns nothing" { + ($actionWarning -join " ") | Should -BeLike "*Error updating dbo.people*" + $actionResults | Should -BeNullOrEmpty + } + + It "Removes the identity column and the index it added" { + $splatActionLeftover = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Query = "SELECT (SELECT COUNT(*) FROM sys.columns WHERE name = N'MaskingID') + (SELECT COUNT(*) FROM sys.indexes WHERE name LIKE N'NIX[_][_]%[_]Masking') AS Leftovers" + } + (Invoke-DbaQuery @splatActionLeftover).Leftovers | Should -Be 0 + } + } + + Context "When a table of the config has composites and actions" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $bothPath = "$tempPath\both" + $null = New-Item -Path $bothPath -ItemType Directory + $splatBothConfig = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Table = "people2" + Path = $bothPath + } + $bothConfigFile = New-DbaDbMaskingConfig @splatBothConfig + $bothConfig = Get-Content -Path $bothConfigFile.FullName -Raw | ConvertFrom-Json + $bothCompositeColumn = $bothConfig.Tables[0].Columns | Where-Object Name -eq "lname" + $bothCompositeColumn.Composite = @( + [PSCustomObject]@{ Type = "Static"; Value = "composite" } + ) + $bothActionColumn = $bothConfig.Tables[0].Columns | Where-Object Name -eq "percenttest" + $bothActionColumn.Action = [PSCustomObject]@{ + Category = "Number" + Type = "Add" + Value = 1 + } + $bothConfigPath = "$bothPath\both.json" + $bothConfig | ConvertTo-Json -Depth 10 | Set-Content -Path $bothConfigPath + $splatBothData = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Query = "SELECT fname, lname, dob, percenttest, bittest FROM dbo.people2 ORDER BY fname" + As = "PSObject" + } + $bothDataBefore = Invoke-DbaQuery @splatBothData | ConvertTo-Json -Compress + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + # The command warned that it cannot use both and then masked the table anyway. + $splatBothMasking = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + FilePath = $bothConfigPath + WarningAction = "SilentlyContinue" + } + $bothResults = @(Invoke-DbaDbDataMasking @splatBothMasking) + $bothWarning = $WarnVar + $bothDataAfter = Invoke-DbaQuery @splatBothData | ConvertTo-Json -Compress + } + + It "Warns and returns nothing" { + ($bothWarning -join " ") | Should -BeLike "*You cannot use both composites and actions*" + $bothResults | Should -BeNullOrEmpty + } + + It "Leaves the data of the table as it was" { + $bothDataAfter | Should -Be $bothDataBefore + } + + It "Removes the identity column and the index it added" { + $splatBothLeftover = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Query = "SELECT (SELECT COUNT(*) FROM sys.columns WHERE name = N'MaskingID') + (SELECT COUNT(*) FROM sys.indexes WHERE name LIKE N'NIX[_][_]%[_]Masking') AS Leftovers" + } + (Invoke-DbaQuery @splatBothLeftover).Leftovers | Should -Be 0 + } + } + Context "When the pipeline is stopped" { BeforeAll { $PSDefaultParameterValues["*-Dba*:EnableException"] = $true @@ -497,6 +700,7 @@ Describe $CommandName -Tag IntegrationTests { AfterAll { # The stopped run did not get to drop its table of deterministic values. + # It drops it in a finally now. The drop here keeps a regression from leaving the table to the next test file. $splatDropValues = @{ SqlInstance = $TestConfig.InstanceSingle Database = "tempdb" @@ -517,5 +721,23 @@ Describe $CommandName -Tag IntegrationTests { $stopRecords | Where-Object Activity -like "Masking *" | Should -Not -BeNullOrEmpty $openIds | Should -BeNullOrEmpty } + + It "Drops its table of deterministic values" { + $splatValuesTable = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = "tempdb" + Query = "SELECT COUNT(*) AS ValuesTables FROM sys.tables WHERE name = N'DeterministicValues'" + } + (Invoke-DbaQuery @splatValuesTable).ValuesTables | Should -Be 0 + } + + It "Removes the identity column and the index it added" { + $splatStopLeftover = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $dbName + Query = "SELECT (SELECT COUNT(*) FROM sys.columns WHERE name = N'MaskingID') + (SELECT COUNT(*) FROM sys.indexes WHERE name LIKE N'NIX[_][_]%[_]Masking') AS Leftovers" + } + (Invoke-DbaQuery @splatStopLeftover).Leftovers | Should -Be 0 + } } } \ No newline at end of file From f1c3ff9e58e137a3bc5b27796cd55c0a8d838057 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:33:44 +0200 Subject: [PATCH 05/15] Invoke-DbaAdvancedRestore - Stop the database after a failed restore and verify every file with -VerifyOnly - "Failed to restore db ..., stopping" was a Stop-Function -Continue followed by a break. Without EnableException the continue won, so the restore went on with the next backup file of the database whose restore had just failed. The -Continue is gone, so the break stops the database and the command goes on with the next one, as #5475 intended. - -VerifyOnly returned the result of the first backup file with a return, which left the whole end block: the other backup files and databases were never verified, and a connection the command had opened itself was not closed. The result is written to the pipeline now. Restore-DbaDatabase calls the command with EnableException, so the first item only affected direct callers; -VerifyOnly affected both. Part of #10776. (do Invoke-DbaAdvancedRestore, Restore-DbaDatabase) Co-Authored-By: Claude Opus 5.5 --- public/Invoke-DbaAdvancedRestore.ps1 | 9 ++-- tests/Invoke-DbaAdvancedRestore.Tests.ps1 | 52 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/public/Invoke-DbaAdvancedRestore.ps1 b/public/Invoke-DbaAdvancedRestore.ps1 index 5bb8e09210f..d559e9617a4 100644 --- a/public/Invoke-DbaAdvancedRestore.ps1 +++ b/public/Invoke-DbaAdvancedRestore.ps1 @@ -542,14 +542,15 @@ function Invoke-DbaAdvancedRestore { Write-Progress -id 1 -activity "Verifying $database backup file on $SqlInstance - Backup $BackupCnt of $($Backups.count)" -percentcomplete 0 -status ([System.String]::Format("Progress: {0} %", 0)) $verify = $restore.SqlVerify($server) Write-Progress -id 1 -activity "Verifying $database backup file on $SqlInstance - Backup $BackupCnt of $($Backups.count)" -status "Complete" -Completed + # No return here: it left the whole end block, so only the first backup file was verified. if ($verify) { Write-Message -Message "VerifyOnly restore Succeeded" -Level Verbose $restoreComplete = $true - return "Verify successful" + "Verify successful" } else { Write-Message -Message "VerifyOnly restore Failed" -Level Warning $restoreComplete = $False - return "Verify failed" + "Verify failed" } } else { $outerProgress = $BackupCnt / $Backups.Count * 100 @@ -616,7 +617,9 @@ function Invoke-DbaAdvancedRestore { Write-Message -Level Verbose -Message "Failed to restore $database" $restoreComplete = $False $ExitError = $_.Exception.InnerException - Stop-Function -Message "Failed to restore db $database, stopping" -ErrorRecord $_ -Continue + # No -Continue: it continued with the next backup file of this database, so the break that + # stops the database and goes on with the next one was never reached. + Stop-Function -Message "Failed to restore db $database, stopping" -ErrorRecord $_ break } finally { # The bar of this backup file ends with the file, also when its restore failed or was stopped. diff --git a/tests/Invoke-DbaAdvancedRestore.Tests.ps1 b/tests/Invoke-DbaAdvancedRestore.Tests.ps1 index da583efc598..182f235906c 100644 --- a/tests/Invoke-DbaAdvancedRestore.Tests.ps1 +++ b/tests/Invoke-DbaAdvancedRestore.Tests.ps1 @@ -326,6 +326,58 @@ Describe $CommandName -Tag IntegrationTests { } } + Context "When -VerifyOnly verifies more than one backup file" { + BeforeAll { + # The result of the first backup file was returned with a return, which left the whole end block, so the + # log backup was never verified. + $verifyResult = $backupHistory | Invoke-DbaAdvancedRestore -SqlInstance $TestConfig.InstanceSingle -VerifyOnly + $verifyWarning = $WarnVar + } + + It "Verifies every backup file" { + $verifyWarning | Should -BeNullOrEmpty + @($verifyResult | Where-Object { $PSItem -is [string] }) | Should -Be @("Verify successful", "Verify successful") + } + } + + Context "When the restore of a backup file fails" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + # A directory below a file cannot exist, so the restore of the full backup fails. The data file of master + # is a file on every instance. + $masterFile = Get-DbaDbFile -SqlInstance $TestConfig.InstanceSingle -Database master | Where-Object TypeDescription -eq "ROWS" | Select-Object -First 1 + $noDirectory = "$($masterFile.PhysicalName)\dbatoolsci_nodirectory" + $failDbName = "dbatoolsci_advrestorefail_$(Get-Random)" + $splatFailFormat = @{ + ReplaceDatabaseName = $failDbName + DataFileDirectory = $noDirectory + LogFileDirectory = $noDirectory + } + $failHistory = Get-DbaBackupInformation -SqlInstance $TestConfig.InstanceSingle -Path $backupPath | Select-DbaBackupInformation | Format-DbaBackupInformation @splatFailFormat + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + # The Stop-Function -Continue after the failure continued with the log backup of the database, so the break + # that was meant to stop the database was never reached. + $failResult = $failHistory | Invoke-DbaAdvancedRestore -SqlInstance $TestConfig.InstanceSingle -WarningAction SilentlyContinue + $failWarning = $WarnVar + } + + AfterAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $null = Remove-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Database $failDbName -ErrorAction SilentlyContinue + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + It "Warns once about the failed restore" { + @($failWarning | Where-Object { $PSItem -like "*Failed to restore db $failDbName, stopping*" }).Count | Should -Be 1 + } + + It "Stops with the backup file that failed" { + @($failResult).Count | Should -Be 1 + $failResult.RestoreComplete | Should -BeFalse + } + } + Context "When the pipeline ends at the first backup file" { BeforeAll { # The command runs in a runspace of its own, created by the PowerShell API without a host. There From 75d3ee466c5784ead7705b5b73771f102dcb52b3 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:35:54 +0200 Subject: [PATCH 06/15] Invoke-DbaAdvancedInstall - Return a result when the configuration file cannot be copied - The failed copy of the configuration file to the remote computer ended the command with a bare return, unlike the other failures, so Install-DbaInstance got no result for that computer. It returns the result object with the reason in Notes now, removes the session and removes the local configuration file, whose cleanup sits in the finally of the installation that is never reached. - "Failed to get reboot status from" used $computer, which the command never sets. Part of #10776. (do Invoke-DbaAdvancedInstall, Install-DbaInstance) Co-Authored-By: Claude Opus 5.5 --- public/Invoke-DbaAdvancedInstall.ps1 | 15 ++++++++++++--- tests/Invoke-DbaAdvancedInstall.Tests.ps1 | 20 +++++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/public/Invoke-DbaAdvancedInstall.ps1 b/public/Invoke-DbaAdvancedInstall.ps1 index 6428f4ad5d6..693c77cb324 100644 --- a/public/Invoke-DbaAdvancedInstall.ps1 +++ b/public/Invoke-DbaAdvancedInstall.ps1 @@ -232,7 +232,7 @@ function Invoke-DbaAdvancedInstall { $restartNeeded = Test-PendingReboot -ComputerName $ComputerName -Credential $Credential -NoPendingRename:$NoPendingRenameCheck } catch { $restartNeeded = $false - Stop-Function -Message "Failed to get reboot status from $computer" -ErrorRecord $_ + Stop-Function -Message "Failed to get reboot status from $ComputerName" -ErrorRecord $_ } if ($restartNeeded -and $Restart) { # Restart the computer prior to doing anything @@ -282,8 +282,17 @@ function Invoke-DbaAdvancedInstall { $null = Send-File -Path $ConfigurationPath -Destination $chosenPath -Session $session -ErrorAction Stop $session | Remove-PSSession } catch { - Stop-Function -Message "Failed to copy file $($ConfigurationPath) to $remoteConfig on $($ComputerName), exiting" -ErrorRecord $_ - return + # Like the other failures below, this one returns its result object, so the caller gets a result for the + # computer. The session and the local configuration file are cleaned up here, because the cleanup in the + # finally of the installation is not reached from here. + $msg = "Failed to copy file $($ConfigurationPath) to $remoteConfig on $($ComputerName), exiting" + Stop-Function -Message $msg -ErrorRecord $_ + $output.Notes += $msg + if ($session) { + $session | Remove-PSSession + } + Remove-Item -Path $ConfigurationPath -ErrorAction SilentlyContinue + return $output } } $installParams = $ArgumentList diff --git a/tests/Invoke-DbaAdvancedInstall.Tests.ps1 b/tests/Invoke-DbaAdvancedInstall.Tests.ps1 index 2b5b6e76a03..b04ade81dab 100644 --- a/tests/Invoke-DbaAdvancedInstall.Tests.ps1 +++ b/tests/Invoke-DbaAdvancedInstall.Tests.ps1 @@ -533,7 +533,7 @@ Describe $CommandName -Tag IntegrationTests { $installShell = [powershell]::Create() $installShell.Runspace = $installRunspace - $null = $installShell.AddCommand("Invoke-DbaAdvancedInstall").AddParameters($splatMissingInstall).Invoke() + $installResult = $installShell.AddCommand("Invoke-DbaAdvancedInstall").AddParameters($splatMissingInstall).Invoke() $installWarnings = @($installShell.Streams.Warning | ForEach-Object { $PSItem.Message }) $installRecords = @($installShell.Streams.Progress) $installShell.Dispose() @@ -544,6 +544,24 @@ Describe $CommandName -Tag IntegrationTests { ($installWarnings -join " ") | Should -Match "Failed to copy file" } + It "Names the computer whose reboot status it cannot read" { + # The message used the variable $computer, which the command never sets. + ($installWarnings -join " ") | Should -BeLike "*Failed to get reboot status from $($splatMissingInstall.ComputerName)*" + } + + It "Returns a failed result for the computer" { + # The failed copy returned nothing, unlike the other failures, so Install-DbaInstance had no result for it. + @($installResult).Count | Should -Be 1 + $installResult[0].ComputerName | Should -Be $splatMissingInstall.ComputerName + $installResult[0].Successful | Should -BeFalse + ($installResult[0].Notes -join " ") | Should -BeLike "*Failed to copy file*" + } + + It "Removes the local configuration file" { + # The cleanup of the configuration file is in the finally of the installation, which the failed copy never reached. + Test-Path -Path $missingInstallConfig | Should -BeFalse + } + It "Completes its progress bar" { # An Id stays on screen when its last record is not a completed one. Windows PowerShell completes its # own bar for loading modules with Id 0 as well, so a completed record somewhere is not enough. From 1460bcaa4e9a6282e659b2a6047bfc54d470908b Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:39:53 +0200 Subject: [PATCH 07/15] Invoke-DbaDbLogShipRecovery - Stop writing a $null before the result and name the instance in the error - The RESTORE ... WITH RECOVERY ran through Query(), which returns a $null for a statement without a result set, so the output of every successful recovery started with a $null. - "You must specify a -Database or -Force for all databases" used $server.name as its target, which is only set further down. Part of #10776. (do Invoke-DbaDbLogShipRecovery) Co-Authored-By: Claude Opus 5.5 --- public/Invoke-DbaDbLogShipRecovery.ps1 | 5 +- tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 | 77 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/public/Invoke-DbaDbLogShipRecovery.ps1 b/public/Invoke-DbaDbLogShipRecovery.ps1 index 51440b82945..b3554bbd6ee 100644 --- a/public/Invoke-DbaDbLogShipRecovery.ps1 +++ b/public/Invoke-DbaDbLogShipRecovery.ps1 @@ -128,7 +128,7 @@ function Invoke-DbaDbLogShipRecovery { process { foreach ($instance in $SqlInstance) { if (-not $Force -and -not $Database) { - Stop-Function -Message "You must specify a -Database or -Force for all databases" -Target $server.name + Stop-Function -Message "You must specify a -Database or -Force for all databases" -Target $instance return } $InputObject += Get-DbaDatabase -SqlInstance $instance -SqlCredential $SqlCredential -Database $Database @@ -308,7 +308,8 @@ function Invoke-DbaDbLogShipRecovery { Write-Message -Message "Restoring the database to it's normal state" -Level Verbose try { $query = "RESTORE DATABASE [$secondarydb] WITH RECOVERY" - $server.Query($query) + # Query() returns a $null for a statement without a result set. + $null = $server.Query($query) } catch { $recoverResult = "Failed" diff --git a/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 b/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 index 0c711480ad6..5402520bbd1 100644 --- a/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 +++ b/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 @@ -74,4 +74,81 @@ Describe $CommandName -Tag IntegrationTests { $openIds | Should -BeNullOrEmpty } } + + Context "When neither -Database nor -Force is given" { + It "Names the instance in the error" { + # The target was $server.name, and $server is only set further down. + $stopError = $null + try { + Invoke-DbaDbLogShipRecovery -SqlInstance $TestConfig.InstanceSingle -EnableException + } catch { + $stopError = $PSItem + } + $stopError | Should -Not -BeNullOrEmpty + "$($stopError.TargetObject)" | Should -Be "$($TestConfig.InstanceSingle)" + } + } + + Context "When a log shipping secondary is recovered" { + BeforeAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + $primaryDatabase = "dbatoolsci_lsrecovery_$(Get-Random)" + $secondaryDatabase = "$($primaryDatabase)_ls" + $sharedPath = Join-Path -Path $TestConfig.Temp -ChildPath "dbatoolsci_lsrecovery_$(Get-Random)" + $null = New-Item -Path $sharedPath -ItemType Directory + $copyDestinationFolder = Join-Path -Path $sharedPath -ChildPath "copy" + $null = New-Item -Path $copyDestinationFolder -ItemType Directory + # The setup follows the one in Invoke-DbaDbLogShipping.Tests.ps1: the command accepts the shared path only in + # the form \\server\share. In the lab $TestConfig.Temp is a share already. On CI it is a local folder on the + # runner, which also hosts the instance, so the administrative share of the drive names the same folder. + $sharedPathUnc = $sharedPath -replace "^(.):", "\\$env:COMPUTERNAME\`$1`$" + + $null = New-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Name $primaryDatabase -RecoveryModel Full + $null = Backup-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Database $primaryDatabase -Path $sharedPath -Type Full + $splatLogShipping = @{ + SourceSqlInstance = $TestConfig.InstanceSingle + DestinationSqlInstance = $TestConfig.InstanceSingle + Database = $primaryDatabase + SharedPath = $sharedPathUnc + IgnoreFileChecks = $true + CopyDestinationFolder = $copyDestinationFolder + UseExistingFullBackup = $true + SecondaryDatabaseSuffix = "_ls" + } + $null = Invoke-DbaDbLogShipping @splatLogShipping + + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + # The RESTORE WITH RECOVERY at the end ran through Query(), which returns a $null for a statement without a + # result set, so the output started with a $null. + $recoveryResults = @(Invoke-DbaDbLogShipRecovery -SqlInstance $TestConfig.InstanceSingle -Database $secondaryDatabase) + $recoveryWarning = $WarnVar + } + + AfterAll { + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + $splatRemoveLogShipping = @{ + PrimarySqlInstance = $TestConfig.InstanceSingle + SecondarySqlInstance = $TestConfig.InstanceSingle + Database = $primaryDatabase + RemoveSecondaryDatabase = $true + } + $null = Remove-DbaDbLogShipping @splatRemoveLogShipping -ErrorAction SilentlyContinue + $null = Remove-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Database $primaryDatabase, $secondaryDatabase -ErrorAction SilentlyContinue + Remove-Item -Path $sharedPath -Recurse -Force -ErrorAction SilentlyContinue + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + It "Returns one result for the database and nothing else" { + $recoveryWarning | Should -BeNullOrEmpty + $recoveryResults.Count | Should -Be 1 + $recoveryResults[0].Database | Should -Be $secondaryDatabase + $recoveryResults[0].RecoverResult | Should -Be "Success" + } + + It "Brings the database online" { + (Get-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Database $secondaryDatabase).Status | Should -Be "Normal" + } + } } \ No newline at end of file From f242ecf9db89cd2e46efa8aa87037233e00bcd15 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:41:43 +0200 Subject: [PATCH 08/15] Install-DbaInstance - Use the name given when the computer name cannot be resolved When Resolve-DbaNetworkName failed, $fullComputerName stayed empty, so Test-PendingReboot failed on the empty name and the warning read "Failed to get reboot status from " without a computer name. The command falls back to the name it was given now, as Update-DbaInstance does. Part of #10776. (do Install-DbaInstance) Co-Authored-By: Claude Opus 5.5 --- public/Install-DbaInstance.ps1 | 13 +++++++++++-- tests/Install-DbaInstance.Tests.ps1 | 5 +++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/public/Install-DbaInstance.ps1 b/public/Install-DbaInstance.ps1 index 24ae782e836..a570b10898f 100644 --- a/public/Install-DbaInstance.ps1 +++ b/public/Install-DbaInstance.ps1 @@ -566,8 +566,17 @@ function Install-DbaInstance { } # resolve names Write-ProgressHelper -TotalSteps $totalSteps -Activity $activity -StepNumber ($stepCounter++) -Message "Resolving computer name" - $resolvedName = Resolve-DbaNetworkName -ComputerName $computer -Credential $Credential - if ($computer.IsLocalHost) { + # Without a resolved name, the name given is used, as Update-DbaInstance does. An empty name made every + # later step fail on it, starting with the check for a pending reboot. + try { + $resolvedName = Resolve-DbaNetworkName -ComputerName $computer -Credential $Credential -EnableException + } catch { + Write-Message -Level Verbose -Message "Could not resolve $($computer.ComputerName) (this may occur with CredSSP or workgroup environments). Using provided name directly." + $resolvedName = $null + } + if (-not $resolvedName) { + $fullComputerName = $computer.ComputerName + } elseif ($computer.IsLocalHost) { # Don't add a domain to localhost as this might add a domain that is later not recognized by .IsLocalHost anymore (#6976). $fullComputerName = $resolvedName.ComputerName } else { diff --git a/tests/Install-DbaInstance.Tests.ps1 b/tests/Install-DbaInstance.Tests.ps1 index 16d9d7725f1..9c457f5f61f 100644 --- a/tests/Install-DbaInstance.Tests.ps1 +++ b/tests/Install-DbaInstance.Tests.ps1 @@ -415,6 +415,11 @@ Describe $CommandName -Tag IntegrationTests { ($installWarnings -join " ") | Should -Match "Failed to get reboot status" } + It "Names the computer it cannot check" { + # The name came from Resolve-DbaNetworkName only, so for a name that does not resolve it was empty. + ($installWarnings -join " ") | Should -BeLike "*Failed to get reboot status from $($splatUnreachableInstall.SqlInstance)*" + } + It "Completes its progress bar" { # An Id stays on screen when its last record is not a completed one. Windows PowerShell completes its # own bar for loading modules with Id 0 as well, so a completed record somewhere is not enough. From e763c3ad9def7db707c629920b38656368a68191 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:44:52 +0200 Subject: [PATCH 09/15] New-DbaComputerCertificate - Remove the request folder in the finally, also with -WhatIf The request folder and its request.inf are created before any ShouldProcess, but the folder was removed at the end inside a ShouldProcess. So -WhatIf left it in the temp folder, and so did a throw under EnableException and a stopped pipeline. The finally of every computer removes it now. Part of #10776. (do New-DbaComputerCertificate) Co-Authored-By: Claude Opus 5.5 --- public/New-DbaComputerCertificate.ps1 | 13 +++++------ tests/New-DbaComputerCertificate.Tests.ps1 | 26 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/public/New-DbaComputerCertificate.ps1 b/public/New-DbaComputerCertificate.ps1 index f88c512cd42..d18dded0018 100644 --- a/public/New-DbaComputerCertificate.ps1 +++ b/public/New-DbaComputerCertificate.ps1 @@ -628,15 +628,14 @@ function New-DbaComputerCertificate { } } } - if ($PScmdlet.ShouldProcess("local", "Removing all files from $certDir")) { - try { - Remove-Item -Force -Recurse $certDir -ErrorAction SilentlyContinue - } catch { - Stop-Function -Message "Issue removing files from $certDir" -Target $certDir -ErrorRecord $PSItem - } - } } finally { Write-ProgressHelper -Completed + # The request folder is created before any ShouldProcess, so it is removed here and not inside one: with + # -WhatIf, a throw under EnableException and a stopped pipeline it stayed in the temp folder. The secondary + # nodes of a cluster use the certificate exported from the first node, which is held in memory. + if ($certDir) { + Remove-Item -Path $certDir -Recurse -Force -ErrorAction SilentlyContinue + } } } } diff --git a/tests/New-DbaComputerCertificate.Tests.ps1 b/tests/New-DbaComputerCertificate.Tests.ps1 index 6652d86dc88..fe11b3342cc 100644 --- a/tests/New-DbaComputerCertificate.Tests.ps1 +++ b/tests/New-DbaComputerCertificate.Tests.ps1 @@ -686,4 +686,30 @@ RequestType = PKCS10 $openIds | Should -BeNullOrEmpty } } + + Context "With -WhatIf" { + BeforeAll { + # The request folder and its request.inf are written before any ShouldProcess, and the folder was removed + # inside one, so -WhatIf left it in the temp folder. With CaServer and CaName given, nothing asks Active + # Directory, and with -WhatIf certreq never runs. + $requestFoldersBefore = @((Get-ChildItem -Path ([System.IO.Path]::GetTempPath()) -Directory -Filter "certreq-*").Name) + $splatWhatIfCertificate = @{ + CaServer = "nosuchca.dbatools.invalid" + CaName = "NoSuchCA" + WhatIf = $true + } + $whatIfResult = New-DbaComputerCertificate @splatWhatIfCertificate + $whatIfWarning = $WarnVar + $requestFoldersAfter = @((Get-ChildItem -Path ([System.IO.Path]::GetTempPath()) -Directory -Filter "certreq-*").Name) + } + + It "Creates no certificate and does not warn" { + $whatIfResult | Should -BeNullOrEmpty + $whatIfWarning | Should -BeNullOrEmpty + } + + It "Leaves no request folder behind" { + $requestFoldersAfter | Where-Object { $PSItem -notin $requestFoldersBefore } | Should -BeNullOrEmpty + } + } } From 80f3b17acd0123a5039d08d41a4ebea383e97bae Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:46:34 +0200 Subject: [PATCH 10/15] Install-DbaSqlPackage - Download nothing with -WhatIf The download ran before the first ShouldProcess, so -WhatIf downloaded SqlPackage, and the verification afterwards warned that the installation had failed, because nothing was installed. The download has a ShouldProcess of its own now, and -WhatIf skips the verification. Part of #10776. (do Install-DbaSqlPackage) Co-Authored-By: Claude Opus 5.5 --- public/Install-DbaSqlPackage.ps1 | 9 ++++++++- tests/Install-DbaSqlPackage.Tests.ps1 | 29 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/public/Install-DbaSqlPackage.ps1 b/public/Install-DbaSqlPackage.ps1 index cc8b787a07d..f917f451a9a 100644 --- a/public/Install-DbaSqlPackage.ps1 +++ b/public/Install-DbaSqlPackage.ps1 @@ -212,7 +212,8 @@ function Install-DbaSqlPackage { } # Download if needed - if (-not (Test-Path -Path $LocalFile) -or $Force) { + # The download is real work, so -WhatIf has to skip it too, and not only the installation. + if ((-not (Test-Path -Path $LocalFile) -or $Force) -and $Pscmdlet.ShouldProcess($url, "Download SqlPackage to $LocalFile")) { try { Write-Progress -Activity "Installing SqlPackage" -Status "Starting download from Microsoft..." -PercentComplete 20 Write-Message -Level Verbose -Message "Downloading SqlPackage from $url" @@ -290,6 +291,12 @@ function Install-DbaSqlPackage { } } + # With -WhatIf nothing was downloaded or installed, so there is nothing to verify. The verification warned that + # the installation failed when SqlPackage was not there yet. + if ($WhatIfPreference) { + return + } + Write-Progress -Activity "Installing SqlPackage" -Status "Verifying installation..." -PercentComplete 90 # Verify installation diff --git a/tests/Install-DbaSqlPackage.Tests.ps1 b/tests/Install-DbaSqlPackage.Tests.ps1 index cbf1433d5ef..dd80054997e 100644 --- a/tests/Install-DbaSqlPackage.Tests.ps1 +++ b/tests/Install-DbaSqlPackage.Tests.ps1 @@ -235,4 +235,33 @@ Describe $CommandName -Tag IntegrationTests { $openIds | Should -BeNullOrEmpty } } + + Context "With -WhatIf" { + BeforeAll { + # Force skips the check for an existing installation, so the command goes on to the download. The download + # ran before the first ShouldProcess, so -WhatIf downloaded SqlPackage, and the verification afterwards warned + # that the installation had failed. + $splatWhatIfInstall = @{ + Path = "$TestDrive\WhatIfInstall" + LocalFile = "$TestDrive\whatif-sqlpackage.zip" + Force = $true + WhatIf = $true + } + $whatIfResult = Install-DbaSqlPackage @splatWhatIfInstall + $whatIfWarning = $WarnVar + } + + It "Downloads nothing" { + Test-Path -Path $splatWhatIfInstall.LocalFile | Should -BeFalse + } + + It "Installs nothing" { + Test-Path -Path $splatWhatIfInstall.Path | Should -BeFalse + } + + It "Returns nothing and does not warn" { + $whatIfResult | Should -BeNullOrEmpty + $whatIfWarning | Should -BeNullOrEmpty + } + } } From 9e99e495f63fca8cfd380714bfb76231fa9821a0 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 14:49:29 +0200 Subject: [PATCH 11/15] Save-DbaKbUpdate - Report a failed download with Stop-Function Invoke-WebRequest reports a failed download as an error that ends only its own statement, so the command neither warned nor threw under EnableException: the caller got a plain error record and no output. The download is in a try now, and the catch reports the link with Stop-Function and goes on with the next link. Part of #10776. (do Save-DbaKbUpdate) Co-Authored-By: Claude Opus 5.5 --- public/Save-DbaKbUpdate.ps1 | 30 +++++++++++++++----------- tests/Save-DbaKbUpdate.Tests.ps1 | 37 ++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/public/Save-DbaKbUpdate.ps1 b/public/Save-DbaKbUpdate.ps1 index f91203c4a8b..327af6ed628 100644 --- a/public/Save-DbaKbUpdate.ps1 +++ b/public/Save-DbaKbUpdate.ps1 @@ -147,11 +147,22 @@ function Save-DbaKbUpdate { $file = "$Path$([IO.Path]::DirectorySeparatorChar)$fileName" } - if (-not $UseWebRequest -and (Get-Command Start-BitsTransfer -ErrorAction Ignore)) { - try { - Start-BitsTransfer -Source $link -Destination $file -ErrorAction Stop - } catch { - Write-Message -Level Verbose -Message "Start-BitsTransfer failed, falling back to Invoke-WebRequest: $PSItem" + # A failed download is reported by Stop-Function. Invoke-WebRequest reports it as an error that ends only its own + # statement, so the command neither warned nor threw under EnableException, and the caller got a plain error. + try { + if (-not $UseWebRequest -and (Get-Command Start-BitsTransfer -ErrorAction Ignore)) { + try { + Start-BitsTransfer -Source $link -Destination $file -ErrorAction Stop + } catch { + Write-Message -Level Verbose -Message "Start-BitsTransfer failed, falling back to Invoke-WebRequest: $PSItem" + Write-Progress -Activity "Downloading $fileName" -Id 1 + try { + Invoke-TlsWebRequest -Uri $link -OutFile $file -ErrorAction Stop + } finally { + Write-Progress -Activity "Downloading $fileName" -Id 1 -Completed + } + } + } else { Write-Progress -Activity "Downloading $fileName" -Id 1 try { Invoke-TlsWebRequest -Uri $link -OutFile $file -ErrorAction Stop @@ -159,13 +170,8 @@ function Save-DbaKbUpdate { Write-Progress -Activity "Downloading $fileName" -Id 1 -Completed } } - } else { - Write-Progress -Activity "Downloading $fileName" -Id 1 - try { - Invoke-TlsWebRequest -Uri $link -OutFile $file -ErrorAction Stop - } finally { - Write-Progress -Activity "Downloading $fileName" -Id 1 -Completed - } + } catch { + Stop-Function -Message "Failed to download $link" -ErrorRecord $PSItem -Target $link -Continue } if (Test-Path -Path $file) { Get-ChildItem -Path $file diff --git a/tests/Save-DbaKbUpdate.Tests.ps1 b/tests/Save-DbaKbUpdate.Tests.ps1 index cccc5748fb5..ca7d124334f 100644 --- a/tests/Save-DbaKbUpdate.Tests.ps1 +++ b/tests/Save-DbaKbUpdate.Tests.ps1 @@ -136,14 +136,15 @@ Describe $CommandName -Tag IntegrationTests { # command would go on to its next line anyway; with -ErrorAction Stop of the caller the failure # throws out of the command, which is when the bar was left on screen. The runspace imports the # manifest: an import of the psm1 without a command line skips the type data. + # The command reports the failed download with Stop-Function now, so it throws under EnableException. $failingUpdate = [PSCustomObject]@{ Link = "https://progressleak$(Get-Random).invalid/sqlserver2022-kb0000000-x64_0000000000000000.exe" } $splatFailingDownload = @{ - InputObject = $failingUpdate - Path = $tempPath - UseWebRequest = $true - ErrorAction = "Stop" + InputObject = $failingUpdate + Path = $tempPath + UseWebRequest = $true + EnableException = $true } $downloadRunspace = [runspacefactory]::CreateRunspace() $downloadRunspace.Open() @@ -178,4 +179,32 @@ Describe $CommandName -Tag IntegrationTests { $openIds | Should -BeNullOrEmpty } } + + Context "When the download fails without EnableException" { + BeforeAll { + # The failed download ended only the statement of Invoke-WebRequest: the caller got a plain error record and + # neither a warning nor a result. The link points to the top level domain invalid, which never resolves. + $quietUpdate = [PSCustomObject]@{ + Link = "https://quietfail$(Get-Random).invalid/sqlserver2022-kb0000000-x64_0000000000000000.exe" + } + $splatQuietDownload = @{ + InputObject = $quietUpdate + Path = $tempPath + UseWebRequest = $true + WarningAction = "SilentlyContinue" + ErrorVariable = "quietErrors" + ErrorAction = "SilentlyContinue" + } + $quietResult = Save-DbaKbUpdate @splatQuietDownload + $quietWarning = $WarnVar + } + + It "Warns about the failed download" { + ($quietWarning -join " ") | Should -BeLike "*Failed to download $($quietUpdate.Link)*" + } + + It "Returns nothing" { + $quietResult | Should -BeNullOrEmpty + } + } } \ No newline at end of file From 2b00530cc5492836c9a2c5b535a1d70dbb50d8cb Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 15:57:33 +0200 Subject: [PATCH 12/15] Invoke-DbaDbLogShipRecovery - Create the folder of the database below the share before the log shipping setup The setup creates that folder from the test session and then asks the instance whether it exists. An instance that reaches the share over the network sees a new folder only some seconds later. (do Invoke-DbaDbLogShipRecovery) Co-Authored-By: Claude Opus 5.5 --- tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 b/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 index 5402520bbd1..1d6f0145d60 100644 --- a/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 +++ b/tests/Invoke-DbaDbLogShipRecovery.Tests.ps1 @@ -106,6 +106,15 @@ Describe $CommandName -Tag IntegrationTests { $null = New-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Name $primaryDatabase -RecoveryModel Full $null = Backup-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Database $primaryDatabase -Path $sharedPath -Type Full + # The command creates the folder of the database below the shared path from this session and then asks the + # instance whether it exists. An instance that reaches the share over the network can see a new folder only + # some seconds later, and the setup then fails with "is not valid or can't be reached". So the folder is + # created here, and the setup starts once the instance sees it. + $null = New-Item -Path (Join-Path -Path $sharedPath -ChildPath $primaryDatabase) -ItemType Directory + $folderWatch = [System.Diagnostics.Stopwatch]::StartNew() + while (-not (Test-DbaPath -SqlInstance $TestConfig.InstanceSingle -Path "$sharedPathUnc\$primaryDatabase") -and $folderWatch.Elapsed.TotalSeconds -lt 60) { + Start-Sleep -Seconds 1 + } $splatLogShipping = @{ SourceSqlInstance = $TestConfig.InstanceSingle DestinationSqlInstance = $TestConfig.InstanceSingle From 136a2fc94ad04b52e937aeea9f19b0028c6a196b Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 16:20:10 +0200 Subject: [PATCH 13/15] New-DbaComputerCertificate - Test the removal of the request folder after a stopped pipeline The -WhatIf case of #10776 was no defect: -WhatIf sets $WhatIfPreference in the function, so New-Item and Set-Content never create the request folder. The real case is a stopped pipeline or a throw, which skipped the removal after the output. The existing pipeline-stop context checks it now; the -WhatIf context stays as a guard. (do New-DbaComputerCertificate) Co-Authored-By: Claude Opus 5.5 --- tests/New-DbaComputerCertificate.Tests.ps1 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/New-DbaComputerCertificate.Tests.ps1 b/tests/New-DbaComputerCertificate.Tests.ps1 index fe11b3342cc..a78fead44f6 100644 --- a/tests/New-DbaComputerCertificate.Tests.ps1 +++ b/tests/New-DbaComputerCertificate.Tests.ps1 @@ -678,6 +678,13 @@ RequestType = PKCS10 $firstOnlyCert.Thumbprint | Should -Not -BeNullOrEmpty } + It "Removes its request folder" { + # The removal sat after the output, so the stopped command left the folder behind; it is in a finally now. + # The AfterAll above still removes it, so that a regression does not leave it in the temp folder. + $firstOnlyFolder | Should -Not -BeNullOrEmpty + Test-Path -Path $firstOnlyFolder | Should -BeFalse + } + It "Completes its progress bar" { # An Id stays on screen when its last record is not a completed one. Windows PowerShell completes its # own bar for loading modules with Id 0 as well, so a completed record somewhere is not enough. @@ -689,9 +696,10 @@ RequestType = PKCS10 Context "With -WhatIf" { BeforeAll { - # The request folder and its request.inf are written before any ShouldProcess, and the folder was removed - # inside one, so -WhatIf left it in the temp folder. With CaServer and CaName given, nothing asks Active - # Directory, and with -WhatIf certreq never runs. + # The request folder and its request.inf are written before any ShouldProcess. With -WhatIf, New-Item and + # Set-Content skip that through $WhatIfPreference, so nothing may be left behind, also now that the folder + # is removed in a finally. With CaServer and CaName given, nothing asks Active Directory, and with -WhatIf + # certreq never runs. $requestFoldersBefore = @((Get-ChildItem -Path ([System.IO.Path]::GetTempPath()) -Directory -Filter "certreq-*").Name) $splatWhatIfCertificate = @{ CaServer = "nosuchca.dbatools.invalid" From f12835092b365ae2161423bd06863b0bfe87efc2 Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Sat, 17 Oct 2026 16:45:12 +0200 Subject: [PATCH 14/15] New-DbaComputerCertificate - Guess the full name of a computer from its own name For a name that does not resolve, the full name was guessed from $ComputerName, which holds all the names of the call, so two names gave "first second.domain". (do New-DbaComputerCertificate) Co-Authored-By: Claude Opus 5.5 --- public/New-DbaComputerCertificate.ps1 | 3 ++- tests/New-DbaComputerCertificate.Tests.ps1 | 25 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/public/New-DbaComputerCertificate.ps1 b/public/New-DbaComputerCertificate.ps1 index d18dded0018..1acc3fc59e4 100644 --- a/public/New-DbaComputerCertificate.ps1 +++ b/public/New-DbaComputerCertificate.ps1 @@ -381,7 +381,8 @@ function New-DbaComputerCertificate { $resolved = Resolve-DbaNetworkName -ComputerName $computer.ComputerName -WarningAction SilentlyContinue if (-not $resolved) { - $fqdn = "$ComputerName.$env:USERDNSDOMAIN" + # The name of this computer: $ComputerName holds all of them, and two names gave "a b.domain". + $fqdn = "$($computer.ComputerName).$env:USERDNSDOMAIN" Write-Message -Level Warning -Message "Server name cannot be resolved. Guessing it's $fqdn" } else { $fqdn = $resolved.fqdn diff --git a/tests/New-DbaComputerCertificate.Tests.ps1 b/tests/New-DbaComputerCertificate.Tests.ps1 index a78fead44f6..a513a97e696 100644 --- a/tests/New-DbaComputerCertificate.Tests.ps1 +++ b/tests/New-DbaComputerCertificate.Tests.ps1 @@ -720,4 +720,29 @@ RequestType = PKCS10 $requestFoldersAfter | Where-Object { $PSItem -notin $requestFoldersBefore } | Should -BeNullOrEmpty } } + + Context "With two computers whose names cannot be resolved" { + BeforeAll { + # For a name that does not resolve, the command guesses the full name from the domain of this session. The + # guess was built from $ComputerName, which holds all the names, so two names gave "first second.domain". + # -WhatIf keeps the command from doing anything with the guessed names. + $unresolvedNames = @("nohostone$(Get-Random)", "nohosttwo$(Get-Random)") + $splatUnresolved = @{ + ComputerName = $unresolvedNames + CaServer = "nosuchca.dbatools.invalid" + CaName = "NoSuchCA" + WhatIf = $true + WarningAction = "SilentlyContinue" + } + $null = New-DbaComputerCertificate @splatUnresolved + $unresolvedWarnings = @($WarnVar | ForEach-Object { "$PSItem" }) + } + + It "Guesses the name of each computer from its own name" { + $guesses = @($unresolvedWarnings | Where-Object { $PSItem -like "*Guessing it's*" }) + $guesses.Count | Should -Be 2 + $guesses[0] | Should -BeLike "*Guessing it's $($unresolvedNames[0]).$env:USERDNSDOMAIN" + $guesses[1] | Should -BeLike "*Guessing it's $($unresolvedNames[1]).$env:USERDNSDOMAIN" + } + } } From fffabc709d36d01d641b59e70444fdc1bc9a37be Mon Sep 17 00:00:00 2001 From: Andreas Jordan Date: Mon, 19 Oct 2026 10:38:31 +0200 Subject: [PATCH 15/15] Invoke-DbaAdvancedRestore - Never restore with -VerifyOnly The branch for -KeepCDC and -ErrorBrokerConversations and the branch for page restores came before the one for -VerifyOnly and execute a restore statement of their own. Now that the verification goes on to the last backup file, it reached the one that recovers: two full backups with -VerifyOnly -KeepCDC verified the first and restored the second, with -WithReplace over an existing database. A single full backup took that branch before as well. Both branches now leave out -VerifyOnly. The new context verifies two full backups with -KeepCDC under a new database name and checks that both are verified and no database was created. (do Invoke-DbaAdvancedRestore) Co-Authored-By: Claude Opus 5.5 --- public/Invoke-DbaAdvancedRestore.ps1 | 7 ++- tests/Invoke-DbaAdvancedRestore.Tests.ps1 | 56 +++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/public/Invoke-DbaAdvancedRestore.ps1 b/public/Invoke-DbaAdvancedRestore.ps1 index d559e9617a4..c2a63cee0f0 100644 --- a/public/Invoke-DbaAdvancedRestore.ps1 +++ b/public/Invoke-DbaAdvancedRestore.ps1 @@ -506,7 +506,10 @@ function Invoke-DbaAdvancedRestore { if ($ExecuteAs -ne "" -and $BackupCnt -eq 1) { $executeAsLogin = $ExecuteAs.Replace("'", "''") } - if (($KeepCDC -or $ErrorBrokerConversations) -and $restore.NoRecovery -eq $false) { + # -VerifyOnly never takes the two branches that execute a restore statement of their own. Since + # the verification goes on to the last backup file, it reaches the one that recovers, which the + # first branch would restore. + if (-not $VerifyOnly -and ($KeepCDC -or $ErrorBrokerConversations) -and $restore.NoRecovery -eq $false) { $script = $restore.Script($server) $withOptions = @() if ($KeepCDC) { $withOptions += "KEEP_CDC" } @@ -524,7 +527,7 @@ function Invoke-DbaAdvancedRestore { $null = $server.ConnectionContext.ExecuteNonQuery($script) Write-Progress -id 1 -activity "Restoring $database to $SqlInstance - Backup $BackupCnt of $($Backups.count)" -status "Complete" -Completed } - } elseif ($null -ne $Pages -and $action -eq 'Database') { + } elseif (-not $VerifyOnly -and $null -ne $Pages -and $action -eq "Database") { $script = $restore.Script($server) $script = $script -replace "] FROM", "] PAGE='$pages' FROM" if ($true -ne $OutputScriptOnly) { diff --git a/tests/Invoke-DbaAdvancedRestore.Tests.ps1 b/tests/Invoke-DbaAdvancedRestore.Tests.ps1 index 182f235906c..1eccf9b4c0f 100644 --- a/tests/Invoke-DbaAdvancedRestore.Tests.ps1 +++ b/tests/Invoke-DbaAdvancedRestore.Tests.ps1 @@ -340,6 +340,62 @@ Describe $CommandName -Tag IntegrationTests { } } + Context "When -VerifyOnly with -KeepCDC verifies two full backups" { + BeforeAll { + # We want to run all commands in the BeforeAll block with EnableException to ensure that the test fails if the setup fails. + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + # Two full backups of the same database in a folder of their own. The instance creates the folder, so it sees it + # right away. + $twoFullPath = "$($TestConfig.Temp)\$CommandName-twofull-$(Get-Random)" + $null = New-DbaDirectory -SqlInstance $TestConfig.InstanceSingle -Path $twoFullPath + foreach ($twoFullFile in "first.bak", "second.bak") { + $splatTwoFull = @{ + SqlInstance = $TestConfig.InstanceSingle + Database = $restoreDbName + BackupDirectory = $twoFullPath + FilePath = $twoFullFile + Type = "Full" + } + $null = Backup-DbaDatabase @splatTwoFull + } + + # The history Restore-DbaDatabase hands over for -VerifyOnly, which skips the selection of a restore: both full + # backups are verified, and the second is the last backup file, the one that recovers. The database and its + # files get a new name, so a restore shows as a database that was created. + $verifyDbName = "dbatoolsci_advrestoreverify_$(Get-Random)" + $twoFullHistory = Get-DbaBackupInformation -SqlInstance $TestConfig.InstanceSingle -Path $twoFullPath | + Format-DbaBackupInformation -ReplaceDatabaseName $verifyDbName -ReplaceDbNameInFile + + # We want to run all commands outside of the BeforeAll block without EnableException to be able to test for specific warnings. + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + + # The branch for -KeepCDC came before the one for -VerifyOnly and restored the backup file that recovers. + $keepCdcResult = $twoFullHistory | Invoke-DbaAdvancedRestore -SqlInstance $TestConfig.InstanceSingle -VerifyOnly -KeepCDC + $keepCdcWarning = $WarnVar + $verifyDb = Get-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Database $verifyDbName + } + + AfterAll { + # We want to run all commands in the AfterAll block with EnableException to ensure that the test fails if the cleanup fails. + $PSDefaultParameterValues["*-Dba*:EnableException"] = $true + + $null = Remove-DbaDatabase -SqlInstance $TestConfig.InstanceSingle -Database $verifyDbName -ErrorAction SilentlyContinue + Remove-Item -Path $twoFullPath -Recurse -ErrorAction SilentlyContinue + + $PSDefaultParameterValues.Remove("*-Dba*:EnableException") + } + + It "Verifies both backup files" { + $keepCdcWarning | Should -BeNullOrEmpty + @($keepCdcResult | Where-Object { $PSItem -is [string] }) | Should -Be @("Verify successful", "Verify successful") + } + + It "Restores no database" { + $verifyDb | Should -BeNullOrEmpty + } + } + Context "When the restore of a backup file fails" { BeforeAll { $PSDefaultParameterValues["*-Dba*:EnableException"] = $true