From c65acdfccba4a104f24c166c83277cb9d9eada9e Mon Sep 17 00:00:00 2001 From: Sander Stad Date: Tue, 26 Feb 2019 12:26:04 +0100 Subject: [PATCH] Fix masking commands (#5071) * Added check for unique indexes * Added procedure to make up what the unique indexes are build with Added check if excluded column is in a unique index * Added table * Added extra options for data generation * Removed unused rowcount Added unique value dictionary creation * Added format for masking types * Added support for using formatting of values * Added check if any masking info is present * Implemented unique index support * Renamed variable * Revert "Renamed variable" This reverts commit 3f7adb8899e54203a26bbe5d242b7ef298154b76. * Renamed variable and applied formatiing * Fixed errors in unique value generation * Added variable for the unique value columns Changed the if statement for the unique columns * Explicitly reset the new value variable * Moved variables to make sure they get cleared in the loop * Removed date selection Added section for finance. Can be used for other types that support max * Revert "Removed date selection" This reverts commit 6ec4f168c7954cb65aab06c30d9d654a104bde19. * Added section for support of max * Added part for finance * Removed finance from other section * Changed if statement to switch * Changed switch back to if statement Removed sections Added finance to other section * Removed section for finance * Added checks for supported data types, masking types and sub types * Moved retrieval of method and properties to begin Changed if statements for types * Changed message of unsupported types * Added new known types Added format key to known types * Added extra known types * Implemented support for text and ntext data types * Added ntext and text data types * Added decimal data type * Moved data retrieval down * Added types and synonyms Rearranged the types to be in alphabetical order * Added types * Added types to automatically assign the right maskingtype * Improved the check for supported data types * Fixed bug missing curly bracket * Moved the retrieval of the data Implemented selective columns for data retrieval * Changed back to SELECT * instead of selective columns * Changed the SELECT * to be selective columns from table and not object * Moved data retrieval Changed where statement build-up * Removed debug query outputs * Moved where build-up * Added special conversion to date and datetime columns * Fixed bug with missing comma * Added error record * Removed unneccesary if statement Fixed bug with TEXT and NTEXT cast in WHERE clause * Implemented handling of temporal values generation Formatted code * Fixed bool handling * Added decimal type * Added fax synonyms * Fixed bool handling * Updates test should statement * Changed back the should statement * Enabled rdp * Changed rdp password * Formatted test * Reset appveyor.yml to original settings * Changed instance * Changed instance * Changed instance back * Added credential * 0.9.773 * change instance * Make Backup-DbaDatabase use Database and ExcludeDatabase when piping (#5069) * fixes #5044 * fix one test * in instead of contains * 0.9.773 changelog (#5118) * Pass Login values to Copy-DbaLogin (#5120) Call to `Copy-DbaLogin` simply didn't pass the values. Fixes #5119 * Updated test to use other instance * Enabled RDP * Set RDP to be started before tests * Removed RDP option again * Added fix for datetime * Formatted code * Added extra check for temporal tables * Changed test back to 2008 instance --- appveyor.yml | 2 +- bin/datamasking/columntypes.json | 117 ++++++++--- functions/Invoke-DbaDbDataMasking.ps1 | 180 ++++++++++++++-- functions/New-DbaDbMaskingConfig.ps1 | 269 +++++++++++++++++++++--- tests/Invoke-DbaDbDataMasking.Tests.ps1 | 25 +-- tests/New-DbaDbMaskingConfig.Tests.ps1 | 2 +- 6 files changed, 509 insertions(+), 86 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index f66ff3e1d5..4d6343589d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,7 +24,7 @@ shallow_clone: true environment: environment: development version: 0.9.$(appveyor_build_number) - #appveyor_rdp_password: 2odCuiKmYiem + appveyor_rdp_password: 2odCuiKmYiem azurepasswd: secure: ZnF3fWSDfHraMCWlHaekvWrXf3sDqY5M28HMK4236PBbNSoqP29wEhsWMQioSSYGomzgIp9vuiwR8Fc9ViNLoqq0bVcErxEojBFTaPMEzOg2ZwO9OnOTiuUEc5JkoLBv6rEBBWef/DvkFfhr1r0K0xQu6OAPYHVTCRajTZbBRNfCTUM2X2o41t+cSa7681rtnJQnB/8cAfVVnPtJ+97s8w== azurelegacypasswd: diff --git a/bin/datamasking/columntypes.json b/bin/datamasking/columntypes.json index f536a9a164..8d6ac5277a 100644 --- a/bin/datamasking/columntypes.json +++ b/bin/datamasking/columntypes.json @@ -1,49 +1,54 @@ [ { - "TypeName": "Firstname", + "TypeName": "Address", "Synonym": [ - "Firstname", - "Forename" + "Address", + "Location", + "Headquarters", + "Street" ] }, { - "TypeName": "Lastname", + "TypeName": "Bic", "Synonym": [ - "Lastname", - "Surname" + "Bic", + "BicAddress" ] }, { - "TypeName": "Fullname", + "TypeName": "Bitcoin", "Synonym": [ - "Fullname", - "TypeName" + "Bitcoin", + "BitcoinAddress" ] }, { - "TypeName": "Phone", + "TypeName": "Email", "Synonym": [ - "Phonenumber", - "Phone", - "Telephone", - "TelephoneNumber" + "Email", + "E-mail", + "EmailAddress" ] }, { - "TypeName": "Address", + "TypeName": "Ethereum", "Synonym": [ - "Address", - "Location", - "Headquarters", - "Street" + "Ethereum", + "EthereumAddress" ] }, { - "TypeName": "Zipcode", + "TypeName": "Creditcard", "Synonym": [ - "Zipcode", - "Zip", - "Postalcode" + "Creditcard", + "CreditcardNumber" + ] + }, + { + "TypeName": "CreditcardCVV", + "Synonym": [ + "Cvv", + "CreditcardCvv" ] }, { @@ -53,9 +58,10 @@ ] }, { - "TypeName": "State", + "TypeName": "Company", "Synonym": [ - "State" + "Company", + "CompanyName" ] }, { @@ -70,6 +76,34 @@ "CountryCode" ] }, + { + "TypeName": "Firstname", + "Synonym": [ + "Firstname", + "Forename" + ] + }, + { + "TypeName": "Fullname", + "Synonym": [ + "Fullname", + "TypeName" + ] + }, + { + "TypeName": "Iban", + "Synonym": [ + "Iban", + "IbanNumber" + ] + }, + { + "TypeName": "Lastname", + "Synonym": [ + "Lastname", + "Surname" + ] + }, { "TypeName": "Latitude", "Synonym": [ @@ -85,19 +119,44 @@ ] }, { - "TypeName": "Creditcard", + "TypeName": "Phone", "Synonym": [ - "Creditcard", - "CreditcardNumber" + "Fax", + "FaxNumber", + "Phonenumber", + "Phone", + "Telephone", + "TelephoneNumber" + ] + }, + { + "TypeName": "State", + "Synonym": [ + "State" + ] + }, + { + "TypeName": "StateAbbr", + "Synonym": [ + "StateAbbreviation", + "StateCode" ] }, { "TypeName": "Username", "Synonym": [ "Login", - "LoginId", + "LoginId", "User", "UserId" ] + }, + { + "TypeName": "Zipcode", + "Synonym": [ + "Zipcode", + "Zip", + "Postalcode" + ] } ] \ No newline at end of file diff --git a/functions/Invoke-DbaDbDataMasking.ps1 b/functions/Invoke-DbaDbDataMasking.ps1 index c7c7dcb0af..db2c0cc443 100644 --- a/functions/Invoke-DbaDbDataMasking.ps1 +++ b/functions/Invoke-DbaDbDataMasking.ps1 @@ -132,6 +132,14 @@ function Invoke-DbaDbDataMasking { # Create the faker objects Add-Type -Path (Resolve-Path -Path "$script:PSModuleRoot\bin\datamasking\Bogus.dll") $faker = New-Object Bogus.Faker($Locale) + + $supportedDataTypes = 'bit', 'bool', 'char', 'date', 'datetime', 'datetime2', 'decimal', 'int', 'money', 'nchar', 'ntext', 'nvarchar', 'smalldatetime', 'text', 'time', 'uniqueidentifier', 'userdefineddatatype', 'varchar' + + $supportedFakerMaskingTypes = ($faker | Get-Member -MemberType Property | Select-Object Name -ExpandProperty Name) + + $supportedFakerSubTypes = ($faker | Get-Member -MemberType Property) | ForEach-Object { ($faker.$($_.Name)) | Get-Member -MemberType Method | Where-Object {$_.Name -notlike 'To*' -and $_.Name -notlike 'Get*' -and $_.Name -notlike 'Trim*' -and $_.Name -notin 'Add', 'Equals', 'CompareTo', 'Clone', 'Contains', 'CopyTo', 'EndsWith', 'IndexOf', 'IndexOfAny', 'Insert', 'IsNormalized', 'LastIndexOf', 'LastIndexOfAny', 'Normalize', 'PadLeft', 'PadRight', 'Remove', 'Replace', 'Split', 'StartsWith', 'Substring', 'Letter', 'Lines', 'Paragraph', 'Paragraphs', 'Sentence', 'Sentences'} | Select-Object name -ExpandProperty Name } + + $supportedFakerSubTypes += "Date" } process { @@ -182,18 +190,23 @@ function Invoke-DbaDbDataMasking { } if ($Database) { - $dbs = Get-DbaDatabase -SqlInstance $server -Database $Database + $dbs = Get-DbaDatabase -SqlInstance $server -SqlCredential $SqlCredential -Database $Database } else { - $dbs = Get-DbaDatabase -SqlInstance $server -Database $tables.Name + $dbs = Get-DbaDatabase -SqlInstance $server -SqlCredential $SqlCredential -Database $tables.Name } $sqlconn = $server.ConnectionContext.SqlConnectionObject.PsObject.Copy() $sqlconn.Open() foreach ($db in $dbs) { + $stepcounter = $nullmod = 0 + foreach ($tableobject in $tables.Tables) { - if ($tableobject.Name -in $ExcludeTable -or ($Table -and $tableobject.Name -notin $Table)) { + $uniqueValues = @() + $uniqueValueColumns = @() + + if ($tableobject.Name -in $ExcludeTable) { Write-Message -Level Verbose -Message "Skipping $($tableobject.Name) because it is explicitly excluded" continue } @@ -201,17 +214,81 @@ function Invoke-DbaDbDataMasking { if ($tableobject.Name -notin $db.Tables.Name) { Stop-Function -Message "Table $($tableobject.Name) is not present in $db" -Target $db -Continue } + + $dbTable = $db.Tables | Where-Object {$_.Schema -eq $tableobject.Schema -and $_.Name -eq $tableobject.Name} + try { if (-not (Test-Bound -ParameterName Query)) { - $query = "SELECT * FROM [$($tableobject.Schema)].[$($tableobject.Name)]" + $columnString = "[" + (($dbTable.Columns | Where-Object DataType -in $supportedDataTypes | Select-Object Name -ExpandProperty Name) -join "],[") + "]" + $query = "SELECT $($columnString) FROM [$($tableobject.Schema)].[$($tableobject.Name)]" } $data = $server.Databases[$($db.Name)].Query($query) | ConvertTo-DbaDataTable } catch { Stop-Function -Message "Failure retrieving the data from table $($tableobject.Name)" -Target $Database -ErrorRecord $_ -Continue } + # Check if the table contains unique indexes + if ($tableobject.HasUniqueIndex) { + + # Loop through the rows and generate a unique value for each row + Write-Message -Level Verbose -Message "Generating unique values for $($tableobject.Name)" + + for ($i = 0; $i -lt $data.Rows.Count; $i++) { + $rowValue = New-Object PSCustomObject + + # Loop through each of the unique indexes + foreach ($index in ($db.Tables[$($tableobject.Name)].Indexes | Where-Object IsUnique -eq $true )) { + + # Loop through the index columns + foreach ($indexColumn in $index.IndexedColumns) { + # Get the column mask info + $columnMaskInfo = $tableobject.Columns | Where-Object Name -eq $indexColumn.Name + + # Generate a new value + $newValue = $faker.$($columnMaskInfo.MaskingType).$($columnMaskInfo.SubType)() + + # Check if the value is already present as a property + if (($rowValue | Get-Member -MemberType NoteProperty).Name -notcontains $indexColumn.Name) { + $rowValue | Add-Member -Name $indexColumn.Name -Type NoteProperty -Value $newValue + } + + } + + # To be sure the values are unique, loop as long as long as needed to generate a unique value + while (($uniqueValues | Select-Object -Property ($rowValue | Get-Member -MemberType NoteProperty | Select-Object -ExpandProperty Name)) -match $rowValue) { + + $rowValue = New-Object PSCustomObject + + # Loop through the index columns + foreach ($indexColumn in $index.IndexedColumns) { + # Get the column mask info + $columnMaskInfo = $tableobject.Columns | Where-Object Name -eq $indexColumn.Name + + # Generate a new value + $newValue = $faker.$($columnMaskInfo.MaskingType).$($columnMaskInfo.SubType)() + + # Check if the value is already present as a property + if (($rowValue | Get-Member -MemberType NoteProperty).Name -notcontains $indexColumn.Name) { + $rowValue | Add-Member -Name $indexColumn.Name -Type NoteProperty -Value $newValue + $uniqueValueColumns += $indexColumn.Name + } + + } + } + + } + + # Add the row value to the array + $uniqueValues += $rowValue + + } + + } + + $uniqueValueColumns = $uniqueValueColumns | Select-Object -Unique + $sqlconn.ChangeDatabase($db.Name) - + $deterministicColumns = $tables.Tables.Columns | Where-Object Deterministic -eq $true $tablecolumns = $tableobject.Columns @@ -220,6 +297,10 @@ function Invoke-DbaDbDataMasking { } if ($ExcludeColumn) { + if ([string]$uniqueIndex.Columns -match ($ExcludeColumn -join "|")) { + Stop-Function -Message "Column present in -ExcludeColumn cannot be excluded because it's part of an unique index" -Target $ExcludeColumn -Continue + } + $tablecolumns = $tablecolumns | Where-Object Name -notin $ExcludeColumn } @@ -229,18 +310,45 @@ function Invoke-DbaDbDataMasking { } if ($Pscmdlet.ShouldProcess($instance, "Masking $($tablecolumns.Name -join ', ') in $($data.Rows.Count) rows in $($db.Name).$($tableobject.Schema).$($tableobject.Name)")) { + $transaction = $sqlconn.BeginTransaction() $elapsed = [System.Diagnostics.Stopwatch]::StartNew() Write-ProgressHelper -StepNumber ($stepcounter++) -TotalSteps $tables.Tables.Count -Activity "Masking data" -Message "Updating $($data.Rows.Count) rows in $($tableobject.Schema).$($tableobject.Name) in $($db.Name) on $instance" # Loop through each of the rows and change them + $rowNumber = 0 + foreach ($row in $data.Rows) { $updates = $wheres = @() + $newValue = $null foreach ($columnobject in $tablecolumns) { + + if ($columnobject.ColumnType -notin $supportedDataTypes) { + Stop-Function -Message "Unsupported data type '$($columnobject.ColumnType)' for column $($columnobject.Name)" -Target $columnobject -Continue + } + + if ($columnobject.MaskingType -notin $supportedFakerMaskingTypes) { + Stop-Function -Message "Unsupported masking type '$($columnobject.MaskingType)' for column $($columnobject.Name)" -Target $columnobject -Continue + } + + if ($columnobject.SubType -notin $supportedFakerSubTypes) { + Stop-Function -Message "Unsupported masking sub type '$($columnobject.SubType)' for column $($columnobject.Name)" -Target $columnobject -Continue + } + if ($columnobject.Nullable -and (($nullmod++) % $ModulusFactor -eq 0)) { $newValue = $null + } elseif ($tableobject.HasUniqueIndex -and $columnobject.Name -in $uniqueValueColumns) { + + if ($uniqueValues.Count -lt 1) { + Stop-Function -Message "Could not find any unique values in dictionary" -Target $tableobject + return + } + + $newValue = $uniqueValues[$rowNumber].$($columnobject.Name) + } else { + # make sure max is good if ($MaxValue) { if ($columnobject.MaxValue -le $MaxValue) { @@ -395,9 +503,13 @@ function Invoke-DbaDbDataMasking { $faker.System.Random.Bool() } { - $psitem -in 'name', 'address', 'finance' + $psitem -in 'address', 'commerce', 'company', 'context', 'database', 'date', 'finance', 'hacker', 'hashids', 'image', 'internet', 'lorem', 'name', 'person', 'phone', 'random', 'rant', 'system' } { - $faker.$($columnobject.MaskingType).$($columnobject.SubType)() + if ($columnobject.Format) { + $faker.$($columnobject.MaskingType).$($columnobject.SubType)("$($columnobject.Format)") + } else { + $faker.$($columnobject.MaskingType).$($columnobject.SubType)() + } } default { if ($max -eq -1) { @@ -410,7 +522,11 @@ function Invoke-DbaDbDataMasking { $faker.System.Random.Bool() } else { try { - $faker.$($columnobject.MaskingType).$($columnobject.SubType)() + if ($columnobject.Format) { + $faker.$($columnobject.MaskingType).$($columnobject.SubType)("$($columnobject.Format)") + } else { + $faker.$($columnobject.MaskingType).$($columnobject.SubType)() + } } catch { $faker.Random.String2($max, $charstring) } @@ -431,13 +547,22 @@ function Invoke-DbaDbDataMasking { } else { $updates += "[$($columnobject.Name)] = '$newValue'" } - } elseif ($columnobject.ColumnType -match 'int') { if ($null -eq $newValue -and $columnobject.Nullable) { $updates += "[$($columnobject.Name)] = NULL" } else { $updates += "[$($columnobject.Name)] = $newValue" } + } elseif ($columnobject.ColumnType -in 'bit', 'bool') { + if ($null -eq $newValue -and $columnobject.Nullable) { + $updates += "[$($columnobject.Name)] = NULL" + } else { + if ($newValue) { + $updates += "[$($columnobject.Name)] = 1" + } else { + $updates += "[$($columnobject.Name)] = 0" + } + } } else { if ($null -eq $newValue -and $columnobject.Nullable) { $updates += "[$($columnobject.Name)] = NULL" @@ -447,20 +572,33 @@ function Invoke-DbaDbDataMasking { } } - if ($columnobject.ColumnType -notin 'xml', 'geography', 'geometry') { - if (($row.$($columnobject.Name)).GetType().Name -match 'DBNull') { - $wheres += "[$($columnobject.Name)] IS NULL" - } else { - $oldValue = ($row.$($columnobject.Name)).Tostring().Replace("'", "''") - $wheres += "[$($columnobject.Name)] = '$oldValue'" - } - } - if ($columnobject.Deterministic -and ($row.$($columnobject.Name) -notin $dictionary.Keys)) { $dictionary.Add($row.$($columnobject.Name), $newValue) } } + $rowItems = $row | Get-Member -MemberType Properties | Select-Object Name -ExpandProperty Name + foreach ($item in $rowItems) { + if (($row.$($item)).GetType().Name -match 'DBNull') { + $wheres += "[$item] IS NULL" + } elseif ($dbTable.Columns[$item].DataType.SqlDataType.ToString().ToLower() -in 'text', 'ntext') { + $oldValue = ($row.$item).Tostring().Replace("'", "''") + $wheres += "CAST([$item] AS VARCHAR(MAX)) = '$oldValue'" + } elseif ($dbTable.Columns[$item].DataType.SqlDataType.ToString().ToLower() -like 'datetime') { + $oldValue = ($row.$item).Tostring("yyyy-MM-dd HH:mm:ss.fff") + $wheres += "[$item] = '$oldValue'" + } elseif ($dbTable.Columns[$item].DataType.SqlDataType.ToString().ToLower() -like 'datetime2') { + $oldValue = ($row.$item).Tostring("yyyy-MM-dd HH:mm:ss.ffffff") + $wheres += "[$item] = '$oldValue'" + } elseif ($dbTable.Columns[$item].DataType.SqlDataType.ToString().ToLower() -like '*date*') { + $oldValue = ($row.$item).Tostring("yyyy-MM-dd HH:mm:ss") + $wheres += "[$item] = '$oldValue'" + } else { + $oldValue = ($row.$item).Tostring().Replace("'", "''") + $wheres += "[$item] = '$oldValue'" + } + } + $updatequery = "UPDATE [$($tableobject.Schema)].[$($tableobject.Name)] SET $($updates -join ', ') WHERE $($wheres -join ' AND ')" try { @@ -471,6 +609,9 @@ function Invoke-DbaDbDataMasking { $errormessage = $_.Exception.Message.ToString() Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name): $errormessage" -Target $updatequery -Continue -ErrorRecord $_ } + + # Increase the row number + $rowNumber++ } try { $null = $transaction.Commit() @@ -490,6 +631,9 @@ function Invoke-DbaDbDataMasking { Stop-Function -Message "Error updating $($tableobject.Schema).$($tableobject.Name)" -Target $updatequery -Continue -ErrorRecord $_ } } + + # Empty the unique values array + $uniqueValues = $null } } try { diff --git a/functions/New-DbaDbMaskingConfig.ps1 b/functions/New-DbaDbMaskingConfig.ps1 index 20106f9db8..8e29f547f0 100644 --- a/functions/New-DbaDbMaskingConfig.ps1 +++ b/functions/New-DbaDbMaskingConfig.ps1 @@ -96,7 +96,7 @@ function New-DbaDbMaskingConfig { try { $columnTypes = Get-Content -Path "$script:PSModuleRoot\bin\datamasking\columntypes.json" | ConvertFrom-Json } catch { - Stop-Function -Message "Something went wrong importing the column types" -Continue + Stop-Function -Message "Something went wrong importing the column types" -ErrorRecord $_ -Continue } # Check if the Path is accessible if (-not (Test-Path -Path $Path)) { @@ -110,6 +110,8 @@ function New-DbaDbMaskingConfig { Stop-Function -Message "$Path is not a directory" } } + + $supportedDataTypes = 'bit', 'bool', 'char', 'date', 'datetime', 'datetime2', 'decimal', 'int', 'money', 'nchar', 'ntext', 'nvarchar', 'smalldatetime', 'text', 'time', 'uniqueidentifier', 'userdefineddatatype', 'varchar' } process { @@ -141,6 +143,12 @@ function New-DbaDbMaskingConfig { foreach ($tableobject in $tablecollection) { Write-Message -Message "Processing table $($tableobject.Name)" -Level Verbose + $hasUniqueIndex = $false + + if ($tableobject.Indexes.IsUnique) { + $hasUniqueIndex = $true + } + $columns = @() # Get the columns @@ -156,26 +164,27 @@ function New-DbaDbMaskingConfig { Write-Message -Level Verbose -Message "Skipping $columnobject because it is an identity column" continue } + if ($columnobject.IsForeignKey) { Write-Message -Level Verbose -Message "Skipping $columnobject because it is a foreign key" continue } + if ($columnobject.Computed) { Write-Message -Level Verbose -Message "Skipping $columnobject because it is a computed column" continue } - if ($columnobject.DataType.Name -eq 'hierarchyid') { - Write-Message -Level Verbose -Message "Skipping $columnobject because it is a hierarchyid column" - continue - } - if ($columnobject.DataType.Name -eq 'geography') { - Write-Message -Level Verbose -Message "Skipping $columnobject because it is a geography column" + + if ($server.VersionMajor -ge 13 -and $columnobject.GeneratedAlwaysType -ne 'None') { + Write-Message -Level Verbose -Message "Skipping $columnobject because it is a computed column for temporal tables" continue } - if ($columnobject.DataType.Name -eq 'geometry') { - Write-Message -Level Verbose -Message "Skipping $columnobject because it is a geometry column" + + if ($columnobject.DataType.Name -notin $supportedDataTypes) { + Write-Message -Level Verbose -Message "Skipping $columnobject because it is not a supported data type" continue } + if ($columnobject.DataType.SqlDataType.ToString().ToLower() -eq 'xml') { Write-Message -Level Verbose -Message "Skipping $columnobject because it is a xml column" continue @@ -202,6 +211,160 @@ function New-DbaDbMaskingConfig { $maskingType = $maskingType | Select-Object TypeName -ExpandProperty TypeName switch ($maskingType.ToLower()) { + "address" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Address" + SubType = "StreetAddress" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "bic" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Finance" + SubType = "Bic" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "bitcoin" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Finance" + SubType = "BitcoinAddress" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "country" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Address" + SubType = "Country" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "countrycode" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Address" + SubType = "CountryCode" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "ethereum" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Finance" + SubType = "EthereumAddress" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "city" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Address" + SubType = "City" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "creditcardcvv" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Finance" + SubType = "CreditCardCvv" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "company" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Company" + SubType = "CompanyName" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "creditcard" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Finance" + SubType = "CreditcardNumber" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "email" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Internet" + SubType = "Email" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } "firstname" { $columns += [PSCustomObject]@{ Name = $columnobject.Name @@ -211,6 +374,35 @@ function New-DbaDbMaskingConfig { MaxValue = $columnLength MaskingType = "Name" SubType = "Firstname" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "fullname" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Name" + SubType = "FullName" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "iban" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Finance" + SubType = "Iban" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } @@ -224,24 +416,26 @@ function New-DbaDbMaskingConfig { MaxValue = $columnLength MaskingType = "Name" SubType = "Lastname" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } } - "creditcard" { + "latitude" { $columns += [PSCustomObject]@{ Name = $columnobject.Name ColumnType = $columnType CharacterString = $null MinValue = $min MaxValue = $columnLength - MaskingType = "Finance" - SubType = "CreditcardNumber" + MaskingType = "Address" + SubType = "Latitude" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } } - "address" { + "longitude" { $columns += [PSCustomObject]@{ Name = $columnobject.Name ColumnType = $columnType @@ -249,12 +443,27 @@ function New-DbaDbMaskingConfig { MinValue = $min MaxValue = $columnLength MaskingType = "Address" - SubType = "StreetAddress" + SubType = "Longitude" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } } - "city" { + "phone" { + $columns += [PSCustomObject]@{ + Name = $columnobject.Name + ColumnType = $columnType + CharacterString = $null + MinValue = $min + MaxValue = $columnLength + MaskingType = "Phone" + SubType = "PhoneNumber" + Format = $null + Deterministic = $false + Nullable = $columnobject.Nullable + } + } + "state" { $columns += [PSCustomObject]@{ Name = $columnobject.Name ColumnType = $columnType @@ -262,12 +471,13 @@ function New-DbaDbMaskingConfig { MinValue = $min MaxValue = $columnLength MaskingType = "Address" - SubType = "City" + SubType = "State" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } } - "zipcode" { + "stateabbr" { $columns += [PSCustomObject]@{ Name = $columnobject.Name ColumnType = $columnType @@ -275,20 +485,22 @@ function New-DbaDbMaskingConfig { MinValue = $min MaxValue = $columnLength MaskingType = "Address" - SubType = "Zipcode" + SubType = "StateAbbr" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } } - "company" { + "zipcode" { $columns += [PSCustomObject]@{ Name = $columnobject.Name ColumnType = $columnType CharacterString = $null MinValue = $min MaxValue = $columnLength - MaskingType = "Company" - SubType = "CompanyName" + MaskingType = "Address" + SubType = "Zipcode" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } @@ -302,6 +514,7 @@ function New-DbaDbMaskingConfig { MaxValue = $columnLength MaskingType = "Internet" SubType = "UserName" + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } @@ -337,6 +550,10 @@ function New-DbaDbMaskingConfig { $subType = "Date" $MaxValue = $null } + "decimal" { + $subType = "Decimal" + $MaxValue = $null + } "float" { $subType = "Float" $MaxValue = $null @@ -390,6 +607,7 @@ function New-DbaDbMaskingConfig { MaxValue = $MaxValue MaskingType = $type SubType = $subType + Format = $null Deterministic = $false Nullable = $columnobject.Nullable } @@ -400,9 +618,10 @@ function New-DbaDbMaskingConfig { # Check if something needs to be generated if ($columns) { $tables += [PSCustomObject]@{ - Name = $tableobject.Name - Schema = $tableobject.Schema - Columns = $columns + Name = $tableobject.Name + Schema = $tableobject.Schema + Columns = $columns + HasUniqueIndex = $hasUniqueIndex } } else { Write-Message -Message "No columns match for masking in table $($tableobject.Name)" -Level Verbose @@ -430,7 +649,7 @@ function New-DbaDbMaskingConfig { Set-Content -Path $temppath -Value ($results | ConvertTo-Json -Depth 5) Get-ChildItem -Path $temppath } catch { - Stop-Function -Message "Something went wrong writing the results to the Path" -Target $Path -Continue -ErrorRecord $_ + Stop-Function -Message "Something went wrong writing the results to the $Path" -Target $Path -Continue -ErrorRecord $_ } } else { Write-Message -Message "No tables to save for database $($db.Name) on $($server.Name)" -Level Verbose diff --git a/tests/Invoke-DbaDbDataMasking.Tests.ps1 b/tests/Invoke-DbaDbDataMasking.Tests.ps1 index 3d4cc36361..ca797a03f5 100644 --- a/tests/Invoke-DbaDbDataMasking.Tests.ps1 +++ b/tests/Invoke-DbaDbDataMasking.Tests.ps1 @@ -5,7 +5,7 @@ Write-Host -Object "Running $PSCommandPath" -ForegroundColor Cyan Describe "$CommandName Unit Tests" -Tag 'UnitTests' { Context "Validate parameters" { [object[]]$params = (Get-Command $CommandName).Parameters.Keys | Where-Object {$_ -notin ('whatif', 'confirm')} - [object[]]$knownParameters = 'SqlInstance','SqlCredential','Database','FilePath','Locale','CharacterString','Table','Column','ExcludeTable','ExcludeColumn','Query','MaxValue','ModulusFactor','ExactLength','EnableException' + [object[]]$knownParameters = 'SqlInstance', 'SqlCredential', 'Database', 'FilePath', 'Locale', 'CharacterString', 'Table', 'Column', 'ExcludeTable', 'ExcludeColumn', 'Query', 'MaxValue', 'ModulusFactor', 'ExactLength', 'EnableException' $knownParameters += [System.Management.Automation.PSCmdlet]::CommonParameters It "Should only contain our specific parameters" { (@(Compare-Object -ReferenceObject ($knownParameters | Where-Object {$_}) -DifferenceObject $params).Count ) | Should Be 0 @@ -33,32 +33,33 @@ Describe "$CommandName Integration Tests" -Tag "IntegrationTests" { GO INSERT INTO people2 (fname, lname, dob) VALUES ('Layla','Schmoe','2/2/2000') INSERT INTO people2 (fname, lname, dob) VALUES ('Eric','Schmee','2/2/1950')" - New-DbaDatabase -SqlInstance $script:instance1 -Name $db - Invoke-DbaQuery -SqlInstance $script:instance1 -Database $db -Query $sql + New-DbaDatabase -SqlInstance $script:instance2 -Name $db + Invoke-DbaQuery -SqlInstance $script:instance2 -Database $db -Query $sql } + AfterAll { - Remove-DbaDatabase -SqlInstance $script:instance1 -Database $db -Confirm:$false + Remove-DbaDatabase -SqlInstance $script:instance2 -Database $db -Confirm:$false $file | Remove-Item -Confirm:$false -ErrorAction Ignore } Context "Command works" { It "starts with the right data" { - Invoke-DbaQuery -SqlInstance $script:instance1 -Database $db -Query "select * from people where fname = 'Joe'" | Should -Not -Be $null - Invoke-DbaQuery -SqlInstance $script:instance1 -Database $db -Query "select * from people where lname = 'Schmee'" | Should -Not -Be $null + Invoke-DbaQuery -SqlInstance $script:instance2 -Database $db -Query "select * from people where fname = 'Joe'" | Should -Not -Be $null + Invoke-DbaQuery -SqlInstance $script:instance2 -Database $db -Query "select * from people where lname = 'Schmee'" | Should -Not -Be $null } It "returns the proper output" { - $file = New-DbaDbMaskingConfig -SqlInstance $script:instance1 -Database $db -Path C:\temp - $results = $file | Invoke-DbaDbDataMasking -SqlInstance $script:instance1 -Database $db -Confirm:$false + $file = New-DbaDbMaskingConfig -SqlInstance $script:instance2 -Database $db -Path C:\temp + $results = $file | Invoke-DbaDbDataMasking -SqlInstance $script:instance2 -Database $db -Confirm:$false foreach ($result in $results) { $result.Rows | Should -Be 2 $result.Database | Should -Contain $db } - + } It "masks the data and does not delete it" { - Invoke-DbaQuery -SqlInstance $script:instance1 -Database $db -Query "select * from people" | Should -Not -Be $null - Invoke-DbaQuery -SqlInstance $script:instance1 -Database $db -Query "select * from people where fname = 'Joe'" | Should -Be $null - Invoke-DbaQuery -SqlInstance $script:instance1 -Database $db -Query "select * from people where lname = 'Schmee'" | Should -Be $null + Invoke-DbaQuery -SqlInstance $script:instance2 -Database $db -Query "select * from people" | Should -Not -Be $null + Invoke-DbaQuery -SqlInstance $script:instance2 -Database $db -Query "select * from people where fname = 'Joe'" | Should -Be $null + Invoke-DbaQuery -SqlInstance $script:instance2 -Database $db -Query "select * from people where lname = 'Schmee'" | Should -Be $null } } } \ No newline at end of file diff --git a/tests/New-DbaDbMaskingConfig.Tests.ps1 b/tests/New-DbaDbMaskingConfig.Tests.ps1 index 123343685f..a1fc3ed4dd 100644 --- a/tests/New-DbaDbMaskingConfig.Tests.ps1 +++ b/tests/New-DbaDbMaskingConfig.Tests.ps1 @@ -5,7 +5,7 @@ Write-Host -Object "Running $PSCommandPath" -ForegroundColor Cyan Describe "$CommandName Unit Tests" -Tag 'UnitTests' { Context "Validate parameters" { [object[]]$params = (Get-Command $CommandName).Parameters.Keys | Where-Object {$_ -notin ('whatif', 'confirm')} - [object[]]$knownParameters = 'SqlInstance','SqlCredential','Database','Table','Column','Path','Locale','Force','EnableException' + [object[]]$knownParameters = 'SqlInstance', 'SqlCredential', 'Database', 'Table', 'Column', 'Path', 'Locale', 'Force', 'EnableException' $knownParameters += [System.Management.Automation.PSCmdlet]::CommonParameters It "Should only contain our specific parameters" { (@(Compare-Object -ReferenceObject ($knownParameters | Where-Object {$_}) -DifferenceObject $params).Count ) | Should Be 0