Skip to content

Commit

Permalink
fixesd backup data
Browse files Browse the repository at this point in the history
  • Loading branch information
@KelvinTegelaar
KelvinTegelaar committed Jul 8, 2024
1 parent 68f1247 commit 09931d9
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 34 deletions.
48 changes: 26 additions & 22 deletions Modules/CIPPCore/Public/Add-CIPPAzDataTableEntity.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,57 +7,61 @@ function Add-CIPPAzDataTableEntity {
[switch]$CreateTableIfNotExists
)

$MaxRowSize = 500000 - 100 #Maximum size of an entity
$MaxSize = 30kb # maximum size of a property value
$MaxRowSize = 500000 - 100 # Maximum size of an entity
$MaxSize = 30kb # Maximum size of a property value

foreach ($SingleEnt in $Entity) {
try {
Add-AzDataTableEntity -context $Context -force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt -ErrorAction Stop
Add-AzDataTableEntity -Context $Context -Force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt -ErrorAction Stop
} catch [System.Exception] {
if ($_.Exception.ErrorCode -eq 'PropertyValueTooLarge' -or $_.Exception.ErrorCode -eq 'EntityTooLarge') {
try {
$largePropertyNames = @()
$largePropertyNames = [System.Collections.ArrayList]::new()
$entitySize = 0
foreach ($key in $SingleEnt.Keys) {
$propertySize = [System.Text.Encoding]::UTF8.GetByteCount($SingleEnt[$key].ToString())
$entitySize = $entitySize + $propertySize
if ($propertySize -gt $MaxSize) {
$largePropertyNames = $largePropertyNames + $key
$largePropertyNames.Add($key)
}

}

if ($largePropertyNames.Count -gt 0) {
$splitInfoList = [System.Collections.ArrayList]@()
foreach ($largePropertyName in $largePropertyNames) {
$dataString = $SingleEnt[$largePropertyName]
$splitCount = [math]::Ceiling($dataString.Length / $MaxSize)
$splitData = @()
$splitData = [System.Collections.ArrayList]@()
for ($i = 0; $i -lt $splitCount; $i++) {
$start = $i * $MaxSize
$splitData = $splitData + $dataString.Substring($start, [Math]::Min($MaxSize, $dataString.Length - $start))
$splitData.Add($dataString.Substring($start, [Math]::Min($MaxSize, $dataString.Length - $start))) > $null
}

$splitPropertyNames = @()
$splitPropertyNames = [System.Collections.ArrayList]@()
for ($i = 0; $i -lt $splitData.Count; $i++) {
$splitPropertyNames = $splitPropertyNames + "${largePropertyName}_Part$i"
$splitPropertyNames.Add("${largePropertyName}_Part$i") > $null
}

$splitInfo = @{
OriginalHeader = $largePropertyName
SplitHeaders = $splitPropertyNames
}
$SingleEnt['SplitOverProps'] = ($splitInfo | ConvertTo-Json).ToString()
$splitInfoList.Add($splitInfo) > $null
$SingleEnt.Remove($largePropertyName)

for ($i = 0; $i -lt $splitData.Count; $i++) {
$SingleEnt[$splitPropertyNames[$i]] = $splitData[$i]
}
}

$SingleEnt['SplitOverProps'] = ($splitInfoList | ConvertTo-Json).ToString()
}

# Check if the entity is still too large
$entitySize = [System.Text.Encoding]::UTF8.GetByteCount($($SingleEnt | ConvertTo-Json))
if ($entitySize -gt $MaxRowSize) {
$rows = @()
$rows = [System.Collections.ArrayList]@()
$originalPartitionKey = $SingleEnt.PartitionKey
$originalRowKey = $SingleEnt.RowKey
$entityIndex = 0
Expand All @@ -75,23 +79,23 @@ function Add-CIPPAzDataTableEntity {
$newEntity['PartIndex'] = $entityIndex
$entityIndex++

$propertiesToRemove = @()
$propertiesToRemove = [System.Collections.ArrayList]@()
foreach ($key in $SingleEnt.Keys) {
$newEntitySize = [System.Text.Encoding]::UTF8.GetByteCount($($newEntity | ConvertTo-Json))
if ($newEntitySize -lt $MaxRowSize) {
$propertySize = [System.Text.Encoding]::UTF8.GetByteCount($SingleEnt[$key].ToString())
if ($propertySize -gt $MaxRowSize) {
$dataString = $SingleEnt[$key]
$splitCount = [math]::Ceiling($dataString.Length / $MaxSize)
$splitData = @()
$splitData = [System.Collections.ArrayList]@()
for ($i = 0; $i -lt $splitCount; $i++) {
$start = $i * $MaxSize
$splitData = $splitData + $dataString.Substring($start, [Math]::Min($MaxSize, $dataString.Length - $start))
$splitData.Add($dataString.Substring($start, [Math]::Min($MaxSize, $dataString.Length - $start))) > $null
}

$splitPropertyNames = @()
$splitPropertyNames = [System.Collections.ArrayList]@()
for ($i = 0; $i -lt $splitData.Count; $i++) {
$splitPropertyNames = $splitPropertyNames + "${key}_Part$i"
$splitPropertyNames.Add("${key}_Part$i") > $null
}

for ($i = 0; $i -lt $splitData.Count; $i++) {
Expand All @@ -100,15 +104,15 @@ function Add-CIPPAzDataTableEntity {
} else {
$newEntity[$key] = $SingleEnt[$key]
}
$propertiesToRemove = $propertiesToRemove + $key
$propertiesToRemove.Add($key) > $null
}
}

foreach ($prop in $propertiesToRemove) {
$SingleEnt.Remove($prop)
}

$rows = $rows + $newEntity
$rows.Add($newEntity) > $null
$entitySize = [System.Text.Encoding]::UTF8.GetByteCount($($SingleEnt | ConvertTo-Json))
}

Expand All @@ -118,19 +122,19 @@ function Add-CIPPAzDataTableEntity {
$SingleEnt['PartIndex'] = $entityIndex
$SingleEnt['PartitionKey'] = $originalPartitionKey

$rows = $rows + $SingleEnt
$rows.Add($SingleEnt) > $null
}

foreach ($row in $rows) {
Write-Host "current entity is $($row.RowKey) with $($row.PartitionKey). Our size is $([System.Text.Encoding]::UTF8.GetByteCount($($row | ConvertTo-Json)))"
Add-AzDataTableEntity -context $Context -force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $row
Add-AzDataTableEntity -Context $Context -Force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $row
}
} else {
Add-AzDataTableEntity -context $Context -force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt
Add-AzDataTableEntity -Context $Context -Force:$Force -CreateTableIfNotExists:$CreateTableIfNotExists -Entity $SingleEnt
}

} catch {
throw "Error processing entity: $($_.Exception.Message)."
throw "Error processing entity: $($_.Exception.Message) Linenumner: $($_.InvocationInfo.ScriptLineNumber)"
}
} else {
Write-Host "THE ERROR IS $($_.Exception.ErrorCode). The size of the entity is $entitySize."
Expand Down
4 changes: 2 additions & 2 deletions Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Push-ExecScheduledCommand.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ function Push-ExecScheduledCommand {
Write-Host "Started Task: $($Item.Command) for tenant: $tenant"
try {
try {
Write-Host "Starting task: $($Item.Command) with parameters: "
Write-Host "Starting task: $($Item.Command) with parameters: $($commandParameters | ConvertTo-Json)"
$results = & $Item.Command @commandParameters
} catch {
$results = "Task Failed: $($_.Exception.Message)"
Expand Down Expand Up @@ -112,4 +112,4 @@ function Push-ExecScheduledCommand {
if ($TaskType -ne 'Alert') {
Write-LogMessage -API 'Scheduler_UserTasks' -tenant $tenant -message "Successfully executed task: $($task.Name)" -sev Info
}
}
}
26 changes: 16 additions & 10 deletions Modules/CIPPCore/Public/Get-CIPPAzDatatableEntity.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,25 @@ function Get-CIPPAzDataTableEntity {
$Results = Get-AzDataTableEntity @PSBoundParameters
$mergedResults = @{}

# First pass: Collect all parts and complete entities
foreach ($entity in $Results) {
if ($entity.OriginalEntityId) {
$entityId = $entity.OriginalEntityId
if (-not $mergedResults.ContainsKey($entityId)) {
$mergedResults[$entityId] = @{
Parts = @()
Parts = New-Object 'System.Collections.ArrayList'
}
}
$mergedResults[$entityId]['Parts'] = $mergedResults[$entityId]['Parts'] + @($entity)
$mergedResults[$entityId]['Parts'].Add($entity) > $null
} else {
$mergedResults[$entity.RowKey] = @{
Entity = $entity
Parts = @()
Parts = New-Object 'System.Collections.ArrayList'
}
}
}

# Second pass: Reassemble entities from parts
$finalResults = @()
foreach ($entityId in $mergedResults.Keys) {
$entityData = $mergedResults[$entityId]
Expand All @@ -38,7 +40,7 @@ function Get-CIPPAzDataTableEntity {
$parts = $entityData.Parts | Sort-Object PartIndex
foreach ($part in $parts) {
foreach ($key in $part.PSObject.Properties.Name) {
if ($key -notin @('OriginalEntityId', 'PartIndex', 'PartitionKey', 'RowKey', 'ETag', 'Timestamp')) {
if ($key -notin @('OriginalEntityId', 'PartIndex', 'PartitionKey', 'RowKey')) {
if ($fullEntity.PSObject.Properties[$key]) {
$fullEntity | Add-Member -MemberType NoteProperty -Name $key -Value ($fullEntity.$key + $part.$key) -Force
} else {
Expand All @@ -55,15 +57,19 @@ function Get-CIPPAzDataTableEntity {
}
}

# Third pass: Process split properties and remerge them
foreach ($entity in $finalResults) {
if ($entity.SplitOverProps) {
$splitInfo = $entity.SplitOverProps | ConvertFrom-Json
$mergedData = [string]::Join('', ($splitInfo.SplitHeaders | ForEach-Object { $entity.$_ }))
$entity | Add-Member -NotePropertyName $splitInfo.OriginalHeader -NotePropertyValue $mergedData -Force
$propsToRemove = $splitInfo.SplitHeaders + 'SplitOverProps'
foreach ($prop in $propsToRemove) {
$entity.PSObject.Properties.Remove($prop)
$splitInfoList = $entity.SplitOverProps | ConvertFrom-Json
foreach ($splitInfo in $splitInfoList) {
$mergedData = [string]::Join('', ($splitInfo.SplitHeaders | ForEach-Object { $entity.$_ }))
$entity | Add-Member -NotePropertyName $splitInfo.OriginalHeader -NotePropertyValue $mergedData -Force
$propsToRemove = $splitInfo.SplitHeaders
foreach ($prop in $propsToRemove) {
$entity.PSObject.Properties.Remove($prop)
}
}
$entity.PSObject.Properties.Remove('SplitOverProps')
}
}

Expand Down

0 comments on commit 09931d9

Please sign in to comment.