From b625a8af654524b9231989bf435885fc5991f9c6 Mon Sep 17 00:00:00 2001 From: kyle Date: Wed, 7 Oct 2026 16:14:03 -0600 Subject: [PATCH] Fix WSB collector against a real server's object model Checked with Get-WsbCollectorDiagnostics.ps1 output from a Server 2019 Hyper-V host (AAOG-EDM-HVR1): - WBJob StartTime/EndTime are culture-formatted local-time strings ("10/6/2026 9:00 PM"). Parse them with the current culture: the [datetime] cast uses the invariant (US) format and would misread dd/MM dates, e.g. on en-CA/en-GB servers. Confirmed local, not UTC, against the Microsoft-Windows-Backup event log - Hyper-V backups nest VMs under a VirtualMachineList item's SubItemList; failures, names and the items list now come from the leaf items, falling back to a parent's SubItemFailed count - Report size from the top-level items' BytesProcessed - FailureLogPath is set even on success: running backups get no message, and the log is only cited for warnings/failures - HRESULTs are strings; conversion tolerates hex and bad values - Mock rebuilt on the real shapes; tested under en-US and en-GB Co-Authored-By: Claude Opus 5.5 --- collectors/wsb/README.md | 16 ++--- collectors/wsb/Send-WsbBackupStatus.ps1 | 74 ++++++++++++++++++----- collectors/wsb/tests/MockWsb.ps1 | 79 ++++++++++++++++++------- 3 files changed, 125 insertions(+), 44 deletions(-) diff --git a/collectors/wsb/README.md b/collectors/wsb/README.md index 65f8fa7..c31e699 100644 --- a/collectors/wsb/README.md +++ b/collectors/wsb/README.md @@ -4,11 +4,10 @@ A PowerShell script that runs on each server using Windows Server Backup (WSB) a backups to the dashboard. Each server appears as one job, named ` - Windows Server Backup` by default. -> **Not yet checked against a real server.** The cmdlets are the `WindowsServerBackup` module's own -> (`Get-WBJob`, `Get-WBSummary`), but the properties of the objects they return, and whether job -> times are local or UTC, aren't fully documented. Before installing on the first server, run -> `Get-WsbCollectorDiagnostics.ps1` and the collector with `-DryRun` (see below) to confirm what it -> reads. +> **Checked against:** Server 2019 Hyper-V host backing up VMs, successful backups only. +> Not yet seen on a real server: a failed backup, a failed VM or volume, or a backup in progress. +> Those paths are written to the same object model, but if one looks wrong, run +> `Get-WsbCollectorDiagnostics.ps1` while it's in that state and send the output. ## Install (on the server, as an administrator) @@ -44,11 +43,12 @@ Re-run the installer to change settings or update the script. |---|---| | Company | `Tenant` from the config | | Job | One per server, ID `wsb:`, so changing `JobName` renames rather than duplicates it | -| Run | One per backup job from `Get-WBJob`, identified by its start time (WSB jobs have no ID). Restores are ignored | +| Run | One per backup job from `Get-WBJob`, identified by its start time (WSB jobs have no ID). Restores are ignored. `details.items` lists what was backed up: the VMs for a Hyper-V backup, otherwise volumes | | Status | Error HRESULT → **Failed**. Success, but a detailed error or a failed item (e.g. one volume) → **Warning**. Cancelled or unrecognised → **Warning**. In progress → **Running** | -| Started / finished | Job start / end time | +| Started / finished | Job start / end time. WSB gives these as local-time text to the minute, in the server's date format; the collector parses them with that format | | Next scheduled | `Get-WBSummary`'s `NextBackupTime`. Empty when there's no scheduled backup | -| Message | Error description, error codes (as `0x8XXXXXXX`), failed items, and the failure log path | +| Message | Error description, error codes (as `0x8XXXXXXX`), failed VMs/volumes, and the failure log path | +| Size | Bytes processed across the backup's items | Each run covers the last 48 hours (`LookbackHours`). Runs already delivered are recorded in `wsb-collector.state.json`. diff --git a/collectors/wsb/Send-WsbBackupStatus.ps1 b/collectors/wsb/Send-WsbBackupStatus.ps1 index 90587f2..3b03349 100644 --- a/collectors/wsb/Send-WsbBackupStatus.ps1 +++ b/collectors/wsb/Send-WsbBackupStatus.ps1 @@ -105,19 +105,33 @@ function Import-Wsb { # ---------- Jobs ---------- +# WBJob times are strings in local time, to the minute, formatted with the current culture +# ("10/6/2026 9:00 PM" on en-US). Parse with that same culture: PowerShell's [datetime] cast would +# use the invariant (US) format and misread dd/MM dates. Empty while not set (e.g. EndTime of a +# running backup). function Get-JobTime($Job, [string]$Field) { $value = Get-FirstValue $Job @($Field) if ($null -eq $value) { return $null } - $dt = [datetime]$value - if ($dt.Year -le 1900) { return $null } # e.g. EndTime while a backup is running + if ($value -is [datetime]) { + $dt = $value + } else { + $dt = [datetime]::MinValue + if (-not [datetime]::TryParse([string]$value, [Globalization.CultureInfo]::CurrentCulture, + [Globalization.DateTimeStyles]::AssumeLocal, [ref]$dt)) { + Write-Log "Couldn't parse $Field '$value'" 'WARN' + return $null + } + } + if ($dt.Year -le 1900) { return $null } return $dt } -# HRESULTs as numbers; 0 = OK. $null when the job doesn't have the property. +# HRESULTs as numbers; 0 = OK. $null when the object doesn't have the property. WSB returns them +# as strings ("0"); PowerShell's conversion also accepts hex ("0x80042302") and negative decimals. function Get-HResult($Object, [string]$Field) { $value = Get-FirstValue $Object @($Field) if ($null -eq $value) { return $null } - return [long]$value + try { return [long]$value } catch { return $null } } function Format-HResult([long]$Value) { @@ -135,15 +149,42 @@ function Get-ItemName($Item) { return 'Item' } -# Items (volumes, system state, ...) that reported an error +# The things actually backed up. Items can nest: a Hyper-V backup has one VirtualMachineList item +# whose SubItemList holds each VM (and "Host Component"). +function Get-LeafItems($Items) { + foreach ($item in $Items) { + $subItems = @(Get-FirstValue $item @('SubItemList') | Where-Object { $_ }) + if ($subItems.Count) { Get-LeafItems $subItems } else { $item } + } +} + +function Test-ItemFailed($Item) { + $hr = Get-HResult $Item 'HResult' + $state = [string](Get-FirstValue $Item @('State', 'JobState')) + return ($hr -and $hr -ne 0) -or $state -match 'Fail' +} + +# Items that reported an error: failed leaves, or a parent that counts failed sub-items without +# saying which function Get-FailedItems($Job) { - return @(Get-JobItems $Job | Where-Object { - $hr = Get-HResult $_ 'HResult' - $state = [string](Get-FirstValue $_ @('State', 'JobState')) - ($hr -and $hr -ne 0) -or $state -match 'Fail' + $items = Get-JobItems $Job + $failed = @(Get-LeafItems $items | Where-Object { Test-ItemFailed $_ }) + if ($failed.Count) { return $failed } + return @($items | Where-Object { + (Test-ItemFailed $_) -or ((Get-FirstValue $_ @('SubItemFailed')) -and [long](Get-FirstValue $_ @('SubItemFailed')) -gt 0) }) } +# Bytes processed, from the top-level items (sub-items are already counted in their parent) +function Get-BytesProcessed($Job) { + $sum = [long]0 + foreach ($item in Get-JobItems $Job) { + $value = Get-FirstValue $item @('BytesProcessed') + if ($value) { $sum += [long]$value } + } + return $sum +} + function Get-RunStatus($Job) { $state = [string](Get-FirstValue $Job @('JobState', 'State')) if (-not (Get-JobTime $Job 'EndTime') -and $state -notmatch 'Complet|Fail|Cancel|Abort|Stop|Succe') { @@ -161,6 +202,7 @@ function Get-RunStatus($Job) { } function Get-RunMessage($Job, [string]$Status) { + if ($Status -eq 'running') { return '' } $lines = New-Object 'System.Collections.Generic.List[string]' $state = [string](Get-FirstValue $Job @('JobState', 'State')) $description = Get-FirstValue $Job @('ErrorDescription', 'DetailedMessage') @@ -178,24 +220,24 @@ function Get-RunMessage($Job, [string]$Status) { foreach ($item in Get-FailedItems $Job) { $itemError = Get-FirstValue $item @('ErrorDescription', 'DetailedMessage') $itemHr = Get-HResult $item 'HResult' + $subFailed = Get-FirstValue $item @('SubItemFailed') $line = "$(Get-ItemName $item) - " # names are often volumes like "D:", so no colon if ($itemError) { $line += ([string]$itemError).Trim() } elseif ($itemHr) { $line += "error $(Format-HResult $itemHr)" } + elseif ($subFailed -and [long]$subFailed -gt 0) { $line += "$subFailed sub-item(s) failed" } else { $line += [string](Get-FirstValue $item @('State', 'JobState')) } $lines.Add($line) } + # WSB fills in FailureLogPath even for successful backups, so only point at it when relevant if ($Status -ne 'success') { $failureLog = Get-FirstValue $Job @('FailureLogPath') if ($failureLog) { $lines.Add("Failure log: $failureLog") } } if ($lines.Count -eq 0) { - switch ($Status) { - 'success' { return 'Backup completed successfully.' } - 'running' { return '' } - default { return "Backup finished with state '$state'." } - } + if ($Status -eq 'success') { return 'Backup completed successfully.' } + return "Backup finished with state '$state'." } # WSB reports "Completed" even for a backup that failed with an error code, so only mention # the state when it says something else (e.g. cancelled) @@ -206,7 +248,7 @@ function Get-RunMessage($Job, [string]$Status) { function ConvertTo-Run($Job) { $status = Get-RunStatus $Job $details = [ordered]@{ server = $env:COMPUTERNAME } - $items = @(Get-JobItems $Job | ForEach-Object { Get-ItemName $_ }) + $items = @(Get-LeafItems (Get-JobItems $Job) | ForEach-Object { Get-ItemName $_ }) if ($items.Count) { $details.items = $items } $state = Get-FirstValue $Job @('JobState', 'State') if ($state) { $details.wsb_state = [string]$state } @@ -223,6 +265,8 @@ function ConvertTo-Run($Job) { if ($status -ne 'running') { $finished = ConvertTo-IsoTime (Get-JobTime $Job 'EndTime') if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time + $bytes = Get-BytesProcessed $Job + if ($bytes -gt 0) { $run.size_bytes = $bytes } } return $run } diff --git a/collectors/wsb/tests/MockWsb.ps1 b/collectors/wsb/tests/MockWsb.ps1 index 7d8e22f..8689f64 100644 --- a/collectors/wsb/tests/MockWsb.ps1 +++ b/collectors/wsb/tests/MockWsb.ps1 @@ -1,5 +1,8 @@ -# Stand-ins for the WindowsServerBackup cmdlets. Object shapes are best guesses until checked -# against Get-WsbCollectorDiagnostics.ps1 output from a real server. +# Stand-ins for the WindowsServerBackup cmdlets. Object shapes follow Get-WsbCollectorDiagnostics.ps1 +# output from a real Hyper-V host (Server 2019): job times are culture-formatted local-time strings +# to the minute ("10/6/2026 9:00 PM" on en-US), HRESULTs are strings, and a Hyper-V backup has one +# VirtualMachineList item whose SubItemList holds the VMs. When nothing is running, Get-WBJob +# returns an empty job (JobType None). # # . ./tests/MockWsb.ps1 # nothing running # . ./tests/MockWsb.ps1 -WithRunningJob # a backup in progress @@ -9,50 +12,84 @@ param([switch]$WithRunningJob) # Fixed to the top of the hour so repeated runs see the same jobs $now = Get-Date -Minute 0 -Second 0 -Millisecond 0 -function New-MockItem($Name, [int]$HResult = 0, $ErrorDescription = '') { - [pscustomobject]@{ Name = $Name; Type = 'Volume'; State = $(if ($HResult) { 'Failed' } else { 'Completed' }) - HResult = $HResult; ErrorDescription = $ErrorDescription } +# Same format as the real cmdlet: short date + short time in the current culture +function Format-WbTime($Value) { + if ($null -eq $Value) { return '' } + return ([datetime]$Value).ToString('g', [Globalization.CultureInfo]::CurrentCulture) } -function New-MockJob($Type, $Start, $End, $State, [int]$HResult = 0, [int]$DetailedHResult = 0, $ErrorDescription = '', $Items = @()) { +function New-MockItem($Name, $Type = 'Volume', [long]$Bytes = 0, [string]$HResult = '0', $ErrorDescription = '', $SubItems = $null) { + $failedSubs = @($SubItems | Where-Object { $_ -and $_.HResult -ne '0' }).Count + [pscustomobject]@{ + Name = $Name + Type = $Type + State = $(if ($HResult -ne '0') { 'Failed' } else { 'Completed' }) + HResult = $HResult + DetailedHResult = '0' + ErrorDescription = $ErrorDescription + BytesProcessed = $Bytes + TotalBytes = $Bytes + CurrentItem = $null + SubItemProcessed = [long]@($SubItems | Where-Object { $_ }).Count + SubItemFailed = [long]$failedSubs + TotalSubItem = [long]@($SubItems | Where-Object { $_ }).Count + SubItemList = $SubItems + } +} + +function New-VmList([long]$VmBytes, [string]$VmHResult = '0', $VmError = '') { + $vms = @( + (New-MockItem 'AAOG-EDM-AD1' 'Component' $VmBytes $VmHResult $VmError), + (New-MockItem 'Host Component' 'Component' 447729) + ) + New-MockItem 'VirtualMachineList' 'VirtualmachineList' ($VmBytes + 447729) -SubItems $vms +} + +function New-MockJob($Type, $Start, $End, $State, [string]$HResult = '0', [string]$DetailedHResult = '0', $ErrorDescription = '', $Items = $null) { [pscustomobject]@{ JobType = $Type - StartTime = $Start - EndTime = $(if ($End) { $End } else { [datetime]::MinValue }) + StartTime = Format-WbTime $Start + EndTime = Format-WbTime $End JobState = $State + CurrentOperation = '' HResult = $HResult DetailedHResult = $DetailedHResult ErrorDescription = $ErrorDescription JobItems = $Items - VersionId = $(if ($End) { $Start.ToUniversalTime().ToString('MM/dd/yyyy-HH:mm') } else { '' }) - SuccessLogPath = 'C:\Windows\Logs\WindowsServerBackup\Backup-success.log' - FailureLogPath = $(if ($HResult -or $DetailedHResult) { 'C:\Windows\Logs\WindowsServerBackup\Backup_Error.log' } else { '' }) + VersionId = $(if ($End) { $Start.ToUniversalTime().ToString('MM/dd/yyyy-HH:mm', [Globalization.CultureInfo]::InvariantCulture) } else { '' }) + SuccessLogPath = 'C:\Windows\Logs\WindowsServerBackup\Backup.log' + # Populated even for successful backups, as on the real server + FailureLogPath = 'C:\Windows\Logs\WindowsServerBackup\Backup_Error.log' } } $global:MockWsbRunning = if ($WithRunningJob) { - @(New-MockJob 'Backup' $now.AddMinutes(-10) $null 'Running' -Items @((New-MockItem 'C:'), (New-MockItem 'D:'))) + @(New-MockJob 'Backup' $now.AddMinutes(-10) $null 'Running' -Items @(New-VmList 10GB)) } else { - # What an idle server might return: an empty job rather than nothing - @([pscustomobject]@{ JobType = 'None'; StartTime = [datetime]::MinValue; EndTime = [datetime]::MinValue; JobState = 'Unknown' }) + @(New-MockJob 'None' $null $null 'Unknown') } $global:MockWsbPrevious = @( - (New-MockJob 'Backup' $now.AddHours(-12) $now.AddHours(-11) 'Completed' -Items @((New-MockItem 'C:'), (New-MockItem 'D:'))), - # Completed overall, but one volume failed - (New-MockJob 'Backup' $now.AddHours(-36) $now.AddHours(-35) 'Completed' 0 -2147024891 '' @( - (New-MockItem 'C:'), (New-MockItem 'D:' -2147024891 'Access is denied.'))), + (New-MockJob 'Backup' $now.AddHours(-12) $now.AddHours(-11.6) 'Completed' -Items @(New-VmList 124756706036)), + # Completed overall, but one VM failed + (New-MockJob 'Backup' $now.AddHours(-36) $now.AddHours(-35.6) 'Completed' '0' '0' '' @( + New-VmList 0 '-2147024891' 'Access is denied.')), # Failed outright: backup target not found (0x80042302) - (New-MockJob 'Backup' $now.AddHours(-30) $now.AddHours(-29.9) 'Completed' -2147212542 0 ` + (New-MockJob 'Backup' $now.AddHours(-30) $now.AddHours(-29.9) 'Completed' '-2147212542' '0' ` 'The backup storage location could not be found.'), + # A volume-based backup, for servers that aren't Hyper-V hosts + (New-MockJob 'Backup' $now.AddHours(-20) $now.AddHours(-19.5) 'Completed' -Items @( + (New-MockItem 'C:' 'Volume' 40GB), (New-MockItem 'D:' 'Volume' 200GB))), (New-MockJob 'Recovery' $now.AddHours(-5) $now.AddHours(-4) 'Completed'), # a restore: must be ignored (New-MockJob 'Backup' $now.AddDays(-5) $now.AddDays(-5).AddHours(1) 'Completed') # outside 48h lookback ) $global:MockWsbSummary = [pscustomobject]@{ NextBackupTime = $now.Date.AddHours(21).AddDays($(if ($now.Hour -ge 21) { 1 } else { 0 })) - LastSuccessfulBackupTime = $now.AddHours(-11) - LastBackupResultHR = 0 + NumberOfVersions = 1 + LastSuccessfulBackupTime = $now.AddHours(-12) + LastBackupResultHR = [uint32]0 + CurrentOperationStatus = $(if ($WithRunningJob) { 'BackupInProgress' } else { 'NoOperationInProgress' }) } function Get-WBJob { param([int]$Previous) if ($PSBoundParameters.ContainsKey('Previous')) { $global:MockWsbPrevious } else { $global:MockWsbRunning } }