Add Windows Server Backup collector

PowerShell script for servers using Windows Server Backup, run hourly as a SYSTEM scheduled task under Windows PowerShell 5.1. Each server is one dashboard job. - Backup jobs from Get-WBJob (in progress and -Previous) within the lookback window; restores are ignored. WSB jobs have no ID, so runs are keyed on start time, which also merges a Running report with the finished one - Error HRESULT -> failed; detailed error or a failed item -> warning; messages include the error description, codes as 0x8XXXXXXX, failed items and the failure log path - Next run from Get-WBSummary's NextBackupTime - Get-WsbCollectorDiagnostics.ps1 dumps the cmdlets' real objects and recent Microsoft-Windows-Backup events (to check whether job times are local or UTC); installer adapted from the MARS one Not yet run against a real server. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-07 15:56:04 -06:00
parent 5982fc9434
commit a14b1b0dc9
6 changed files with 673 additions and 1 deletions
@@ -0,0 +1,350 @@
+<#
+.SYNOPSIS
+    Reports Windows Server Backup (WSB) results to the Backup Dashboard.
+
+.DESCRIPTION
+    Runs on a server with the Windows Server Backup feature, normally as a SYSTEM scheduled task
+    every 60 minutes (see Install-WsbCollector.ps1). The server appears on the dashboard as one job.
+    Each run sends:
+      - the next scheduled backup (Get-WBSummary's NextBackupTime), which drives the dashboard's
+        Overdue flag, and
+      - every backup from the last LookbackHours that the dashboard hasn't already received in its
+        final state. A backup in progress is sent as Running until it finishes.
+
+    WSB jobs have no ID, so a run is identified by its start time. Already-reported runs are
+    tracked in a state file, and the ingest API is idempotent, so a failed or repeated run is
+    harmless: anything not confirmed is retried next time.
+
+    Written for Windows PowerShell 5.1, which the WindowsServerBackup module targets.
+
+.PARAMETER ConfigPath
+    JSON config with ApiUrl, ApiKey and Tenant (optional: JobName, SourceSystem, LookbackHours).
+
+.PARAMETER LookbackHours
+    Overrides the config value. Use 720 once to backfill 30 days of history.
+
+.PARAMETER DryRun
+    Print the payloads instead of sending them. Doesn't touch the state or log files.
+
+.EXAMPLE
+    .\Send-WsbBackupStatus.ps1 -DryRun
+.EXAMPLE
+    .\Send-WsbBackupStatus.ps1 -LookbackHours 720
+#>
+[CmdletBinding()]
+param(
+    [string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\wsb-collector.json'),
+    [int]$LookbackHours = 0,
+    [switch]$DryRun
+)
+
+$ErrorActionPreference = 'Stop'
+
+$DataDir = Split-Path -Parent $ConfigPath
+$LogPath = Join-Path $DataDir 'wsb-collector.log'
+$StatePath = Join-Path $DataDir 'wsb-collector.state.json'
+$MaxLogBytes = 5MB
+# Covers 30 days at several backups a day
+$MaxPreviousJobs = 200
+$Invariant = [Globalization.CultureInfo]::InvariantCulture
+
+# ---------- Helpers ----------
+
+function Write-Log {
+    param([string]$Message, [string]$Level = 'INFO')
+    $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
+    Write-Host $line
+    if ($DryRun) { return }
+    if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) {
+        Move-Item $LogPath "$LogPath.1" -Force
+    }
+    Add-Content -Path $LogPath -Value $line -Encoding UTF8
+}
+
+function Get-ErrorText($ErrorRecord) {
+    # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
+    if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
+        return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
+    }
+    return $ErrorRecord.Exception.Message
+}
+
+# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a
+# Utc time as +00:00.
+function ConvertTo-IsoTime($Value) {
+    if ($null -eq $Value) { return $null }
+    $dt = [datetime]$Value
+    if ($dt.Year -le 1900) { return $null }  # "not set" (DateTime.MinValue and friends)
+    return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant)
+}
+
+# First property of $Object, out of $Names, that exists and isn't empty. The WSB object model isn't
+# fully documented, so reads go through this rather than assuming one spelling.
+function Get-FirstValue($Object, [string[]]$Names) {
+    if ($null -eq $Object) { return $null }
+    foreach ($name in $Names) {
+        try {
+            $prop = $Object.PSObject.Properties[$name]
+            if (-not $prop) { continue }
+            $value = $prop.Value
+            # Check emptiness by type: casting to [string] would make a one-item list of objects ''
+            if ($null -eq $value) { continue }
+            if ($value -is [string] -and $value -eq '') { continue }
+            if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue }
+            return $value
+        } catch { }
+    }
+    return $null
+}
+
+function Import-Wsb {
+    if (Get-Command Get-WBJob -ErrorAction SilentlyContinue) { return }  # already loaded
+    # Present only when the Windows Server Backup feature (with its command-line tools) is installed
+    Import-Module WindowsServerBackup
+}
+
+# ---------- Jobs ----------
+
+function Get-JobTime($Job, [string]$Field) {
+    $value = Get-FirstValue $Job @($Field)
+    if ($null -eq $value) { return $null }
+    $dt = [datetime]$value
+    if ($dt.Year -le 1900) { return $null }  # e.g. EndTime while a backup is running
+    return $dt
+}
+
+# HRESULTs as numbers; 0 = OK. $null when the job doesn't have the property.
+function Get-HResult($Object, [string]$Field) {
+    $value = Get-FirstValue $Object @($Field)
+    if ($null -eq $value) { return $null }
+    return [long]$value
+}
+
+function Format-HResult([long]$Value) {
+    if ($Value -lt 0) { $Value += 4294967296 }  # show as the familiar unsigned 0x8XXXXXXX
+    return '0x{0:X8}' -f $Value
+}
+
+function Get-JobItems($Job) {
+    return @(Get-FirstValue $Job @('JobItems', 'Items') | Where-Object { $_ })
+}
+
+function Get-ItemName($Item) {
+    $name = Get-FirstValue $Item @('Name', 'VolumeLabel', 'MountPath', 'Path', 'Type')
+    if ($name) { return [string]$name }
+    return 'Item'
+}
+
+# Items (volumes, system state, ...) that reported an error
+function Get-FailedItems($Job) {
+    return @(Get-JobItems $Job | Where-Object {
+        $hr = Get-HResult $_ 'HResult'
+        $state = [string](Get-FirstValue $_ @('State', 'JobState'))
+        ($hr -and $hr -ne 0) -or $state -match 'Fail'
+    })
+}
+
+function Get-RunStatus($Job) {
+    $state = [string](Get-FirstValue $Job @('JobState', 'State'))
+    if (-not (Get-JobTime $Job 'EndTime') -and $state -notmatch 'Complet|Fail|Cancel|Abort|Stop|Succe') {
+        return 'running'
+    }
+    if ($state -match 'Fail') { return 'failed' }
+    if ($state -match 'Cancel|Abort|Stop') { return 'warning' }
+    $hr = Get-HResult $Job 'HResult'
+    if ($hr -and $hr -ne 0) { return 'failed' }
+    # Overall success but something underneath went wrong (e.g. a volume or files skipped)
+    $detailed = Get-HResult $Job 'DetailedHResult'
+    if (($detailed -and $detailed -ne 0) -or (Get-FailedItems $Job).Count) { return 'warning' }
+    if (-not $state -or $state -match 'Complet|Succe') { return 'success' }
+    return 'warning'  # finished in a state we don't recognise: surface it rather than hide it
+}
+
+function Get-RunMessage($Job, [string]$Status) {
+    $lines = New-Object 'System.Collections.Generic.List[string]'
+    $state = [string](Get-FirstValue $Job @('JobState', 'State'))
+    $description = Get-FirstValue $Job @('ErrorDescription', 'DetailedMessage')
+    if ($description) { $lines.Add(([string]$description).Trim()) }
+
+    $hr = Get-HResult $Job 'HResult'
+    $detailed = Get-HResult $Job 'DetailedHResult'
+    if (($hr -and $hr -ne 0) -or ($detailed -and $detailed -ne 0)) {
+        $codes = @()
+        if ($hr -and $hr -ne 0) { $codes += "error $(Format-HResult $hr)" }
+        if ($detailed -and $detailed -ne 0) { $codes += "detailed error $(Format-HResult $detailed)" }
+        $lines.Add('Windows Server Backup reported ' + ($codes -join ', ') + '.')
+    }
+
+    foreach ($item in Get-FailedItems $Job) {
+        $itemError = Get-FirstValue $item @('ErrorDescription', 'DetailedMessage')
+        $itemHr = Get-HResult $item 'HResult'
+        $line = "$(Get-ItemName $item) - "  # names are often volumes like "D:", so no colon
+        if ($itemError) { $line += ([string]$itemError).Trim() }
+        elseif ($itemHr) { $line += "error $(Format-HResult $itemHr)" }
+        else { $line += [string](Get-FirstValue $item @('State', 'JobState')) }
+        $lines.Add($line)
+    }
+
+    if ($Status -ne 'success') {
+        $failureLog = Get-FirstValue $Job @('FailureLogPath')
+        if ($failureLog) { $lines.Add("Failure log: $failureLog") }
+    }
+
+    if ($lines.Count -eq 0) {
+        switch ($Status) {
+            'success' { return 'Backup completed successfully.' }
+            'running' { return '' }
+            default   { return "Backup finished with state '$state'." }
+        }
+    }
+    # WSB reports "Completed" even for a backup that failed with an error code, so only mention
+    # the state when it says something else (e.g. cancelled)
+    if ($state -and $Status -ne 'success' -and $state -notmatch 'Complet') { $lines.Insert(0, "Job state: $state") }
+    return ($lines -join "`n")
+}
+
+function ConvertTo-Run($Job) {
+    $status = Get-RunStatus $Job
+    $details = [ordered]@{ server = $env:COMPUTERNAME }
+    $items = @(Get-JobItems $Job | ForEach-Object { Get-ItemName $_ })
+    if ($items.Count) { $details.items = $items }
+    $state = Get-FirstValue $Job @('JobState', 'State')
+    if ($state) { $details.wsb_state = [string]$state }
+    $version = Get-FirstValue $Job @('VersionId')
+    if ($version) { $details.version_id = [string]$version }
+
+    # No external_id: WSB jobs have no ID, so the API keys the run on started_at
+    $run = [ordered]@{
+        status     = $status
+        started_at = ConvertTo-IsoTime (Get-JobTime $Job 'StartTime')
+        message    = Get-RunMessage $Job $status
+        details    = $details
+    }
+    if ($status -ne 'running') {
+        $finished = ConvertTo-IsoTime (Get-JobTime $Job 'EndTime')
+        if ($finished) { $run.finished_at = $finished }  # otherwise the API uses receipt time
+    }
+    return $run
+}
+
+function Test-IsBackupJob($Job) {
+    # Get-WBJob also lists recoveries; only backups belong on the dashboard
+    $type = [string](Get-FirstValue $Job @('JobType', 'Type'))
+    return (-not $type) -or $type -match 'Backup'
+}
+
+function Get-NextRun {
+    try {
+        $summary = Get-WBSummary
+        return ConvertTo-IsoTime (Get-FirstValue $summary @('NextBackupTime'))
+    } catch {
+        Write-Log "Couldn't read the backup summary: $(Get-ErrorText $_)" 'WARN'
+        return $null
+    }
+}
+
+function Send-Report($Body) {
+    $json = $Body | ConvertTo-Json -Depth 6 -Compress
+    if ($DryRun) {
+        Write-Host $json
+        return
+    }
+    # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1
+    $params = @{
+        Uri         = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
+        Method      = 'Post'
+        Headers     = @{ Authorization = "Bearer $($Config.ApiKey)" }
+        ContentType = 'application/json; charset=utf-8'
+        Body        = [Text.Encoding]::UTF8.GetBytes($json)
+        TimeoutSec  = 30
+    }
+    Invoke-RestMethod @params | Out-Null
+}
+
+# ---------- Main ----------
+
+try {
+    $Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
+    foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') {
+        if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" }
+    }
+    $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Windows Server Backup' }
+    $JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Windows Server Backup" }
+    if ($LookbackHours -le 0) {
+        $LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 }
+    }
+
+    # Windows PowerShell 5.1 may default to TLS 1.0
+    [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
+
+    Import-Wsb
+
+    # Run start time (ISO, as sent) -> Unix seconds, for runs already delivered in their final state
+    $sent = @{}
+    if (Test-Path $StatePath) {
+        $saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json
+        if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } }
+    }
+
+    $since = (Get-Date).AddHours(-$LookbackHours)
+    # Get-WBJob with no arguments returns the job in progress (if any); -Previous lists finished ones
+    $jobs = @(@(Get-WBJob) + @(Get-WBJob -Previous $MaxPreviousJobs) | Where-Object {
+        $_ -and (Test-IsBackupJob $_) -and (Get-JobTime $_ 'StartTime')
+    })
+    $byStart = [ordered]@{}
+    foreach ($job in $jobs) {
+        $start = Get-JobTime $job 'StartTime'
+        if ($start -lt $since) { continue }
+        $byStart[(ConvertTo-IsoTime $start)] = $job  # a job in both lists: the finished entry (later) wins
+    }
+    $pending = @($byStart.Keys | Sort-Object | Where-Object { -not $sent.ContainsKey($_) } |
+        ForEach-Object { $byStart[$_] })
+
+    $jobInfo = [ordered]@{
+        # Stable per server, so renaming the dashboard job via JobName doesn't create a new one
+        external_id = "wsb:$($env:COMPUTERNAME.ToLowerInvariant())"
+        name        = $JobName
+        # Always sent (null when unscheduled) so the dashboard's value tracks WSB's
+        next_run_at = Get-NextRun
+    }
+} catch {
+    Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR'
+    exit 2
+}
+
+$failures = 0
+$runsSent = 0
+try {
+    if ($pending.Count -eq 0) {
+        # Keep next_run_at fresh even when nothing new ran
+        Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo })
+    }
+    foreach ($job in $pending) {
+        $run = ConvertTo-Run $job
+        Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run })
+        $runsSent++
+        if ($run.status -ne 'running') {
+            $sent[$run.started_at] = ([DateTimeOffset](Get-JobTime $job 'StartTime')).ToUnixTimeSeconds()
+        }
+    }
+} catch {
+    $failures++
+    Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR'
+}
+
+if (-not $DryRun) {
+    # Forget runs that have aged out of the lookback window (plus a day of slack)
+    $cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds()
+    $keep = [ordered]@{}
+    foreach ($key in $sent.Keys) {
+        if ($sent[$key] -ge $cutoff) { $keep[$key] = $sent[$key] }
+    }
+    try {
+        $keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8
+    } catch {
+        Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
+    }
+}
+
+Write-Log ("Reported {0} runs, {1} failures" -f $runsSent, $failures)
+if ($failures) { exit 1 }