kyleandClaude Opus 5.5
118a7f978d
Checked with Get-Vb365CollectorDiagnostics.ps1 output from a VB365 8.4
server (GDC-VEEAM-04, 11 organizations, Windows PowerShell 5.1). Times
(session CreationTime/EndTime, job NextRun) are local, and property
names matched. Two fixes:
- Session log items carry their status as a Title prefix ("[Warning]
...") with no separate status property. Match on the prefix and strip
it, instead of matching words in the text, which would have pulled in
lines like "[Success] Processing site: .../error-reports"
- Retries are separate sessions (MainSessionId -> the retried session,
which has JobWillBeRetried = True). Mark them in details (retry_of,
will_retry) and say "Veeam will retry this job." on a pending retry
Mock rebuilt on the real shapes, including a warning + retry pair.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
373 lines
16 KiB
PowerShell
373 lines
16 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Reports Veeam Backup for Microsoft 365 (VB365) job results to the Backup Dashboard.
|
|
|
|
.DESCRIPTION
|
|
Runs on the VB365 server, normally as a SYSTEM scheduled task every 60 minutes
|
|
(see Install-Vb365Collector.ps1). For every backup job (and backup copy job) it sends:
|
|
- the job's next scheduled run, which drives the dashboard's Overdue flag, and
|
|
- every session from the last LookbackHours that the dashboard hasn't already received in
|
|
its final state. Running sessions are re-sent each time until they finish.
|
|
|
|
One VB365 server can protect several Microsoft 365 organizations, so the dashboard company is
|
|
chosen per job from its organization: OrganizationTenants in the config maps an organization
|
|
name (e.g. "contoso.onmicrosoft.com") to a company; otherwise Tenant is used; otherwise the
|
|
organization's own name.
|
|
|
|
Already-reported sessions are tracked in a state file, and the ingest API is idempotent, so a
|
|
failed or repeated run is harmless: anything not confirmed is retried next time.
|
|
|
|
.PARAMETER ConfigPath
|
|
JSON config with ApiUrl and ApiKey (optional: Tenant, OrganizationTenants, SourceSystem,
|
|
LookbackHours).
|
|
|
|
.PARAMETER LookbackHours
|
|
Overrides the config value. Use 720 once to backfill 30 days of history.
|
|
|
|
.PARAMETER DryRun
|
|
Print the payloads instead of sending them. Doesn't touch the state or log files.
|
|
|
|
.EXAMPLE
|
|
.\Send-Vb365BackupStatus.ps1 -DryRun
|
|
.EXAMPLE
|
|
.\Send-Vb365BackupStatus.ps1 -LookbackHours 720
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\vb365-collector.json'),
|
|
[int]$LookbackHours = 0,
|
|
[switch]$DryRun
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
$DataDir = Split-Path -Parent $ConfigPath
|
|
$LogPath = Join-Path $DataDir 'vb365-collector.log'
|
|
$StatePath = Join-Path $DataDir 'vb365-collector.state.json'
|
|
$MaxLogBytes = 5MB
|
|
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
|
|
|
# ---------- Helpers ----------
|
|
|
|
function Write-Log {
|
|
param([string]$Message, [string]$Level = 'INFO')
|
|
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
|
|
Write-Host $line
|
|
if ($DryRun) { return }
|
|
if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) {
|
|
Move-Item $LogPath "$LogPath.1" -Force
|
|
}
|
|
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
|
}
|
|
|
|
function Get-ErrorText($ErrorRecord) {
|
|
# Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
|
|
if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
|
|
return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
|
|
}
|
|
return $ErrorRecord.Exception.Message
|
|
}
|
|
|
|
# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a
|
|
# Utc time as +00:00.
|
|
function ConvertTo-IsoTime($Value) {
|
|
if ($null -eq $Value) { return $null }
|
|
$dt = [datetime]$Value
|
|
if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue and friends)
|
|
return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant)
|
|
}
|
|
|
|
# First property of $Object, out of $Names, that exists and isn't empty. The VB365 object model isn't
|
|
# fully documented, so reads go through this rather than assuming one spelling.
|
|
function Get-FirstValue($Object, [string[]]$Names) {
|
|
if ($null -eq $Object) { return $null }
|
|
foreach ($name in $Names) {
|
|
try {
|
|
$prop = $Object.PSObject.Properties[$name]
|
|
if (-not $prop) { continue }
|
|
$value = $prop.Value
|
|
# Check emptiness by type: casting to [string] would make a one-item list of objects ''
|
|
if ($null -eq $value) { continue }
|
|
if ($value -is [string] -and $value -eq '') { continue }
|
|
if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue }
|
|
return $value
|
|
} catch { }
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Import-Vb365 {
|
|
if (Get-Command Get-VBOJob -ErrorAction SilentlyContinue) { return } # already loaded
|
|
if (Get-Module -ListAvailable -Name Veeam.Archiver.PowerShell) {
|
|
Import-Module Veeam.Archiver.PowerShell
|
|
return
|
|
}
|
|
# Fall back to the default install path
|
|
Import-Module (Join-Path $env:ProgramFiles 'Veeam\Backup365\Veeam.Archiver.PowerShell\Veeam.Archiver.PowerShell.psd1')
|
|
}
|
|
|
|
function Get-Time($Object, [string]$Field) {
|
|
$value = Get-FirstValue $Object @($Field)
|
|
if ($null -eq $value) { return $null }
|
|
$dt = [datetime]$value
|
|
if ($dt.Year -le 1900) { return $null } # e.g. EndTime while a session is running
|
|
return $dt
|
|
}
|
|
|
|
# "1.5 GB", "512 MB", "123456" -> bytes; $null when it can't be read
|
|
function ConvertTo-Bytes($Value) {
|
|
if ($null -eq $Value) { return $null }
|
|
if ($Value -is [ValueType]) { return [long]$Value }
|
|
$match = [regex]::Match([string]$Value, '^\s*([\d.,]+)\s*([KMGTP]?B)?\s*$', 'IgnoreCase')
|
|
if (-not $match.Success) { return $null }
|
|
$number = 0.0
|
|
if (-not [double]::TryParse(($match.Groups[1].Value -replace ',', '.'), [Globalization.NumberStyles]::Float, $Invariant, [ref]$number)) {
|
|
return $null
|
|
}
|
|
$power = @{ '' = 0; 'B' = 0; 'KB' = 1; 'MB' = 2; 'GB' = 3; 'TB' = 4; 'PB' = 5 }[$match.Groups[2].Value.ToUpperInvariant()]
|
|
return [long]($number * [math]::Pow(1024, $power))
|
|
}
|
|
|
|
# ---------- Jobs and tenants ----------
|
|
|
|
function Get-OrganizationName($Job) {
|
|
$org = Get-FirstValue $Job @('Organization')
|
|
if ($null -eq $org) {
|
|
# Copy jobs point at their source backup job
|
|
$org = Get-FirstValue (Get-FirstValue $Job @('BackupJob')) @('Organization')
|
|
}
|
|
if ($org -is [string]) { return $org }
|
|
$name = Get-FirstValue $org @('Name', 'OfficeName', 'DisplayName')
|
|
if ($name) { return [string]$name }
|
|
return $null
|
|
}
|
|
|
|
function Get-TenantForJob($Job) {
|
|
$org = Get-OrganizationName $Job
|
|
if ($org -and $Config.OrganizationTenants) {
|
|
$mapped = $Config.OrganizationTenants.PSObject.Properties[$org] # case-insensitive lookup
|
|
if ($mapped -and $mapped.Value) { return [string]$mapped.Value }
|
|
}
|
|
if ($Config.Tenant) { return [string]$Config.Tenant }
|
|
if ($org) { return $org }
|
|
return $env:COMPUTERNAME
|
|
}
|
|
|
|
function Get-NextRun($Job) {
|
|
try {
|
|
$enabled = Get-FirstValue $Job @('IsEnabled', 'Enabled')
|
|
if ($enabled -eq $false) { return $null }
|
|
return ConvertTo-IsoTime (Get-Time $Job 'NextRun')
|
|
} catch {
|
|
Write-Log "Couldn't read schedule for job '$($Job.Name)': $(Get-ErrorText $_)" 'WARN'
|
|
return $null
|
|
}
|
|
}
|
|
|
|
# ---------- Sessions ----------
|
|
|
|
function Get-RunStatus($Session) {
|
|
$status = [string](Get-FirstValue $Session @('Status', 'Result', 'State'))
|
|
if ($status -match 'Fail|Error') { return 'failed' }
|
|
if ($status -match 'Warn') { return 'warning' }
|
|
if ($status -match 'Succe') { return 'success' }
|
|
if ($status -match 'Run|Queue|Progress|Start|Wait') { return 'running' }
|
|
if (-not (Get-Time $Session 'EndTime')) { return 'running' }
|
|
# Stopped (cancelled), Disconnected, or a state we don't recognise: surface it rather than hide it
|
|
return 'warning'
|
|
}
|
|
|
|
function Test-EmptyGuid($Value) {
|
|
return (-not $Value) -or [string]$Value -eq [string][guid]::Empty
|
|
}
|
|
|
|
# Warning/error lines from the session log. Each log item's status is a prefix of its Title, e.g.
|
|
# "[Warning] Item was skipped" (there's no separate status property), so match on that rather than
|
|
# on words in the text, which would catch e.g. "[Success] Processing site: .../error-reports".
|
|
function Get-RunMessage($Session, [string]$Status) {
|
|
if ($Status -eq 'running') { return '' }
|
|
$lines = New-Object 'System.Collections.Generic.List[string]'
|
|
$sessionStatus = [string](Get-FirstValue $Session @('Status', 'Result', 'State'))
|
|
if ($Status -ne 'success') {
|
|
foreach ($entry in @(Get-FirstValue $Session @('Log', 'Logs') | Where-Object { $_ })) {
|
|
$title = ([string](Get-FirstValue $entry @('Title', 'Message', 'Text'))).Trim()
|
|
$prefix = [regex]::Match($title, '^\[(\w+)\]\s*')
|
|
if ($prefix.Success) {
|
|
if ($prefix.Groups[1].Value -notmatch 'Warn|Fail|Error') { continue }
|
|
$title = $title.Substring($prefix.Length)
|
|
} elseif ([string](Get-FirstValue $entry @('Status', 'Type', 'Severity')) -notmatch 'Warn|Fail|Error') {
|
|
continue
|
|
}
|
|
if ($title -and -not $lines.Contains($title)) { $lines.Add($title) }
|
|
}
|
|
}
|
|
if ($lines.Count -eq 0) {
|
|
if ($Status -eq 'success') { $lines.Add('Job completed successfully.') }
|
|
else { $lines.Add("Job finished with status '$sessionStatus'.") }
|
|
} elseif ($sessionStatus -and $sessionStatus -notmatch 'Fail|Warn') {
|
|
$lines.Insert(0, "Session status: $sessionStatus")
|
|
}
|
|
# A warning/failure VB365 is about to retry; the retry arrives as its own run
|
|
if ($Status -ne 'success' -and (Get-FirstValue $Session @('JobWillBeRetried')) -eq $true) {
|
|
$lines.Add('Veeam will retry this job.')
|
|
}
|
|
return ($lines -join "`n")
|
|
}
|
|
|
|
function ConvertTo-Run($Session, $Job, [string]$JobKind) {
|
|
$status = Get-RunStatus $Session
|
|
$details = [ordered]@{ vb365_server = $env:COMPUTERNAME; job_kind = $JobKind }
|
|
$org = Get-OrganizationName $Job
|
|
if ($org) { $details.organization = $org }
|
|
$sessionStatus = Get-FirstValue $Session @('Status', 'Result', 'State')
|
|
if ($sessionStatus) { $details.vb365_status = [string]$sessionStatus }
|
|
# Retries are separate sessions pointing at the session they retry
|
|
$mainSession = Get-FirstValue $Session @('MainSessionId')
|
|
if (-not (Test-EmptyGuid $mainSession)) { $details.retry_of = [string]$mainSession }
|
|
if ((Get-FirstValue $Session @('JobWillBeRetried')) -eq $true) { $details.will_retry = $true }
|
|
|
|
$run = [ordered]@{
|
|
external_id = [string](Get-FirstValue $Session @('Id', 'SessionId'))
|
|
status = $status
|
|
started_at = ConvertTo-IsoTime (Get-Time $Session 'CreationTime')
|
|
message = Get-RunMessage $Session $status
|
|
details = $details
|
|
}
|
|
if ($status -ne 'running') {
|
|
$finished = ConvertTo-IsoTime (Get-Time $Session 'EndTime')
|
|
if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time
|
|
$stats = Get-FirstValue $Session @('Statistics')
|
|
$processed = Get-FirstValue $stats @('ProcessedObjects')
|
|
if ($null -ne $processed) { $details.processed_objects = $processed }
|
|
$transferred = ConvertTo-Bytes (Get-FirstValue $stats @('TransferredData', 'TransferredDataSize'))
|
|
if ($transferred -gt 0) { $run.size_bytes = $transferred }
|
|
}
|
|
return $run
|
|
}
|
|
|
|
function Send-Report($Body) {
|
|
$json = $Body | ConvertTo-Json -Depth 6 -Compress
|
|
if ($DryRun) {
|
|
Write-Host $json
|
|
return
|
|
}
|
|
# Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1
|
|
$params = @{
|
|
Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
|
|
Method = 'Post'
|
|
Headers = @{ Authorization = "Bearer $($Config.ApiKey)" }
|
|
ContentType = 'application/json; charset=utf-8'
|
|
Body = [Text.Encoding]::UTF8.GetBytes($json)
|
|
TimeoutSec = 30
|
|
}
|
|
Invoke-RestMethod @params | Out-Null
|
|
}
|
|
|
|
# ---------- Main ----------
|
|
|
|
try {
|
|
$Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
|
|
foreach ($key in 'ApiUrl', 'ApiKey') {
|
|
if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" }
|
|
}
|
|
$SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Veeam Backup for M365' }
|
|
if ($LookbackHours -le 0) {
|
|
$LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 }
|
|
}
|
|
|
|
# Windows PowerShell 5.1 may default to TLS 1.0
|
|
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
|
|
Import-Vb365
|
|
|
|
# Session ID -> start time (Unix seconds), for sessions already delivered in their final state
|
|
$sent = @{}
|
|
if (Test-Path $StatePath) {
|
|
$saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json
|
|
if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } }
|
|
}
|
|
|
|
$since = (Get-Date).AddHours(-$LookbackHours)
|
|
|
|
# (job, kind, sessions) for backup jobs, plus backup copy jobs where the cmdlets exist
|
|
$work = New-Object 'System.Collections.Generic.List[object]'
|
|
foreach ($job in @(Get-VBOJob)) {
|
|
$sessions = @(Get-VBOJobSession -Job $job | Where-Object { (Get-Time $_ 'CreationTime') -ge $since })
|
|
$work.Add([pscustomobject]@{ Job = $job; Kind = 'backup'; Sessions = $sessions })
|
|
}
|
|
if ((Get-Command Get-VBOCopyJob -ErrorAction SilentlyContinue) -and (Get-Command Get-VBOCopyJobSession -ErrorAction SilentlyContinue)) {
|
|
$copyJobs = @(Get-VBOCopyJob)
|
|
if ($copyJobs.Count) {
|
|
# Fetch once and match by job ID, since the per-job parameter name isn't consistent
|
|
$copySessions = @(Get-VBOCopyJobSession | Where-Object { (Get-Time $_ 'CreationTime') -ge $since })
|
|
foreach ($job in $copyJobs) {
|
|
$id = [string]$job.Id
|
|
$work.Add([pscustomobject]@{
|
|
Job = $job; Kind = 'copy'
|
|
Sessions = @($copySessions | Where-Object { [string](Get-FirstValue $_ @('JobId')) -eq $id })
|
|
})
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR'
|
|
exit 2
|
|
}
|
|
|
|
$failures = 0
|
|
$runsSent = 0
|
|
foreach ($item in $work) {
|
|
$job = $item.Job
|
|
$tenant = Get-TenantForJob $job
|
|
$jobInfo = [ordered]@{
|
|
external_id = [string]$job.Id
|
|
name = [string]$job.Name
|
|
# Always sent (null when disabled/unscheduled) so the dashboard's value tracks VB365's
|
|
next_run_at = Get-NextRun $job
|
|
}
|
|
$pending = @($item.Sessions |
|
|
Where-Object { $_ -and -not $sent.ContainsKey([string](Get-FirstValue $_ @('Id', 'SessionId'))) } |
|
|
Sort-Object { Get-Time $_ 'CreationTime' })
|
|
|
|
try {
|
|
if ($pending.Count -eq 0) {
|
|
# Keep next_run_at fresh even when nothing new ran
|
|
Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo })
|
|
}
|
|
foreach ($session in $pending) {
|
|
$run = ConvertTo-Run $session $job $item.Kind
|
|
Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo; run = $run })
|
|
$runsSent++
|
|
if ($run.status -ne 'running') {
|
|
$sent[$run.external_id] = ([DateTimeOffset](Get-Time $session 'CreationTime')).ToUnixTimeSeconds()
|
|
}
|
|
}
|
|
} catch {
|
|
$failures++
|
|
Write-Log "Failed to report job '$($job.Name)': $(Get-ErrorText $_)" 'ERROR'
|
|
$httpStatus = 0
|
|
try { $httpStatus = [int]$_.Exception.Response.StatusCode } catch { }
|
|
if ($httpStatus -eq 401) {
|
|
Write-Log 'API key rejected; check ApiKey in the config. Stopping.' 'ERROR'
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if (-not $DryRun) {
|
|
# Forget sessions that have aged out of the lookback window (plus a day of slack)
|
|
$cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds()
|
|
$keep = [ordered]@{}
|
|
foreach ($id in $sent.Keys) {
|
|
if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] }
|
|
}
|
|
try {
|
|
$keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8
|
|
} catch {
|
|
Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
|
|
}
|
|
}
|
|
|
|
Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $work.Count, $runsSent, $failures)
|
|
if ($failures) { exit 1 }
|