diff --git a/collectors/vb365/README.md b/collectors/vb365/README.md index f2dbbcd..9c005d4 100644 --- a/collectors/vb365/README.md +++ b/collectors/vb365/README.md @@ -3,10 +3,10 @@ A PowerShell script that runs on the Veeam Backup for Microsoft 365 (VB365) server and reports its backup jobs (and backup copy jobs) to the dashboard, one dashboard job per VB365 job. -> **Not yet checked against a real server.** The cmdlets are VB365's own (`Get-VBOJob`, -> `Get-VBOJobSession`), but the properties of the objects they return aren't fully documented. -> Before installing on the first server, run `Get-Vb365CollectorDiagnostics.ps1` and the collector -> with `-DryRun` (see below) to confirm what it reads. +> **Checked against:** VB365 8.4 under Windows PowerShell 5.1, a shared server protecting 11 +> organizations: successful, warning and retried sessions. Not yet seen on a real server: a failed +> session, a session in progress, or a backup copy job. If one looks wrong, run +> `Get-Vb365CollectorDiagnostics.ps1` while it's in that state and send the output. ## Which company a job appears under @@ -63,7 +63,8 @@ Re-run the installer to change settings (including the organization mapping) or | Status | Session status: Success / Warning / Failed. Running or queued → **Running**. Stopped (cancelled) or unrecognised → **Warning** | | Started / finished | Session `CreationTime` / `EndTime` | | Next scheduled | Job's `NextRun`. Empty for disabled jobs | -| Message | Warning and error lines from the session log | +| Message | Warning and error lines from the session log (items titled `[Warning] ...`, `[Failed] ...`), plus "Veeam will retry this job." when a retry is pending | +| Retries | VB365 runs a retry as a separate session, so it appears as its own run, marked `details.retry_of` (the session it retried). The retried run is marked `details.will_retry` | | Size | Data transferred (`Statistics.TransferredData`) | | Details | Organization, objects processed, job kind (`backup` / `copy`) | diff --git a/collectors/vb365/Send-Vb365BackupStatus.ps1 b/collectors/vb365/Send-Vb365BackupStatus.ps1 index 6140b52..ce4b701 100644 --- a/collectors/vb365/Send-Vb365BackupStatus.ps1 +++ b/collectors/vb365/Send-Vb365BackupStatus.ps1 @@ -177,26 +177,40 @@ function Get-RunStatus($Session) { return 'warning' } -# Warning/error lines from the session log +function Test-EmptyGuid($Value) { + return (-not $Value) -or [string]$Value -eq [string][guid]::Empty +} + +# Warning/error lines from the session log. Each log item's status is a prefix of its Title, e.g. +# "[Warning] Item was skipped" (there's no separate status property), so match on that rather than +# on words in the text, which would catch e.g. "[Success] Processing site: .../error-reports". function Get-RunMessage($Session, [string]$Status) { if ($Status -eq 'running') { return '' } $lines = New-Object 'System.Collections.Generic.List[string]' $sessionStatus = [string](Get-FirstValue $Session @('Status', 'Result', 'State')) if ($Status -ne 'success') { foreach ($entry in @(Get-FirstValue $Session @('Log', 'Logs') | Where-Object { $_ })) { - $title = [string](Get-FirstValue $entry @('Title', 'Message', 'Text')) - $entryText = "$title $([string](Get-FirstValue $entry @('Status', 'Type', 'Severity')))" - if ($title -and $entryText -match 'Warn|Fail|Error') { - $title = $title.Trim() - if (-not $lines.Contains($title)) { $lines.Add($title) } + $title = ([string](Get-FirstValue $entry @('Title', 'Message', 'Text'))).Trim() + $prefix = [regex]::Match($title, '^\[(\w+)\]\s*') + if ($prefix.Success) { + if ($prefix.Groups[1].Value -notmatch 'Warn|Fail|Error') { continue } + $title = $title.Substring($prefix.Length) + } elseif ([string](Get-FirstValue $entry @('Status', 'Type', 'Severity')) -notmatch 'Warn|Fail|Error') { + continue } + if ($title -and -not $lines.Contains($title)) { $lines.Add($title) } } } if ($lines.Count -eq 0) { - if ($Status -eq 'success') { return 'Job completed successfully.' } - return "Job finished with status '$sessionStatus'." + if ($Status -eq 'success') { $lines.Add('Job completed successfully.') } + else { $lines.Add("Job finished with status '$sessionStatus'.") } + } elseif ($sessionStatus -and $sessionStatus -notmatch 'Fail|Warn') { + $lines.Insert(0, "Session status: $sessionStatus") + } + # A warning/failure VB365 is about to retry; the retry arrives as its own run + if ($Status -ne 'success' -and (Get-FirstValue $Session @('JobWillBeRetried')) -eq $true) { + $lines.Add('Veeam will retry this job.') } - if ($sessionStatus -and $sessionStatus -notmatch 'Fail|Warn') { $lines.Insert(0, "Session status: $sessionStatus") } return ($lines -join "`n") } @@ -207,6 +221,10 @@ function ConvertTo-Run($Session, $Job, [string]$JobKind) { if ($org) { $details.organization = $org } $sessionStatus = Get-FirstValue $Session @('Status', 'Result', 'State') if ($sessionStatus) { $details.vb365_status = [string]$sessionStatus } + # Retries are separate sessions pointing at the session they retry + $mainSession = Get-FirstValue $Session @('MainSessionId') + if (-not (Test-EmptyGuid $mainSession)) { $details.retry_of = [string]$mainSession } + if ((Get-FirstValue $Session @('JobWillBeRetried')) -eq $true) { $details.will_retry = $true } $run = [ordered]@{ external_id = [string](Get-FirstValue $Session @('Id', 'SessionId')) diff --git a/collectors/vb365/tests/MockVb365.ps1 b/collectors/vb365/tests/MockVb365.ps1 index e95f2d6..5b3446d 100644 --- a/collectors/vb365/tests/MockVb365.ps1 +++ b/collectors/vb365/tests/MockVb365.ps1 @@ -1,5 +1,8 @@ -# Stand-ins for the Veeam Backup for Microsoft 365 cmdlets. Object shapes are best guesses until -# checked against Get-Vb365CollectorDiagnostics.ps1 output from a real server. +# Stand-ins for the Veeam Backup for Microsoft 365 cmdlets. Object shapes follow +# Get-Vb365CollectorDiagnostics.ps1 output from a real VB365 8.4 server: session and NextRun times +# are local, Statistics.TransferredData is a string ("187.2 MB"), log items carry their status as a +# Title prefix ("[Warning] ..."), and retries are separate sessions whose MainSessionId points at +# the session they retry (which has JobWillBeRetried = True). # # . ./tests/MockVb365.ps1 # ./Send-Vb365BackupStatus.ps1 -ConfigPath ./my-test-config.json -DryRun @@ -10,62 +13,66 @@ function New-StableGuid([string]$Seed) { [guid]::new([Security.Cryptography.MD5]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($Seed))) } -$acme = [pscustomobject]@{ Name = 'acme.onmicrosoft.com'; Type = 'Office365' } -$blue = [pscustomobject]@{ Name = 'bluewater.onmicrosoft.com'; Type = 'Office365' } +function New-MockOrg($Name) { [pscustomobject]@{ Name = $Name; OfficeName = $Name; Type = 'Office365'; IsBackedUp = $true } } +$acme = New-MockOrg 'acme.onmicrosoft.com' +$blue = New-MockOrg 'bluewater.onmicrosoft.com' function New-MockJob($Name, $Org, $NextRun, [bool]$Enabled = $true) { [pscustomobject]@{ - Id = New-StableGuid $Name; Name = $Name; Organization = $Org + Id = New-StableGuid $Name; Name = $Name; Organization = $Org; JobType = 'Backup' IsEnabled = $Enabled; NextRun = $NextRun; LastStatus = 'Success' + LastRun = $(if ($NextRun) { $NextRun.AddDays(-1).ToUniversalTime() }) # UTC, as on the real server } } -function New-MockSession($Job, $Start, $End, $Status, $Transferred = '0 B', [int]$Objects = 0, $Log = @()) { +function New-LogItem($Status, $Text, $At) { + [pscustomobject]@{ Id = [guid]::NewGuid(); USN = 1; Title = "[$Status] $Text"; CreationTime = $At; EndTime = $At } +} + +function New-MockSession($Job, $Start, $End, $Status, $Transferred = '0 B', [long]$Objects = 0, $Log = @(), + $MainSessionId = [guid]::Empty, [bool]$WillRetry = $false) { + $logItems = @((New-LogItem 'Success' "Job started at: $($Start.ToString('G'))" $Start)) + @($Log) [pscustomobject]@{ - Id = New-StableGuid "$($Job.Name)|$($Start.Ticks)"; JobId = $Job.Id; JobName = $Job.Name - CreationTime = $Start; EndTime = $(if ($End) { $End } else { [datetime]::MinValue }) - Status = $Status - Statistics = [pscustomobject]@{ ProcessedObjects = $Objects; TransferredData = $Transferred } - Log = $Log + JobId = $Job.Id; JobName = $Job.Name; Status = $Status + Statistics = [pscustomobject]@{ ProcessingRate = '1.6 MB/s (4 items/s)'; TransferredData = $Transferred; ProcessedObjects = $Objects; Bottleneck = 'Target' } + JobSessionConfigType = 'Incremental' + Id = New-StableGuid "$($Job.Name)|$($Start.Ticks)" + CreationTime = $Start + EndTime = $(if ($End) { $End } else { [datetime]::MinValue }) + Progress = 49; MainSessionId = $MainSessionId; JobWillBeRetried = $WillRetry + Log = $logItems } } -function New-LogEntry($Title, $Status = 'Success') { [pscustomobject]@{ Title = $Title; Status = $Status } } - $global:MockVboJobs = @( - (New-MockJob 'Acme - Exchange & OneDrive' $acme $now.AddHours(3)), - (New-MockJob 'Acme - SharePoint & Teams' $acme $now.AddHours(5)), - (New-MockJob 'Bluewater - Full Org' $blue $now.AddMinutes(30)), - (New-MockJob 'Bluewater - Old Mailboxes' $blue $now.AddDays(1) $false) # disabled + (New-MockJob 'Acme 365 Backup' $acme $now.Date.AddHours(17).AddDays($(if ($now.Hour -ge 17) { 1 } else { 0 }))), + (New-MockJob 'Bluewater 365 Backup' $blue $now.Date.AddHours(17.75).AddDays($(if ($now.Hour -ge 17) { 1 } else { 0 }))), + (New-MockJob 'Bluewater Old Mailboxes' $blue $now.AddDays(1) $false) # disabled ) -$exo, $spo, $bw, $old = $global:MockVboJobs +$acmeJob, $blueJob, $oldJob = $global:MockVboJobs + +$y = $now.Date.AddDays(-1) # yesterday +$blueWarn = New-MockSession $blueJob $y.AddHours(17.75) $y.AddHours(17.75).AddMinutes(3.5) 'Warning' '32.9 MB' 47 @( + (New-LogItem 'Success' 'Found 94 objects' $y.AddHours(17.75)), + # A success line mentioning "error" must not end up in the message + (New-LogItem 'Success' 'Processing site: https://bluewater.sharepoint.com/sites/error-reports' $y.AddHours(17.76)), + (New-LogItem 'Warning' 'Processing mailbox reception@bluewater.ca finished with warning: Item "Q3.pst" exceeds the maximum size' $y.AddHours(17.78)) +) -WillRetry $true $global:MockVboSessions = @( - (New-MockSession $exo $now.AddHours(-21) $now.AddHours(-20.5) 'Success' '1.5 GB' 412 @( - (New-LogEntry 'Processing mailbox jsmith@acme.com'), (New-LogEntry 'Job finished'))), - (New-MockSession $exo $now.AddHours(-1) $null 'Running' '0 B' 10), - (New-MockSession $spo $now.AddHours(-19) $now.AddHours(-18) 'Warning' '512 MB' 38 @( - (New-LogEntry 'Processing site Marketing'), - (New-LogEntry 'Item "Budget 2026.xlsx" was skipped: file is locked' 'Warning'))), - (New-MockSession $bw $now.AddHours(-6) $now.AddHours(-5.9) 'Failed' '0 B' 0 @( - (New-LogEntry 'Failed to connect to Exchange Online: (401) Unauthorized' 'Failed'))), - (New-MockSession $bw $now.AddHours(-30) $now.AddHours(-29.5) 'Stopped' '2 GB' 120), # cancelled - (New-MockSession $old $now.AddDays(-5) $now.AddDays(-5).AddHours(1) 'Success') # outside 48h lookback -) - -# A backup copy job (to object storage) for Acme's mailbox job -$global:MockVboCopyJobs = @( - [pscustomobject]@{ Id = New-StableGuid 'acme-copy'; Name = 'Acme - Exchange copy to Wasabi'; BackupJob = $exo; IsEnabled = $true; NextRun = $null } -) -$global:MockVboCopySessions = @( - [pscustomobject]@{ - Id = New-StableGuid 'acme-copy-s1'; JobId = $global:MockVboCopyJobs[0].Id; CreationTime = $now.AddHours(-20); EndTime = $now.AddHours(-19.8) - Status = 'Success'; Statistics = [pscustomobject]@{ ProcessedObjects = 412; TransferredData = '1.4 GB' }; Log = @() - } + (New-MockSession $acmeJob $y.AddHours(17) $y.AddHours(17).AddMinutes(4.8) 'Success' '187.2 MB' 568 @( + (New-LogItem 'Success' 'Found 49 objects' $y.AddHours(17)), (New-LogItem 'Success' 'Connected to organization' $y.AddHours(17)))), + (New-MockSession $acmeJob $now.AddMinutes(-20) $null 'Running' '0 B' 12), + $blueWarn, + # Its retry 13 minutes later, which succeeded + (New-MockSession $blueJob $y.AddHours(17.75).AddMinutes(13.5) $y.AddHours(17.75).AddMinutes(16.8) 'Success' '0 B' 0 -MainSessionId $blueWarn.Id), + (New-MockSession $blueJob $now.AddHours(-30) $now.AddHours(-29.9) 'Failed' '0 B' 0 @( + (New-LogItem 'Failed' 'Failed to connect to organization: The remote server returned an error: (401) Unauthorized.' $now.AddHours(-30)))), + (New-MockSession $oldJob $now.AddDays(-5) $now.AddDays(-5).AddHours(1) 'Success') # outside 48h lookback ) function Get-VBOJob { $global:MockVboJobs } function Get-VBOJobSession { param($Job) $global:MockVboSessions | Where-Object { -not $Job -or $_.JobId -eq $Job.Id } } function Get-VBOOrganization { $acme, $blue } -function Get-VBOCopyJob { $global:MockVboCopyJobs } -function Get-VBOCopyJobSession { $global:MockVboCopySessions } +function Get-VBOCopyJob { } # none, as on the real server +function Get-VBOCopyJobSession { }