diff --git a/collectors/veeam/Get-VeeamCollectorDiagnostics.ps1 b/collectors/veeam/Get-VeeamCollectorDiagnostics.ps1 new file mode 100644 index 0000000..199a834 --- /dev/null +++ b/collectors/veeam/Get-VeeamCollectorDiagnostics.ps1 @@ -0,0 +1,154 @@ +<# +.SYNOPSIS + Read-only report of how Veeam jobs and sessions relate, for troubleshooting the collector. + +.DESCRIPTION + The collector matches sessions to jobs by Session.JobId = Job.Id. When a job shows as + "Unknown" on the dashboard, no session matched. This lists: + 1. every job from Get-VBRJob and how many sessions matched it + 2. sessions whose JobId matches no job (the runs that are going missing) + 3. for jobs with no matching sessions, their type/link-related properties and methods + 4. what other job cmdlets (e.g. Get-VBRBackupCopyJob) return + Nothing is changed in Veeam or the dashboard. Output is also saved to a text file. + +.EXAMPLE + .\Get-VeeamCollectorDiagnostics.ps1 +.EXAMPLE + .\Get-VeeamCollectorDiagnostics.ps1 -LookbackHours 168 +#> +[CmdletBinding()] +param( + [int]$LookbackHours = 48, + [string]$OutFile = (Join-Path $env:TEMP 'veeam-collector-diagnostics.txt') +) + +$ErrorActionPreference = 'Stop' +$LinkPattern = 'Parent|Linked|Worker|Child|Policy|Source|Target|Type|Cloud|Mode|Session' + +if (-not (Get-Command Get-VBRJob -ErrorAction SilentlyContinue)) { + if (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell) { + Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + } else { + Add-PSSnapin VeeamPSSnapin + } +} + +function Format-Value($Value) { + if ($null -eq $Value) { return '' } + $text = ($Value | ForEach-Object { [string]$_ }) -join ', ' + if ($text.Length -gt 120) { $text = $text.Substring(0, 117) + '...' } + return $text +} + +# Property and method names matching $LinkPattern, with property values. Every read is guarded: +# some Veeam properties throw when accessed. +function Show-LinkMembers($Object) { + foreach ($p in $Object.PSObject.Properties | Where-Object { $_.Name -match $LinkPattern }) { + $value = try { Format-Value $p.Value } catch { "" } + ' {0} = {1}' -f $p.Name, $value + } + $methods = @($Object | Get-Member -MemberType Methods -ErrorAction SilentlyContinue | + Where-Object { $_.Name -match $LinkPattern } | ForEach-Object Name) + if ($methods) { ' methods: ' + ($methods -join ', ') } +} + +function Get-Prop($Object, [string]$Name) { + try { Format-Value $Object.$Name } catch { '' } +} + +$report = & { + $vbrVersion = try { + (Get-Item (Join-Path ${env:ProgramFiles} 'Veeam\Backup and Replication\Backup\Veeam.Backup.Core.dll')).VersionInfo.ProductVersion + } catch { 'unknown' } + "Generated $(Get-Date -Format o) on $env:COMPUTERNAME, VBR $vbrVersion, PowerShell $($PSVersionTable.PSVersion)" + "Lookback: $LookbackHours hours" + '' + + $since = (Get-Date).AddHours(-$LookbackHours) + $jobs = @(Get-VBRJob -WarningAction SilentlyContinue) + $sessions = @(Get-VBRBackupSession -WarningAction SilentlyContinue | Where-Object { $_.CreationTime -ge $since }) + $jobIds = @{} + foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $j } + + '== 1. Jobs from Get-VBRJob' + $jobs | ForEach-Object { + $id = $_.Id.ToString() + [pscustomobject]@{ + Name = $_.Name + JobType = Get-Prop $_ 'JobType' + TypeToString = Get-Prop $_ 'TypeToString' + Id = $id + MatchingSessions = @($sessions | Where-Object { $_.JobId.ToString() -eq $id }).Count + } + } | Format-Table -AutoSize | Out-String -Width 300 + + '== 2. Sessions whose JobId matches no job from Get-VBRJob' + $orphans = @($sessions | Where-Object { -not $jobIds.ContainsKey($_.JobId.ToString()) }) + if ($orphans.Count -eq 0) { + ' (none)' + } else { + $orphans | Group-Object { $_.JobId.ToString() } | ForEach-Object { + $latest = $_.Group | Sort-Object CreationTime | Select-Object -Last 1 + [pscustomobject]@{ + JobId = $_.Name + JobName = Get-Prop $latest 'JobName' + JobType = Get-Prop $latest 'JobType' + Sessions = $_.Count + LatestStart = $latest.CreationTime + State = Get-Prop $latest 'State' + Result = Get-Prop $latest 'Result' + } + } | Format-Table -AutoSize | Out-String -Width 300 + + ' Link-related members of one orphan session:' + Show-LinkMembers $orphans[0] + '' + ' Job object for that session (via GetJob(), if available):' + try { + $owner = $orphans[0].GetJob() + ' Name = {0}, Id = {1}' -f (Get-Prop $owner 'Name'), (Get-Prop $owner 'Id') + Show-LinkMembers $owner + } catch { + " not available: $($_.Exception.Message)" + } + } + '' + + '== 3. Jobs with no matching sessions in the window' + $empty = @($jobs | Where-Object { $id = $_.Id.ToString(); -not ($sessions | Where-Object { $_.JobId.ToString() -eq $id }) }) + if ($empty.Count -eq 0) { ' (none)' } + foreach ($j in $empty) { + " $($j.Name) [$($j.Id)]" + Show-LinkMembers $j + try { + $last = $j.FindLastSession() + if ($last) { + ' FindLastSession(): JobId = {0}, JobName = {1}, Start = {2}, Result = {3}' -f ` + (Get-Prop $last 'JobId'), (Get-Prop $last 'JobName'), $last.CreationTime, (Get-Prop $last 'Result') + } else { + ' FindLastSession(): ' + } + } catch { + " FindLastSession(): not available: $($_.Exception.Message)" + } + '' + } + + '== 4. Other job cmdlets' + foreach ($cmd in 'Get-VBRBackupCopyJob', 'Get-VBRComputerBackupCopyJob', 'Get-VBRComputerBackupJob') { + if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) { " ${cmd}: not available"; continue } + try { + $found = @(& $cmd -WarningAction SilentlyContinue) + " ${cmd}: $($found.Count) job(s)" + foreach ($f in $found) { + ' {0} [{1}] Type = {2}, Mode = {3}' -f (Get-Prop $f 'Name'), (Get-Prop $f 'Id'), (Get-Prop $f 'Type'), (Get-Prop $f 'Mode') + } + } catch { + " ${cmd}: error: $($_.Exception.Message)" + } + } +} | Out-String -Width 300 + +$report +$report | Set-Content -Path $OutFile -Encoding UTF8 +Write-Host "Saved to $OutFile" diff --git a/collectors/veeam/README.md b/collectors/veeam/README.md index da63567..896bb47 100644 --- a/collectors/veeam/README.md +++ b/collectors/veeam/README.md @@ -38,6 +38,10 @@ delete the scheduled task and both `BackupDashboard` folders. | Message | Session reason, each VM or object that didn't succeed, and warning/error lines from the session log | | Size | `BackupStats.BackupSize` | +**Backup copy jobs (VBR 12+):** Veeam records their sessions under hidden per-source "worker" jobs. +The collector files each worker session under its copy job (via the worker's `PolicyJobId`), so a +copy job gets one run per source job it copies, with the source in `copied_from`. + Each run covers the last 48 hours (`LookbackHours`), so a few missed runs (server rebooting, API down) fill in automatically. Sessions already delivered are recorded in `veeam-collector.state.json`. Running sessions are re-sent until they finish. @@ -48,6 +52,9 @@ down) fill in automatically. Sessions already delivered are recorded in - Exit codes: `0` OK, `1` some jobs failed to send (they're retried next run), `2` setup failed (config, Veeam module, or Veeam connection). - `API key rejected`: the key was revoked or mistyped. Re-run the installer with a new key. +- A job shows **Unknown** on the dashboard: the collector sent the job but matched no sessions to it. + Run `.\Get-VeeamCollectorDiagnostics.ps1` (read-only). It lists sessions whose job ID matches no + job, plus the type and link properties of the jobs left without sessions. - `Get-VBRBackupSession` loads every session in the VBR database. On a server with years of history, this can make each run take a minute or more. The task's 30-minute limit and "don't start if already running" setting keep that safe. diff --git a/collectors/veeam/Send-VeeamBackupStatus.ps1 b/collectors/veeam/Send-VeeamBackupStatus.ps1 index f377b24..79ff658 100644 --- a/collectors/veeam/Send-VeeamBackupStatus.ps1 +++ b/collectors/veeam/Send-VeeamBackupStatus.ps1 @@ -154,6 +154,17 @@ function Get-RunMessage($Session, [string]$Status, $Tasks) { return ($lines -join "`n") } +# VBR 12+ backup copy jobs: Get-VBRJob returns the copy (policy) job, but Get-VBRBackupSession +# returns only the sessions of its hidden worker jobs, one per source job, never the policy's own. +# Each worker job records its policy in PolicyJobId, so those sessions can be filed under it. +function Get-PolicyJobId($Session, [hashtable]$KnownJobIds) { + try { + $policyId = [string]$Session.GetJob().PolicyJobId + if ($policyId -and $KnownJobIds.ContainsKey($policyId)) { return $policyId } + } catch { } + return '' +} + function ConvertTo-Run($Session, $Job) { $status = Get-RunStatus $Session $tasks = @() @@ -169,6 +180,11 @@ function ConvertTo-Run($Session, $Job) { if ($failedTasks) { $details.objects_failed = $failedTasks } if ($warningTasks) { $details.objects_warning = $warningTasks } try { if ($Session.IsRetryMode) { $details.retry = $true } } catch { } + # Worker session of a backup copy job: say which source job it copied + # (worker sessions are named "\") + if ($Session.JobId.ToString() -ne $Job.Id.ToString()) { + $details.copied_from = ([string]$Session.JobName -split '\\')[-1] + } $run = [ordered]@{ external_id = $Session.Id.ToString() @@ -238,11 +254,20 @@ try { $since = (Get-Date).AddHours(-$LookbackHours) $jobs = @(Get-VBRJob -WarningAction SilentlyContinue) + $jobIds = @{} + foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $true } + # Get-VBRBackupSession returns every session in the database, so fetch once and bucket by job $sessionsByJob = @{} + $policyByWorker = @{} # worker job ID -> policy job ID ('' if it isn't a worker of a listed job) foreach ($s in Get-VBRBackupSession -WarningAction SilentlyContinue) { if ($s.CreationTime -lt $since) { continue } $key = $s.JobId.ToString() + if (-not $jobIds.ContainsKey($key)) { + if (-not $policyByWorker.ContainsKey($key)) { $policyByWorker[$key] = Get-PolicyJobId $s $jobIds } + if (-not $policyByWorker[$key]) { continue } + $key = $policyByWorker[$key] + } if (-not $sessionsByJob.ContainsKey($key)) { $sessionsByJob[$key] = New-Object 'System.Collections.Generic.List[object]' } $sessionsByJob[$key].Add($s) } diff --git a/collectors/veeam/tests/MockVeeam.ps1 b/collectors/veeam/tests/MockVeeam.ps1 index 28ad26e..230785c 100644 --- a/collectors/veeam/tests/MockVeeam.ps1 +++ b/collectors/veeam/tests/MockVeeam.ps1 @@ -44,9 +44,23 @@ $global:MockJobs = @( # Older VBR versions return NextRun as a culture-formatted string (New-MockJob 'SQL01 – Logs' 'Backup' $now.AddMinutes(40).ToString()), (New-MockJob 'Offsite Copy' 'BackupSync' $null), - (New-MockJob 'Old Exchange' 'Backup' $now.AddDays(1) $false) + (New-MockJob 'Old Exchange' 'Backup' $now.AddDays(1) $false), + # VBR 12+ backup copy job (immediate mode, no schedule): the "policy" job Get-VBRJob returns + (New-MockJob 'Cloud Copy' 'SimpleBackupCopyPolicy' $null) ) -$fs, $sql, $copy, $old = $global:MockJobs +$fs, $sql, $copy, $old, $cloud = $global:MockJobs + +# Its hidden worker job, one per source job. Get-VBRJob doesn't return it, and Get-VBRBackupSession +# returns only the worker's sessions. Shape taken from a real VBR 13 server. +$cloudWorker = [pscustomobject]@{ + Id = New-StableGuid 'Cloud Copy\File Server – Nightly'; Name = 'Cloud Copy\File Server – Nightly' + JobType = 'SimpleBackupCopyWorker'; PolicyJobId = $cloud.Id +} +function New-MockWorkerSession($Start, $End, $Result) { + $s = New-MockSession $cloudWorker $Start $End 'Stopped' $Result -Size 40GB + $s | Add-Member ScriptMethod GetJob { $cloudWorker } + $s +} $global:MockSessions = @( (New-MockSession $fs $now.AddDays(-1).AddHours(-2) $now.AddDays(-1).AddHours(-1) 'Stopped' 'Success' -Size 128GB -Tasks @( @@ -58,7 +72,11 @@ $global:MockSessions = @( -LogRecords @( [pscustomobject]@{ Title = 'Preparing backup chain'; Status = 'ESucceeded' }, [pscustomobject]@{ Title = 'Network path not found (0x80070035)'; Status = 'EFailed' })), - (New-MockSession $old $now.AddDays(-10) $now.AddDays(-10).AddHours(1) 'Stopped' 'Success') # outside 48h lookback + (New-MockSession $old $now.AddDays(-10) $now.AddDays(-10).AddHours(1) 'Stopped' 'Success'), # outside 48h lookback + (New-MockWorkerSession $now.AddDays(-1).AddMinutes(-55) $now.AddDays(-1).AddMinutes(-30) 'Success'), + (New-MockWorkerSession $now.AddMinutes(-55) $now.AddMinutes(-35) 'Warning'), + # A session from a job the collector doesn't know (e.g. deleted): must be ignored, not crash + (New-MockSession ([pscustomobject]@{ Id = New-StableGuid 'gone'; Name = 'Deleted job' }) $now.AddHours(-2) $now.AddHours(-1) 'Stopped' 'Success') ) function Get-VBRJob { [CmdletBinding()] param() $global:MockJobs }