Report Veeam backup copy job sessions

VBR 12+ backup copy jobs showed as Unknown. Get-VBRJob returns the copy
(policy) job, but Get-VBRBackupSession only returns sessions of its
hidden worker jobs, whose JobId matches no listed job, so they were
dropped. Confirmed on a VBR 13.0.2 server with the new diagnostics
script.

- File worker sessions under their policy job via
  Session.GetJob().PolicyJobId (looked up once per worker job), and
  record the source job in details.copied_from
- Add Get-VeeamCollectorDiagnostics.ps1, a read-only report of jobs,
  unmatched sessions and their link properties
- Mock: backup copy policy/worker case and a session from a deleted job

Co-Authored-By: Claude Opus 5.5 <[email protected]>

collectors/veeam/Get-VeeamCollectorDiagnostics.ps1

@@ -0,0 +1,154 @@
+<#
+.SYNOPSIS
+    Read-only report of how Veeam jobs and sessions relate, for troubleshooting the collector.
+
+.DESCRIPTION
+    The collector matches sessions to jobs by Session.JobId = Job.Id. When a job shows as
+    "Unknown" on the dashboard, no session matched. This lists:
+      1. every job from Get-VBRJob and how many sessions matched it
+      2. sessions whose JobId matches no job (the runs that are going missing)
+      3. for jobs with no matching sessions, their type/link-related properties and methods
+      4. what other job cmdlets (e.g. Get-VBRBackupCopyJob) return
+    Nothing is changed in Veeam or the dashboard. Output is also saved to a text file.
+
+.EXAMPLE
+    .\Get-VeeamCollectorDiagnostics.ps1
+.EXAMPLE
+    .\Get-VeeamCollectorDiagnostics.ps1 -LookbackHours 168
+#>
+[CmdletBinding()]
+param(
+    [int]$LookbackHours = 48,
+    [string]$OutFile = (Join-Path $env:TEMP 'veeam-collector-diagnostics.txt')
+)
+
+$ErrorActionPreference = 'Stop'
+$LinkPattern = 'Parent|Linked|Worker|Child|Policy|Source|Target|Type|Cloud|Mode|Session'
+
+if (-not (Get-Command Get-VBRJob -ErrorAction SilentlyContinue)) {
+    if (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell) {
+        Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue
+    } else {
+        Add-PSSnapin VeeamPSSnapin
+    }
+}
+
+function Format-Value($Value) {
+    if ($null -eq $Value) { return '<null>' }
+    $text = ($Value | ForEach-Object { [string]$_ }) -join ', '
+    if ($text.Length -gt 120) { $text = $text.Substring(0, 117) + '...' }
+    return $text
+}
+
+# Property and method names matching $LinkPattern, with property values. Every read is guarded:
+# some Veeam properties throw when accessed.
+function Show-LinkMembers($Object) {
+    foreach ($p in $Object.PSObject.Properties | Where-Object { $_.Name -match $LinkPattern }) {
+        $value = try { Format-Value $p.Value } catch { "<error: $($_.Exception.Message)>" }
+        '    {0} = {1}' -f $p.Name, $value
+    }
+    $methods = @($Object | Get-Member -MemberType Methods -ErrorAction SilentlyContinue |
+        Where-Object { $_.Name -match $LinkPattern } | ForEach-Object Name)
+    if ($methods) { '    methods: ' + ($methods -join ', ') }
+}
+
+function Get-Prop($Object, [string]$Name) {
+    try { Format-Value $Object.$Name } catch { '<error>' }
+}
+
+$report = & {
+    $vbrVersion = try {
+        (Get-Item (Join-Path ${env:ProgramFiles} 'Veeam\Backup and Replication\Backup\Veeam.Backup.Core.dll')).VersionInfo.ProductVersion
+    } catch { 'unknown' }
+    "Generated $(Get-Date -Format o) on $env:COMPUTERNAME, VBR $vbrVersion, PowerShell $($PSVersionTable.PSVersion)"
+    "Lookback: $LookbackHours hours"
+    ''
+
+    $since = (Get-Date).AddHours(-$LookbackHours)
+    $jobs = @(Get-VBRJob -WarningAction SilentlyContinue)
+    $sessions = @(Get-VBRBackupSession -WarningAction SilentlyContinue | Where-Object { $_.CreationTime -ge $since })
+    $jobIds = @{}
+    foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $j }
+
+    '== 1. Jobs from Get-VBRJob'
+    $jobs | ForEach-Object {
+        $id = $_.Id.ToString()
+        [pscustomobject]@{
+            Name             = $_.Name
+            JobType          = Get-Prop $_ 'JobType'
+            TypeToString     = Get-Prop $_ 'TypeToString'
+            Id               = $id
+            MatchingSessions = @($sessions | Where-Object { $_.JobId.ToString() -eq $id }).Count
+        }
+    } | Format-Table -AutoSize | Out-String -Width 300
+
+    '== 2. Sessions whose JobId matches no job from Get-VBRJob'
+    $orphans = @($sessions | Where-Object { -not $jobIds.ContainsKey($_.JobId.ToString()) })
+    if ($orphans.Count -eq 0) {
+        '  (none)'
+    } else {
+        $orphans | Group-Object { $_.JobId.ToString() } | ForEach-Object {
+            $latest = $_.Group | Sort-Object CreationTime | Select-Object -Last 1
+            [pscustomobject]@{
+                JobId       = $_.Name
+                JobName     = Get-Prop $latest 'JobName'
+                JobType     = Get-Prop $latest 'JobType'
+                Sessions    = $_.Count
+                LatestStart = $latest.CreationTime
+                State       = Get-Prop $latest 'State'
+                Result      = Get-Prop $latest 'Result'
+            }
+        } | Format-Table -AutoSize | Out-String -Width 300
+
+        '  Link-related members of one orphan session:'
+        Show-LinkMembers $orphans[0]
+        ''
+        '  Job object for that session (via GetJob(), if available):'
+        try {
+            $owner = $orphans[0].GetJob()
+            '    Name = {0}, Id = {1}' -f (Get-Prop $owner 'Name'), (Get-Prop $owner 'Id')
+            Show-LinkMembers $owner
+        } catch {
+            "    not available: $($_.Exception.Message)"
+        }
+    }
+    ''
+
+    '== 3. Jobs with no matching sessions in the window'
+    $empty = @($jobs | Where-Object { $id = $_.Id.ToString(); -not ($sessions | Where-Object { $_.JobId.ToString() -eq $id }) })
+    if ($empty.Count -eq 0) { '  (none)' }
+    foreach ($j in $empty) {
+        "  $($j.Name)  [$($j.Id)]"
+        Show-LinkMembers $j
+        try {
+            $last = $j.FindLastSession()
+            if ($last) {
+                '    FindLastSession(): JobId = {0}, JobName = {1}, Start = {2}, Result = {3}' -f `
+                    (Get-Prop $last 'JobId'), (Get-Prop $last 'JobName'), $last.CreationTime, (Get-Prop $last 'Result')
+            } else {
+                '    FindLastSession(): <null>'
+            }
+        } catch {
+            "    FindLastSession(): not available: $($_.Exception.Message)"
+        }
+        ''
+    }
+
+    '== 4. Other job cmdlets'
+    foreach ($cmd in 'Get-VBRBackupCopyJob', 'Get-VBRComputerBackupCopyJob', 'Get-VBRComputerBackupJob') {
+        if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) { "  ${cmd}: not available"; continue }
+        try {
+            $found = @(& $cmd -WarningAction SilentlyContinue)
+            "  ${cmd}: $($found.Count) job(s)"
+            foreach ($f in $found) {
+                '    {0}  [{1}]  Type = {2}, Mode = {3}' -f (Get-Prop $f 'Name'), (Get-Prop $f 'Id'), (Get-Prop $f 'Type'), (Get-Prop $f 'Mode')
+            }
+        } catch {
+            "  ${cmd}: error: $($_.Exception.Message)"
+        }
+    }
+} | Out-String -Width 300
+
+$report
+$report | Set-Content -Path $OutFile -Encoding UTF8
+Write-Host "Saved to $OutFile"

collectors/veeam/README.md

@@ -38,6 +38,10 @@ delete the scheduled task and both `BackupDashboard` folders.
 | Message | Session reason, each VM or object that didn't succeed, and warning/error lines from the session log |
 | Size | `BackupStats.BackupSize` |
 
+**Backup copy jobs (VBR 12+):** Veeam records their sessions under hidden per-source "worker" jobs.
+The collector files each worker session under its copy job (via the worker's `PolicyJobId`), so a
+copy job gets one run per source job it copies, with the source in `copied_from`.
+
 Each run covers the last 48 hours (`LookbackHours`), so a few missed runs (server rebooting, API
 down) fill in automatically. Sessions already delivered are recorded in
 `veeam-collector.state.json`. Running sessions are re-sent until they finish.
@@ -48,6 +52,9 @@ down) fill in automatically. Sessions already delivered are recorded in
 - Exit codes: `0` OK, `1` some jobs failed to send (they're retried next run), `2` setup failed
   (config, Veeam module, or Veeam connection).
 - `API key rejected`: the key was revoked or mistyped. Re-run the installer with a new key.
+- A job shows **Unknown** on the dashboard: the collector sent the job but matched no sessions to it.
+  Run `.\Get-VeeamCollectorDiagnostics.ps1` (read-only). It lists sessions whose job ID matches no
+  job, plus the type and link properties of the jobs left without sessions.
 - `Get-VBRBackupSession` loads every session in the VBR database. On a server with years of history,
   this can make each run take a minute or more. The task's 30-minute limit and "don't start if
   already running" setting keep that safe.

collectors/veeam/Send-VeeamBackupStatus.ps1

@@ -154,6 +154,17 @@ function Get-RunMessage($Session, [string]$Status, $Tasks) {
     return ($lines -join "`n")
 }
 
+# VBR 12+ backup copy jobs: Get-VBRJob returns the copy (policy) job, but Get-VBRBackupSession
+# returns only the sessions of its hidden worker jobs, one per source job, never the policy's own.
+# Each worker job records its policy in PolicyJobId, so those sessions can be filed under it.
+function Get-PolicyJobId($Session, [hashtable]$KnownJobIds) {
+    try {
+        $policyId = [string]$Session.GetJob().PolicyJobId
+        if ($policyId -and $KnownJobIds.ContainsKey($policyId)) { return $policyId }
+    } catch { }
+    return ''
+}
+
 function ConvertTo-Run($Session, $Job) {
     $status = Get-RunStatus $Session
     $tasks = @()
@@ -169,6 +180,11 @@ function ConvertTo-Run($Session, $Job) {
     if ($failedTasks) { $details.objects_failed = $failedTasks }
     if ($warningTasks) { $details.objects_warning = $warningTasks }
     try { if ($Session.IsRetryMode) { $details.retry = $true } } catch { }
+    # Worker session of a backup copy job: say which source job it copied
+    # (worker sessions are named "<copy job>\<source job>")
+    if ($Session.JobId.ToString() -ne $Job.Id.ToString()) {
+        $details.copied_from = ([string]$Session.JobName -split '\\')[-1]
+    }
 
     $run = [ordered]@{
         external_id = $Session.Id.ToString()
@@ -238,11 +254,20 @@ try {
     $since = (Get-Date).AddHours(-$LookbackHours)
     $jobs = @(Get-VBRJob -WarningAction SilentlyContinue)
 
+    $jobIds = @{}
+    foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $true }
+
     # Get-VBRBackupSession returns every session in the database, so fetch once and bucket by job
     $sessionsByJob = @{}
+    $policyByWorker = @{}  # worker job ID -> policy job ID ('' if it isn't a worker of a listed job)
     foreach ($s in Get-VBRBackupSession -WarningAction SilentlyContinue) {
         if ($s.CreationTime -lt $since) { continue }
         $key = $s.JobId.ToString()
+        if (-not $jobIds.ContainsKey($key)) {
+            if (-not $policyByWorker.ContainsKey($key)) { $policyByWorker[$key] = Get-PolicyJobId $s $jobIds }
+            if (-not $policyByWorker[$key]) { continue }
+            $key = $policyByWorker[$key]
+        }
         if (-not $sessionsByJob.ContainsKey($key)) { $sessionsByJob[$key] = New-Object 'System.Collections.Generic.List[object]' }
         $sessionsByJob[$key].Add($s)
     }

collectors/veeam/tests/MockVeeam.ps1

@@ -44,9 +44,23 @@ $global:MockJobs = @(
     # Older VBR versions return NextRun as a culture-formatted string
     (New-MockJob 'SQL01 – Logs' 'Backup' $now.AddMinutes(40).ToString()),
     (New-MockJob 'Offsite Copy' 'BackupSync' $null),
-    (New-MockJob 'Old Exchange' 'Backup' $now.AddDays(1) $false)
+    (New-MockJob 'Old Exchange' 'Backup' $now.AddDays(1) $false),
+    # VBR 12+ backup copy job (immediate mode, no schedule): the "policy" job Get-VBRJob returns
+    (New-MockJob 'Cloud Copy' 'SimpleBackupCopyPolicy' $null)
 )
-$fs, $sql, $copy, $old = $global:MockJobs
+$fs, $sql, $copy, $old, $cloud = $global:MockJobs
+
+# Its hidden worker job, one per source job. Get-VBRJob doesn't return it, and Get-VBRBackupSession
+# returns only the worker's sessions. Shape taken from a real VBR 13 server.
+$cloudWorker = [pscustomobject]@{
+    Id = New-StableGuid 'Cloud Copy\File Server – Nightly'; Name = 'Cloud Copy\File Server – Nightly'
+    JobType = 'SimpleBackupCopyWorker'; PolicyJobId = $cloud.Id
+}
+function New-MockWorkerSession($Start, $End, $Result) {
+    $s = New-MockSession $cloudWorker $Start $End 'Stopped' $Result -Size 40GB
+    $s | Add-Member ScriptMethod GetJob { $cloudWorker }
+    $s
+}
 
 $global:MockSessions = @(
     (New-MockSession $fs $now.AddDays(-1).AddHours(-2) $now.AddDays(-1).AddHours(-1) 'Stopped' 'Success' -Size 128GB -Tasks @(
@@ -58,7 +72,11 @@ $global:MockSessions = @(
         -LogRecords @(
             [pscustomobject]@{ Title = 'Preparing backup chain'; Status = 'ESucceeded' },
             [pscustomobject]@{ Title = 'Network path not found (0x80070035)'; Status = 'EFailed' })),
-    (New-MockSession $old $now.AddDays(-10) $now.AddDays(-10).AddHours(1) 'Stopped' 'Success')  # outside 48h lookback
+    (New-MockSession $old $now.AddDays(-10) $now.AddDays(-10).AddHours(1) 'Stopped' 'Success'),  # outside 48h lookback
+    (New-MockWorkerSession $now.AddDays(-1).AddMinutes(-55) $now.AddDays(-1).AddMinutes(-30) 'Success'),
+    (New-MockWorkerSession $now.AddMinutes(-55) $now.AddMinutes(-35) 'Warning'),
+    # A session from a job the collector doesn't know (e.g. deleted): must be ignored, not crash
+    (New-MockSession ([pscustomobject]@{ Id = New-StableGuid 'gone'; Name = 'Deleted job' }) $now.AddHours(-2) $now.AddHours(-1) 'Stopped' 'Success')
 )
 
 function Get-VBRJob { [CmdletBinding()] param() $global:MockJobs }