diff --git a/Get-VeeamCollectorDiagnostics.ps1 b/Get-VeeamCollectorDiagnostics.ps1 new file mode 100644 index 0000000..199a834 --- /dev/null +++ b/Get-VeeamCollectorDiagnostics.ps1 @@ -0,0 +1,154 @@ +<# +.SYNOPSIS + Read-only report of how Veeam jobs and sessions relate, for troubleshooting the collector. + +.DESCRIPTION + The collector matches sessions to jobs by Session.JobId = Job.Id. When a job shows as + "Unknown" on the dashboard, no session matched. This lists: + 1. every job from Get-VBRJob and how many sessions matched it + 2. sessions whose JobId matches no job (the runs that are going missing) + 3. for jobs with no matching sessions, their type/link-related properties and methods + 4. what other job cmdlets (e.g. Get-VBRBackupCopyJob) return + Nothing is changed in Veeam or the dashboard. Output is also saved to a text file. + +.EXAMPLE + .\Get-VeeamCollectorDiagnostics.ps1 +.EXAMPLE + .\Get-VeeamCollectorDiagnostics.ps1 -LookbackHours 168 +#> +[CmdletBinding()] +param( + [int]$LookbackHours = 48, + [string]$OutFile = (Join-Path $env:TEMP 'veeam-collector-diagnostics.txt') +) + +$ErrorActionPreference = 'Stop' +$LinkPattern = 'Parent|Linked|Worker|Child|Policy|Source|Target|Type|Cloud|Mode|Session' + +if (-not (Get-Command Get-VBRJob -ErrorAction SilentlyContinue)) { + if (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell) { + Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue + } else { + Add-PSSnapin VeeamPSSnapin + } +} + +function Format-Value($Value) { + if ($null -eq $Value) { return '' } + $text = ($Value | ForEach-Object { [string]$_ }) -join ', ' + if ($text.Length -gt 120) { $text = $text.Substring(0, 117) + '...' } + return $text +} + +# Property and method names matching $LinkPattern, with property values. Every read is guarded: +# some Veeam properties throw when accessed. +function Show-LinkMembers($Object) { + foreach ($p in $Object.PSObject.Properties | Where-Object { $_.Name -match $LinkPattern }) { + $value = try { Format-Value $p.Value } catch { "" } + ' {0} = {1}' -f $p.Name, $value + } + $methods = @($Object | Get-Member -MemberType Methods -ErrorAction SilentlyContinue | + Where-Object { $_.Name -match $LinkPattern } | ForEach-Object Name) + if ($methods) { ' methods: ' + ($methods -join ', ') } +} + +function Get-Prop($Object, [string]$Name) { + try { Format-Value $Object.$Name } catch { '' } +} + +$report = & { + $vbrVersion = try { + (Get-Item (Join-Path ${env:ProgramFiles} 'Veeam\Backup and Replication\Backup\Veeam.Backup.Core.dll')).VersionInfo.ProductVersion + } catch { 'unknown' } + "Generated $(Get-Date -Format o) on $env:COMPUTERNAME, VBR $vbrVersion, PowerShell $($PSVersionTable.PSVersion)" + "Lookback: $LookbackHours hours" + '' + + $since = (Get-Date).AddHours(-$LookbackHours) + $jobs = @(Get-VBRJob -WarningAction SilentlyContinue) + $sessions = @(Get-VBRBackupSession -WarningAction SilentlyContinue | Where-Object { $_.CreationTime -ge $since }) + $jobIds = @{} + foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $j } + + '== 1. Jobs from Get-VBRJob' + $jobs | ForEach-Object { + $id = $_.Id.ToString() + [pscustomobject]@{ + Name = $_.Name + JobType = Get-Prop $_ 'JobType' + TypeToString = Get-Prop $_ 'TypeToString' + Id = $id + MatchingSessions = @($sessions | Where-Object { $_.JobId.ToString() -eq $id }).Count + } + } | Format-Table -AutoSize | Out-String -Width 300 + + '== 2. Sessions whose JobId matches no job from Get-VBRJob' + $orphans = @($sessions | Where-Object { -not $jobIds.ContainsKey($_.JobId.ToString()) }) + if ($orphans.Count -eq 0) { + ' (none)' + } else { + $orphans | Group-Object { $_.JobId.ToString() } | ForEach-Object { + $latest = $_.Group | Sort-Object CreationTime | Select-Object -Last 1 + [pscustomobject]@{ + JobId = $_.Name + JobName = Get-Prop $latest 'JobName' + JobType = Get-Prop $latest 'JobType' + Sessions = $_.Count + LatestStart = $latest.CreationTime + State = Get-Prop $latest 'State' + Result = Get-Prop $latest 'Result' + } + } | Format-Table -AutoSize | Out-String -Width 300 + + ' Link-related members of one orphan session:' + Show-LinkMembers $orphans[0] + '' + ' Job object for that session (via GetJob(), if available):' + try { + $owner = $orphans[0].GetJob() + ' Name = {0}, Id = {1}' -f (Get-Prop $owner 'Name'), (Get-Prop $owner 'Id') + Show-LinkMembers $owner + } catch { + " not available: $($_.Exception.Message)" + } + } + '' + + '== 3. Jobs with no matching sessions in the window' + $empty = @($jobs | Where-Object { $id = $_.Id.ToString(); -not ($sessions | Where-Object { $_.JobId.ToString() -eq $id }) }) + if ($empty.Count -eq 0) { ' (none)' } + foreach ($j in $empty) { + " $($j.Name) [$($j.Id)]" + Show-LinkMembers $j + try { + $last = $j.FindLastSession() + if ($last) { + ' FindLastSession(): JobId = {0}, JobName = {1}, Start = {2}, Result = {3}' -f ` + (Get-Prop $last 'JobId'), (Get-Prop $last 'JobName'), $last.CreationTime, (Get-Prop $last 'Result') + } else { + ' FindLastSession(): ' + } + } catch { + " FindLastSession(): not available: $($_.Exception.Message)" + } + '' + } + + '== 4. Other job cmdlets' + foreach ($cmd in 'Get-VBRBackupCopyJob', 'Get-VBRComputerBackupCopyJob', 'Get-VBRComputerBackupJob') { + if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) { " ${cmd}: not available"; continue } + try { + $found = @(& $cmd -WarningAction SilentlyContinue) + " ${cmd}: $($found.Count) job(s)" + foreach ($f in $found) { + ' {0} [{1}] Type = {2}, Mode = {3}' -f (Get-Prop $f 'Name'), (Get-Prop $f 'Id'), (Get-Prop $f 'Type'), (Get-Prop $f 'Mode') + } + } catch { + " ${cmd}: error: $($_.Exception.Message)" + } + } +} | Out-String -Width 300 + +$report +$report | Set-Content -Path $OutFile -Encoding UTF8 +Write-Host "Saved to $OutFile" diff --git a/Install-VeeamCollector.ps1 b/Install-VeeamCollector.ps1 new file mode 100644 index 0000000..74ab304 --- /dev/null +++ b/Install-VeeamCollector.ps1 @@ -0,0 +1,102 @@ +<# +.SYNOPSIS + Installs the Veeam collector on a VBR server as a SYSTEM scheduled task. + +.DESCRIPTION + - Copies Send-VeeamBackupStatus.ps1 to Program Files (only admins can modify it, which + matters because it runs as SYSTEM). + - Writes the config to ProgramData\BackupDashboard and restricts that folder to SYSTEM and + Administrators, since it contains the API key. + - Registers a scheduled task that runs every IntervalMinutes under PowerShell 7 (pwsh.exe), + which the Veeam module on VBR 13 expects. Install PowerShell 7 machine-wide (MSI or winget) + first: Microsoft Store installs can't be run by SYSTEM. + Safe to re-run to change settings or update the script. + +.EXAMPLE + .\Install-VeeamCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" + (prompts for the API key) +#> +#Requires -RunAsAdministrator +[CmdletBinding()] +param( + [Parameter(Mandatory)][string]$ApiUrl, + [Parameter(Mandatory)][string]$Tenant, + [string]$SourceSystem = 'Veeam B&R', + [ValidateRange(5, 1440)][int]$IntervalMinutes = 15, + [ValidateRange(1, 720)][int]$LookbackHours = 48, + [string]$InstallDir = (Join-Path $env:ProgramFiles 'BackupDashboard'), + # Defaults to the machine-wide PowerShell 7 install + [string]$PwshPath +) + +$ErrorActionPreference = 'Stop' +$TaskName = 'Backup Dashboard - Veeam collector' +$DataDir = Join-Path $env:ProgramData 'BackupDashboard' +$ConfigPath = Join-Path $DataDir 'veeam-collector.json' + +# Find PowerShell 7 before changing anything, so a missing install fails cleanly +if (-not $PwshPath) { + $PwshPath = Join-Path $env:ProgramFiles 'PowerShell\7\pwsh.exe' + if (-not (Test-Path $PwshPath)) { + $onPath = Get-Command pwsh.exe -ErrorAction SilentlyContinue + if ($onPath) { $PwshPath = $onPath.Source } + } +} +if (-not (Test-Path $PwshPath)) { + throw "PowerShell 7 not found at '$PwshPath'. Install it machine-wide (e.g. winget install Microsoft.PowerShell) or pass -PwshPath." +} +if ($PwshPath -match '\\WindowsApps\\') { + throw "'$PwshPath' is a Microsoft Store install, which the SYSTEM account can't run. Install the MSI or winget version instead." +} +$pwshVersion = & $PwshPath -NoProfile -NonInteractive -Command '$PSVersionTable.PSVersion.ToString()' +$pwshMajor = & $PwshPath -NoProfile -NonInteractive -Command '$PSVersionTable.PSVersion.Major' +if ($LASTEXITCODE -or ($pwshMajor -as [int]) -lt 7) { + throw "'$PwshPath' isn't a working PowerShell 7 (reported '$pwshVersion')." +} + +if ($ApiUrl -notmatch '^https://') { + Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.' +} + +# Read the key as a SecureString so it isn't echoed to the console or kept in shell history +$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key )' +$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) +try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) } +finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) } +if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." } + +# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows) +New-Item -ItemType Directory -Path $DataDir -Force | Out-Null +& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null +if ($LASTEXITCODE) { throw "icacls failed on $DataDir" } + +[ordered]@{ + ApiUrl = $ApiUrl.TrimEnd('/') + ApiKey = $apiKey + Tenant = $Tenant + SourceSystem = $SourceSystem + LookbackHours = $LookbackHours +} | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8 + +New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null +$scriptPath = Join-Path $InstallDir 'Send-VeeamBackupStatus.ps1' +Copy-Item -Path (Join-Path $PSScriptRoot 'Send-VeeamBackupStatus.ps1') -Destination $scriptPath -Force +Unblock-File -Path $scriptPath + +$action = New-ScheduledTaskAction -Execute $PwshPath ` + -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`"" +$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) +$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` + -MultipleInstances IgnoreNew -StartWhenAvailable +Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal ` + -Settings $settings -Description 'Sends Veeam job results to the Backup Dashboard' -Force | Out-Null + +Write-Host "Installed. Config: $ConfigPath" +Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using PowerShell $pwshVersion ($PwshPath)." +Write-Host '' +Write-Host 'Next steps:' +Write-Host " 1. Preview what will be sent: & '$PwshPath' -File '$scriptPath' -DryRun" +Write-Host " 2. Backfill 30 days of history: & '$PwshPath' -File '$scriptPath' -LookbackHours 720" +Write-Host " 3. Check the log: Get-Content '$DataDir\veeam-collector.log' -Tail 20" diff --git a/MockVeeam.ps1 b/MockVeeam.ps1 new file mode 100644 index 0000000..230785c --- /dev/null +++ b/MockVeeam.ps1 @@ -0,0 +1,83 @@ +# Stand-ins for the Veeam cmdlets, shaped like the real objects' properties the collector reads. +# Dot-source this before running the collector to try it without a VBR server: +# +# . ./tests/MockVeeam.ps1 +# ./Send-VeeamBackupStatus.ps1 -ConfigPath ./tests/config.json -DryRun + +# Fixed to the top of the hour, and IDs derived from names/times, so repeated runs see the same +# jobs and sessions, like a real server +$now = Get-Date -Minute 0 -Second 0 -Millisecond 0 +function New-StableGuid([string]$Seed) { + [guid]::new([Security.Cryptography.MD5]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($Seed))) +} +$notSet = Get-Date '1900-01-01' + +function New-MockJob($Name, $Type, $NextRun, [bool]$Enabled = $true) { + [pscustomobject]@{ + Id = New-StableGuid $Name; Name = $Name; JobType = $Type + IsScheduleEnabled = $Enabled + ScheduleOptions = [pscustomobject]@{ NextRun = $NextRun } + } +} + +function New-MockSession($Job, $Start, $End, $State, $Result, $Reason = '', $Tasks = @(), $LogRecords = @(), $Size = 0) { + $s = [pscustomobject]@{ + Id = New-StableGuid "$($Job.Name)|$($Start.Ticks)"; JobId = $Job.Id; JobName = $Job.Name + CreationTime = $Start; EndTime = $End; State = $State; Result = $Result + IsRetryMode = $false + Info = [pscustomobject]@{ Reason = $Reason } + BackupStats = [pscustomobject]@{ BackupSize = $Size; DataSize = $Size * 3 } + Logger = [pscustomobject]@{ Records = $LogRecords } + Tasks = $Tasks + } + $s.Logger | Add-Member ScriptMethod GetLog { [pscustomobject]@{ UpdatedRecords = $this.Records } } + $s | Add-Member ScriptMethod GetTaskSessions { $this.Tasks } + $s +} + +function New-MockTask($Name, $Status, $Reason = '') { + [pscustomobject]@{ Name = $Name; Status = $Status; Info = [pscustomobject]@{ Reason = $Reason } } +} + +$global:MockJobs = @( + (New-MockJob 'File Server – Nightly' 'Backup' $now.Date.AddDays(1).AddHours(22)), + # Older VBR versions return NextRun as a culture-formatted string + (New-MockJob 'SQL01 – Logs' 'Backup' $now.AddMinutes(40).ToString()), + (New-MockJob 'Offsite Copy' 'BackupSync' $null), + (New-MockJob 'Old Exchange' 'Backup' $now.AddDays(1) $false), + # VBR 12+ backup copy job (immediate mode, no schedule): the "policy" job Get-VBRJob returns + (New-MockJob 'Cloud Copy' 'SimpleBackupCopyPolicy' $null) +) +$fs, $sql, $copy, $old, $cloud = $global:MockJobs + +# Its hidden worker job, one per source job. Get-VBRJob doesn't return it, and Get-VBRBackupSession +# returns only the worker's sessions. Shape taken from a real VBR 13 server. +$cloudWorker = [pscustomobject]@{ + Id = New-StableGuid 'Cloud Copy\File Server – Nightly'; Name = 'Cloud Copy\File Server – Nightly' + JobType = 'SimpleBackupCopyWorker'; PolicyJobId = $cloud.Id +} +function New-MockWorkerSession($Start, $End, $Result) { + $s = New-MockSession $cloudWorker $Start $End 'Stopped' $Result -Size 40GB + $s | Add-Member ScriptMethod GetJob { $cloudWorker } + $s +} + +$global:MockSessions = @( + (New-MockSession $fs $now.AddDays(-1).AddHours(-2) $now.AddDays(-1).AddHours(-1) 'Stopped' 'Success' -Size 128GB -Tasks @( + (New-MockTask 'FS01' 'Success'), (New-MockTask 'FS02' 'Success'))), + (New-MockSession $fs $now.AddMinutes(-20) $notSet 'Working' 'None' -Tasks @((New-MockTask 'FS01' 'InProgress'))), + (New-MockSession $sql $now.AddHours(-1) $now.AddMinutes(-55) 'Stopped' 'Warning' -Size 2GB ` + -Tasks @((New-MockTask 'SQL01' 'Warning' 'Failed to truncate transaction logs for database "ERP_PROD"'))), + (New-MockSession $copy $now.AddHours(-5) $now.AddHours(-4) 'Stopped' 'Failed' 'Error: Unable to connect to repository NAS02' ` + -LogRecords @( + [pscustomobject]@{ Title = 'Preparing backup chain'; Status = 'ESucceeded' }, + [pscustomobject]@{ Title = 'Network path not found (0x80070035)'; Status = 'EFailed' })), + (New-MockSession $old $now.AddDays(-10) $now.AddDays(-10).AddHours(1) 'Stopped' 'Success'), # outside 48h lookback + (New-MockWorkerSession $now.AddDays(-1).AddMinutes(-55) $now.AddDays(-1).AddMinutes(-30) 'Success'), + (New-MockWorkerSession $now.AddMinutes(-55) $now.AddMinutes(-35) 'Warning'), + # A session from a job the collector doesn't know (e.g. deleted): must be ignored, not crash + (New-MockSession ([pscustomobject]@{ Id = New-StableGuid 'gone'; Name = 'Deleted job' }) $now.AddHours(-2) $now.AddHours(-1) 'Stopped' 'Success') +) + +function Get-VBRJob { [CmdletBinding()] param() $global:MockJobs } +function Get-VBRBackupSession { [CmdletBinding()] param() $global:MockSessions } diff --git a/README.md b/README.md index 2b324f3..7e96948 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,81 @@ -# Backup-Dashboard +# Veeam Backup & Replication collector -Proof of concept backup dashboard, intended to display backup jobs from multiple sources. +A PowerShell script that runs on each client's VBR server and reports job results to the dashboard. +It covers jobs returned by `Get-VBRJob`: VM backup, replication and backup copy jobs. -``` -collectors ──POST /api/v1/ingest (API key)──▶ ┌──────────────┐ - │ FastAPI + │ -browser ──nginx (basic auth, TLS)──▶ /api/* ─▶│ SQLite │ - └─ static files: public/ └──────────────┘ -``` +## Install (on the VBR server, as an administrator) -- `public/`: static frontend (no build step). Add `?mock=1` to the URL to use built-in sample data. -- `backend/`: FastAPI API backed by a single SQLite file. -- `deploy/`: nginx site and systemd unit. -- `collectors/`: scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md)). +Requires PowerShell 7 installed machine-wide (`winget install Microsoft.PowerShell` or the MSI), +which VBR 13's Veeam module expects. The Microsoft Store version won't work, because the SYSTEM +account can't run it. The installer checks for this before changing anything. -## Data model +1. On the dashboard host, create a key for this server: + ```bash + python manage.py create-key veeam-acme + ``` +2. Copy this folder to the VBR server and run, from PowerShell 7 (`pwsh`): + ```powershell + .\Install-VeeamCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" + ``` + It prompts for the key, then: + - writes `C:\ProgramData\BackupDashboard\veeam-collector.json` (readable only by SYSTEM and Administrators) + - copies the script to `C:\Program Files\BackupDashboard\` + - registers the scheduled task **Backup Dashboard - Veeam collector** (SYSTEM, every 15 minutes, + run by `C:\Program Files\PowerShell\7\pwsh.exe`; use `-PwshPath` if it's installed elsewhere) +3. Preview the payloads, then backfill 30 days so the history view is populated straight away + (from `pwsh`): + ```powershell + & 'C:\Program Files\BackupDashboard\Send-VeeamBackupStatus.ps1' -DryRun + & 'C:\Program Files\BackupDashboard\Send-VeeamBackupStatus.ps1' -LookbackHours 720 + ``` -- **Tenant** (company) → **Job** → **Run**. -- A job is keyed by `(tenant, source_system, job.external_id)`, so renaming a job at the source updates it instead of creating a duplicate. -- A run has `status` (`running` | `success` | `warning` | `failed`), `started_at` and `finished_at`. - The dashboard's **Last Run** is the most recent `finished_at`. While a job is running it shows - **Running**, and Last Run still shows when the previous run finished. -- Runs older than `BACKUP_DASH_RETENTION_DAYS` (default 90, minimum 30) are purged every 6 hours. +Re-run the installer to change settings or update the script. To remove the collector, +delete the scheduled task and both `BackupDashboard` folders. -## Reporting results (collectors) +## What it sends -Each collector (a script on a Veeam server, a webhook relay, etc.) gets its own API key: - -```bash -cd backend && python manage.py create-key veeam-acme -``` - -Then it POSTs one report per run. Send `running` when a run starts (optional) and the final status -when it ends. Reports with the same `started_at` (or `run.external_id`) update the same run, and -retries are safe. - -```bash -curl -X POST https://backups.example.com/api/v1/ingest \ - -H "Authorization: Bearer bdk_..." -H "Content-Type: application/json" \ - -d '{ - "tenant": "Acme Corp", - "source_system": "Veeam B&R", - "job": { "external_id": "4f1c…", "name": "File Server – Nightly", "next_run_at": "2026-10-05T22:00:00-04:00" }, - "run": { - "status": "success", - "started_at": "2026-10-04T22:00:05-04:00", - "finished_at": "2026-10-04T22:47:12-04:00", - "message": "Job completed successfully.", - "size_bytes": 128849018880, - "details": { "repository": "REPO-01" } - } - }' -``` - -- All timestamps must include a timezone offset (`Get-Date -Format o` in PowerShell does this). -- `job.next_run_at` controls the **Overdue** flag. If you omit it, the stored value is kept. If you send `null`, it's cleared. -- `run` can be omitted to only register or update a job. -- Unknown fields are rejected (HTTP 422), so typos in collector scripts fail loudly. - -## API (read) - -| Endpoint | Returns | +| Dashboard | Veeam | |---|---| -| `GET /api/v1/jobs` | All jobs with current status, `running_since`, `last_run_at`, `next_run_at` | -| `GET /api/v1/jobs/{id}` | One job, plus the last finished run's duration, size, message and details | -| `GET /api/v1/jobs/{id}/runs?days=30` | Run history (no messages), newest first | -| `GET /api/v1/runs/{id}` | One run including message and details | -| `GET /api/v1/health` | `{"status": "ok"}` | +| Company | `Tenant` from the config | +| Job | Job name. The job's GUID is the ID, so renaming a job in Veeam doesn't duplicate it | +| Status | Session result (Success / Warning / Failed). Any state other than Stopped is **Running**. A session that stopped with no result (e.g. cancelled) is reported as **Warning** | +| Started / finished | Session `CreationTime` / `EndTime` | +| Next scheduled | Job's next run. Sent as empty for disabled jobs and chained ("run after") jobs, so those are never flagged Overdue | +| Message | Session reason, each VM or object that didn't succeed, and warning/error lines from the session log | +| Size | `BackupStats.BackupSize` | -The OpenAPI schema is at `/api/v1/openapi.json`. +**Backup copy jobs (VBR 12+):** Veeam records their sessions under hidden per-source "worker" jobs. +The collector files each worker session under its copy job (via the worker's `PolicyJobId`), so a +copy job gets one run per source job it copies, with the source in `copied_from`. -## Authentication +Each run covers the last 48 hours (`LookbackHours`), so a few missed runs (server rebooting, API +down) fill in automatically. Sessions already delivered are recorded in +`veeam-collector.state.json`. Running sessions are re-sent until they finish. -- **Dashboard users:** nginx basic auth over HTTPS (`deploy/nginx.conf`). Every user can see every tenant. - *Planned:* Microsoft 365 sign-in through [oauth2-proxy](https://oauth2-proxy.github.io/oauth2-proxy/) - with the Entra ID provider and nginx `auth_request`. That's an nginx-only change. The frontend - already tells the user to reload when a session expires (HTTP 401). -- **Collectors:** per-collector bearer keys, checked by the app. Only a SHA-256 hash is stored. - To revoke one, run `python manage.py revoke-key NAME`. +## Troubleshooting -## Development +- Log: `C:\ProgramData\BackupDashboard\veeam-collector.log` (rotates at 5 MB). +- Exit codes: `0` OK, `1` some jobs failed to send (they're retried next run), `2` setup failed + (config, Veeam module, or Veeam connection). +- `API key rejected`: the key was revoked or mistyped. Re-run the installer with a new key. +- A job shows **Unknown** on the dashboard: the collector sent the job but matched no sessions to it. + Run `.\Get-VeeamCollectorDiagnostics.ps1` (read-only). It lists sessions whose job ID matches no + job, plus the type and link properties of the jobs left without sessions. +- `Get-VBRBackupSession` loads every session in the VBR database. On a server with years of history, + this can make each run take a minute or more. The task's 30-minute limit and "don't start if + already running" setting keep that safe. -```bash -cd backend -python -m venv .venv && . .venv/bin/activate -pip install -r requirements.txt -r requirements-dev.txt -pytest -python manage.py seed-demo # 30 days of fake history -BACKUP_DASH_SERVE_STATIC=1 uvicorn --factory app.main:create_app --reload -# → http://localhost:8000 +## Testing without Veeam + +`tests/MockVeeam.ps1` defines stand-in `Get-VBRJob` / `Get-VBRBackupSession` cmdlets with a mix of +success, warning, failed, running and disabled jobs: + +```powershell +. ./tests/MockVeeam.ps1 +./Send-VeeamBackupStatus.ps1 -ConfigPath ./my-test-config.json -DryRun ``` -## Deployment (Linux, nginx) +## Not covered yet -1. Copy the repo to `/opt/backup-dashboard`, create a venv at `/opt/backup-dashboard/venv`, and run `pip install -r backend/requirements.txt`. -2. Create a `backupdash` system user, then install `deploy/backup-dashboard-api.service` into `/etc/systemd/system/` and enable it. -3. Copy `public/` to `/var/www/backup-dashboard/public`, install `deploy/nginx.conf`, get a certificate with certbot, and create users with `htpasswd -B`. -4. Run the `manage.py` commands as the service user with `BACKUP_DASH_DB=/var/lib/backup-dashboard/backup-dashboard.db`. - -Back up the SQLite file with `sqlite3 backup-dashboard.db ".backup copy.db"`. Don't use a plain `cp` -while the service is running. +- Veeam Agent jobs managed by VBR (`Get-VBRComputerBackupJob`), tape jobs, and SureBackup. +- Veeam Backup for Microsoft 365, which has its own PowerShell module. diff --git a/Send-VeeamBackupStatus.ps1 b/Send-VeeamBackupStatus.ps1 new file mode 100644 index 0000000..79ff658 --- /dev/null +++ b/Send-VeeamBackupStatus.ps1 @@ -0,0 +1,335 @@ +<# +.SYNOPSIS + Reports Veeam Backup & Replication job results to the Backup Dashboard. + +.DESCRIPTION + Runs on the VBR server, normally as a SYSTEM scheduled task every 15 minutes + (see Install-VeeamCollector.ps1). For every backup, replica and backup copy job it sends: + - the job's next scheduled run (drives the dashboard's Overdue flag), and + - every session from the last LookbackHours that the dashboard hasn't already received + in its final state. Running sessions are re-sent each time until they finish. + + Already-reported sessions are tracked in a state file, and the ingest API is idempotent, + so a failed or repeated run is harmless: anything not confirmed is retried next time. + + Compatible with Windows PowerShell 5.1 and PowerShell 7. + +.PARAMETER ConfigPath + JSON config with ApiUrl, ApiKey and Tenant (optional: SourceSystem, LookbackHours). + +.PARAMETER LookbackHours + Overrides the config value. Use 720 once to backfill 30 days of history. + +.PARAMETER DryRun + Print the payloads instead of sending them. Doesn't touch the state or log files. + +.EXAMPLE + .\Send-VeeamBackupStatus.ps1 -DryRun +.EXAMPLE + .\Send-VeeamBackupStatus.ps1 -LookbackHours 720 +#> +[CmdletBinding()] +param( + [string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\veeam-collector.json'), + [int]$LookbackHours = 0, + [switch]$DryRun +) + +$ErrorActionPreference = 'Stop' + +$DataDir = Split-Path -Parent $ConfigPath +$LogPath = Join-Path $DataDir 'veeam-collector.log' +$StatePath = Join-Path $DataDir 'veeam-collector.state.json' +$MaxLogBytes = 5MB +$Invariant = [Globalization.CultureInfo]::InvariantCulture + +# ---------- Helpers ---------- + +function Write-Log { + param([string]$Message, [string]$Level = 'INFO') + $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message + Write-Host $line + if ($DryRun) { return } + if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) { + Move-Item $LogPath "$LogPath.1" -Force + } + Add-Content -Path $LogPath -Value $line -Encoding UTF8 +} + +function Get-ErrorText($ErrorRecord) { + # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here + if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { + return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)" + } + return $ErrorRecord.Exception.Message +} + +# The API needs an explicit UTC offset. Veeam returns local times, often with Kind=Unspecified, +# which DateTimeOffset treats as local, exactly what we want. +function ConvertTo-IsoTime($Value) { + if ($null -eq $Value) { return $null } + $dt = [datetime]$Value + if ($dt.Year -le 1900) { return $null } # Veeam's "not set" value (e.g. EndTime while running) + return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant) +} + +function Import-Veeam { + if (Get-Command Get-VBRJob -ErrorAction SilentlyContinue) { return } # already loaded + if (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell) { + Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue # VBR 11+ + } else { + Add-PSSnapin VeeamPSSnapin # VBR 10 and earlier + } +} + +function Get-NextRun($Job) { + try { + if (-not $Job.IsScheduleEnabled) { return $null } + $raw = $Job.ScheduleOptions.NextRun + if (-not $raw) { return $null } # e.g. chained jobs that run "after" another job + if ($raw -is [datetime]) { return ConvertTo-IsoTime $raw } + # Older versions return a string formatted with the server's culture, so parse with it + $parsed = [datetime]::MinValue + if ([datetime]::TryParse([string]$raw, [Globalization.CultureInfo]::CurrentCulture, + [Globalization.DateTimeStyles]::None, [ref]$parsed)) { + return ConvertTo-IsoTime $parsed + } + Write-Log "Couldn't parse next run '$raw' for job '$($Job.Name)'" 'WARN' + } catch { + Write-Log "Couldn't read schedule for job '$($Job.Name)': $(Get-ErrorText $_)" 'WARN' + } + return $null +} + +function Get-RunStatus($Session) { + if ([string]$Session.State -ne 'Stopped') { return 'running' } + switch ([string]$Session.Result) { + 'Success' { return 'success' } + 'Warning' { return 'warning' } + 'Failed' { return 'failed' } + # Stopped without a result (e.g. cancelled): surface it rather than hide it + default { return 'warning' } + } +} + +# Builds a readable summary: the session's reason, any task (VM/object) that didn't succeed, +# and warning/error lines from the session log. +function Get-RunMessage($Session, [string]$Status, $Tasks) { + $lines = New-Object 'System.Collections.Generic.List[string]' + try { + $reason = [string]$Session.Info.Reason + if ($reason.Trim()) { $lines.Add($reason.Trim()) } + } catch { } + + foreach ($task in $Tasks) { + $taskStatus = [string]$task.Status + if ($taskStatus -and $taskStatus -ne 'Success' -and $taskStatus -ne 'InProgress' -and $taskStatus -ne 'Pending') { + $taskReason = '' + try { $taskReason = ([string]$task.Info.Reason).Trim() } catch { } + $line = "$($task.Name): $taskStatus" + if ($taskReason) { $line += " - $taskReason" } + $lines.Add($line) + } + } + + if ($Status -eq 'warning' -or $Status -eq 'failed') { + try { + foreach ($record in $Session.Logger.GetLog().UpdatedRecords) { + $recordStatus = [string]$record.Status + if ($recordStatus -eq 'EWarning' -or $recordStatus -eq 'EFailed') { + $title = ([string]$record.Title).Trim() + if ($title -and -not $lines.Contains($title)) { $lines.Add($title) } + } + } + } catch { } + } + + if ($lines.Count -eq 0) { + switch ($Status) { + 'success' { return 'Job completed successfully.' } + 'running' { return '' } + default { return "Job finished with result '$($Session.Result)'." } + } + } + return ($lines -join "`n") +} + +# VBR 12+ backup copy jobs: Get-VBRJob returns the copy (policy) job, but Get-VBRBackupSession +# returns only the sessions of its hidden worker jobs, one per source job, never the policy's own. +# Each worker job records its policy in PolicyJobId, so those sessions can be filed under it. +function Get-PolicyJobId($Session, [hashtable]$KnownJobIds) { + try { + $policyId = [string]$Session.GetJob().PolicyJobId + if ($policyId -and $KnownJobIds.ContainsKey($policyId)) { return $policyId } + } catch { } + return '' +} + +function ConvertTo-Run($Session, $Job) { + $status = Get-RunStatus $Session + $tasks = @() + try { $tasks = @($Session.GetTaskSessions()) } catch { } + + $details = [ordered]@{ + job_type = [string]$Job.JobType + vbr_server = $env:COMPUTERNAME + objects = $tasks.Count + } + $failedTasks = @($tasks | Where-Object { [string]$_.Status -eq 'Failed' }).Count + $warningTasks = @($tasks | Where-Object { [string]$_.Status -eq 'Warning' }).Count + if ($failedTasks) { $details.objects_failed = $failedTasks } + if ($warningTasks) { $details.objects_warning = $warningTasks } + try { if ($Session.IsRetryMode) { $details.retry = $true } } catch { } + # Worker session of a backup copy job: say which source job it copied + # (worker sessions are named "\") + if ($Session.JobId.ToString() -ne $Job.Id.ToString()) { + $details.copied_from = ([string]$Session.JobName -split '\\')[-1] + } + + $run = [ordered]@{ + external_id = $Session.Id.ToString() + status = $status + started_at = ConvertTo-IsoTime $Session.CreationTime + message = Get-RunMessage $Session $status $tasks + details = $details + } + if ($status -ne 'running') { + $finished = ConvertTo-IsoTime $Session.EndTime + if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time + try { + $size = [long]$Session.BackupStats.BackupSize + if ($size -gt 0) { $run.size_bytes = $size } + $dataSize = [long]$Session.BackupStats.DataSize + if ($dataSize -gt 0) { $details.data_size_bytes = $dataSize } + } catch { } + } + return $run +} + +function Send-Report($Body) { + $json = $Body | ConvertTo-Json -Depth 6 -Compress + if ($DryRun) { + Write-Host $json + return + } + # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1, + # mangling non-ASCII job names (e.g. en dashes). + $params = @{ + Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest" + Method = 'Post' + Headers = @{ Authorization = "Bearer $($Config.ApiKey)" } + ContentType = 'application/json; charset=utf-8' + Body = [Text.Encoding]::UTF8.GetBytes($json) + TimeoutSec = 30 + } + Invoke-RestMethod @params | Out-Null +} + +# ---------- Main ---------- + +try { + $Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json + foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') { + if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" } + } + $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Veeam B&R' } + if ($LookbackHours -le 0) { + $LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 } + } + + # Windows PowerShell 5.1 may default to TLS 1.0 + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + + Import-Veeam + + # Session ID -> start time (Unix seconds), for sessions already delivered in their final state. + # Numbers rather than ISO strings: PowerShell 7's ConvertFrom-Json turns date strings into + # DateTimes and loses the offset. + $sent = @{} + if (Test-Path $StatePath) { + $saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json + if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } } + } + + $since = (Get-Date).AddHours(-$LookbackHours) + $jobs = @(Get-VBRJob -WarningAction SilentlyContinue) + + $jobIds = @{} + foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $true } + + # Get-VBRBackupSession returns every session in the database, so fetch once and bucket by job + $sessionsByJob = @{} + $policyByWorker = @{} # worker job ID -> policy job ID ('' if it isn't a worker of a listed job) + foreach ($s in Get-VBRBackupSession -WarningAction SilentlyContinue) { + if ($s.CreationTime -lt $since) { continue } + $key = $s.JobId.ToString() + if (-not $jobIds.ContainsKey($key)) { + if (-not $policyByWorker.ContainsKey($key)) { $policyByWorker[$key] = Get-PolicyJobId $s $jobIds } + if (-not $policyByWorker[$key]) { continue } + $key = $policyByWorker[$key] + } + if (-not $sessionsByJob.ContainsKey($key)) { $sessionsByJob[$key] = New-Object 'System.Collections.Generic.List[object]' } + $sessionsByJob[$key].Add($s) + } +} catch { + Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR' + exit 2 +} + +$failures = 0 +$runsSent = 0 +foreach ($job in $jobs) { + $jobInfo = [ordered]@{ + external_id = $job.Id.ToString() + name = $job.Name + # Always sent (null when unscheduled/disabled) so the dashboard's value tracks Veeam's + next_run_at = Get-NextRun $job + } + $pending = @() + if ($sessionsByJob.ContainsKey($jobInfo.external_id)) { + $pending = @($sessionsByJob[$jobInfo.external_id] | + Where-Object { -not $sent.ContainsKey($_.Id.ToString()) } | + Sort-Object CreationTime) + } + + try { + if ($pending.Count -eq 0) { + # Keep next_run_at fresh even when nothing new ran + Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo }) + } + foreach ($session in $pending) { + $run = ConvertTo-Run $session $job + Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run }) + $runsSent++ + if ($run.status -ne 'running') { + $sent[$run.external_id] = ([DateTimeOffset][datetime]$session.CreationTime).ToUnixTimeSeconds() + } + } + } catch { + $failures++ + Write-Log "Failed to report job '$($job.Name)': $(Get-ErrorText $_)" 'ERROR' + $httpStatus = 0 + try { $httpStatus = [int]$_.Exception.Response.StatusCode } catch { } + if ($httpStatus -eq 401) { + Write-Log 'API key rejected; check ApiKey in the config. Stopping.' 'ERROR' + break + } + } +} + +if (-not $DryRun) { + # Forget sessions that have aged out of the lookback window (plus a day of slack) + $cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds() + $keep = [ordered]@{} + foreach ($id in $sent.Keys) { + if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] } + } + try { + $keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8 + } catch { + Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN' + } +} + +Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $jobs.Count, $runsSent, $failures) +if ($failures) { exit 1 }