Upload files to "/"
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Read-only report of how Veeam jobs and sessions relate, for troubleshooting the collector.
|
||||
|
||||
.DESCRIPTION
|
||||
The collector matches sessions to jobs by Session.JobId = Job.Id. When a job shows as
|
||||
"Unknown" on the dashboard, no session matched. This lists:
|
||||
1. every job from Get-VBRJob and how many sessions matched it
|
||||
2. sessions whose JobId matches no job (the runs that are going missing)
|
||||
3. for jobs with no matching sessions, their type/link-related properties and methods
|
||||
4. what other job cmdlets (e.g. Get-VBRBackupCopyJob) return
|
||||
Nothing is changed in Veeam or the dashboard. Output is also saved to a text file.
|
||||
|
||||
.EXAMPLE
|
||||
.\Get-VeeamCollectorDiagnostics.ps1
|
||||
.EXAMPLE
|
||||
.\Get-VeeamCollectorDiagnostics.ps1 -LookbackHours 168
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[int]$LookbackHours = 48,
|
||||
[string]$OutFile = (Join-Path $env:TEMP 'veeam-collector-diagnostics.txt')
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$LinkPattern = 'Parent|Linked|Worker|Child|Policy|Source|Target|Type|Cloud|Mode|Session'
|
||||
|
||||
if (-not (Get-Command Get-VBRJob -ErrorAction SilentlyContinue)) {
|
||||
if (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell) {
|
||||
Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue
|
||||
} else {
|
||||
Add-PSSnapin VeeamPSSnapin
|
||||
}
|
||||
}
|
||||
|
||||
function Format-Value($Value) {
|
||||
if ($null -eq $Value) { return '<null>' }
|
||||
$text = ($Value | ForEach-Object { [string]$_ }) -join ', '
|
||||
if ($text.Length -gt 120) { $text = $text.Substring(0, 117) + '...' }
|
||||
return $text
|
||||
}
|
||||
|
||||
# Property and method names matching $LinkPattern, with property values. Every read is guarded:
|
||||
# some Veeam properties throw when accessed.
|
||||
function Show-LinkMembers($Object) {
|
||||
foreach ($p in $Object.PSObject.Properties | Where-Object { $_.Name -match $LinkPattern }) {
|
||||
$value = try { Format-Value $p.Value } catch { "<error: $($_.Exception.Message)>" }
|
||||
' {0} = {1}' -f $p.Name, $value
|
||||
}
|
||||
$methods = @($Object | Get-Member -MemberType Methods -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.Name -match $LinkPattern } | ForEach-Object Name)
|
||||
if ($methods) { ' methods: ' + ($methods -join ', ') }
|
||||
}
|
||||
|
||||
function Get-Prop($Object, [string]$Name) {
|
||||
try { Format-Value $Object.$Name } catch { '<error>' }
|
||||
}
|
||||
|
||||
$report = & {
|
||||
$vbrVersion = try {
|
||||
(Get-Item (Join-Path ${env:ProgramFiles} 'Veeam\Backup and Replication\Backup\Veeam.Backup.Core.dll')).VersionInfo.ProductVersion
|
||||
} catch { 'unknown' }
|
||||
"Generated $(Get-Date -Format o) on $env:COMPUTERNAME, VBR $vbrVersion, PowerShell $($PSVersionTable.PSVersion)"
|
||||
"Lookback: $LookbackHours hours"
|
||||
''
|
||||
|
||||
$since = (Get-Date).AddHours(-$LookbackHours)
|
||||
$jobs = @(Get-VBRJob -WarningAction SilentlyContinue)
|
||||
$sessions = @(Get-VBRBackupSession -WarningAction SilentlyContinue | Where-Object { $_.CreationTime -ge $since })
|
||||
$jobIds = @{}
|
||||
foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $j }
|
||||
|
||||
'== 1. Jobs from Get-VBRJob'
|
||||
$jobs | ForEach-Object {
|
||||
$id = $_.Id.ToString()
|
||||
[pscustomobject]@{
|
||||
Name = $_.Name
|
||||
JobType = Get-Prop $_ 'JobType'
|
||||
TypeToString = Get-Prop $_ 'TypeToString'
|
||||
Id = $id
|
||||
MatchingSessions = @($sessions | Where-Object { $_.JobId.ToString() -eq $id }).Count
|
||||
}
|
||||
} | Format-Table -AutoSize | Out-String -Width 300
|
||||
|
||||
'== 2. Sessions whose JobId matches no job from Get-VBRJob'
|
||||
$orphans = @($sessions | Where-Object { -not $jobIds.ContainsKey($_.JobId.ToString()) })
|
||||
if ($orphans.Count -eq 0) {
|
||||
' (none)'
|
||||
} else {
|
||||
$orphans | Group-Object { $_.JobId.ToString() } | ForEach-Object {
|
||||
$latest = $_.Group | Sort-Object CreationTime | Select-Object -Last 1
|
||||
[pscustomobject]@{
|
||||
JobId = $_.Name
|
||||
JobName = Get-Prop $latest 'JobName'
|
||||
JobType = Get-Prop $latest 'JobType'
|
||||
Sessions = $_.Count
|
||||
LatestStart = $latest.CreationTime
|
||||
State = Get-Prop $latest 'State'
|
||||
Result = Get-Prop $latest 'Result'
|
||||
}
|
||||
} | Format-Table -AutoSize | Out-String -Width 300
|
||||
|
||||
' Link-related members of one orphan session:'
|
||||
Show-LinkMembers $orphans[0]
|
||||
''
|
||||
' Job object for that session (via GetJob(), if available):'
|
||||
try {
|
||||
$owner = $orphans[0].GetJob()
|
||||
' Name = {0}, Id = {1}' -f (Get-Prop $owner 'Name'), (Get-Prop $owner 'Id')
|
||||
Show-LinkMembers $owner
|
||||
} catch {
|
||||
" not available: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
''
|
||||
|
||||
'== 3. Jobs with no matching sessions in the window'
|
||||
$empty = @($jobs | Where-Object { $id = $_.Id.ToString(); -not ($sessions | Where-Object { $_.JobId.ToString() -eq $id }) })
|
||||
if ($empty.Count -eq 0) { ' (none)' }
|
||||
foreach ($j in $empty) {
|
||||
" $($j.Name) [$($j.Id)]"
|
||||
Show-LinkMembers $j
|
||||
try {
|
||||
$last = $j.FindLastSession()
|
||||
if ($last) {
|
||||
' FindLastSession(): JobId = {0}, JobName = {1}, Start = {2}, Result = {3}' -f `
|
||||
(Get-Prop $last 'JobId'), (Get-Prop $last 'JobName'), $last.CreationTime, (Get-Prop $last 'Result')
|
||||
} else {
|
||||
' FindLastSession(): <null>'
|
||||
}
|
||||
} catch {
|
||||
" FindLastSession(): not available: $($_.Exception.Message)"
|
||||
}
|
||||
''
|
||||
}
|
||||
|
||||
'== 4. Other job cmdlets'
|
||||
foreach ($cmd in 'Get-VBRBackupCopyJob', 'Get-VBRComputerBackupCopyJob', 'Get-VBRComputerBackupJob') {
|
||||
if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) { " ${cmd}: not available"; continue }
|
||||
try {
|
||||
$found = @(& $cmd -WarningAction SilentlyContinue)
|
||||
" ${cmd}: $($found.Count) job(s)"
|
||||
foreach ($f in $found) {
|
||||
' {0} [{1}] Type = {2}, Mode = {3}' -f (Get-Prop $f 'Name'), (Get-Prop $f 'Id'), (Get-Prop $f 'Type'), (Get-Prop $f 'Mode')
|
||||
}
|
||||
} catch {
|
||||
" ${cmd}: error: $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
} | Out-String -Width 300
|
||||
|
||||
$report
|
||||
$report | Set-Content -Path $OutFile -Encoding UTF8
|
||||
Write-Host "Saved to $OutFile"
|
||||
@@ -0,0 +1,102 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Installs the Veeam collector on a VBR server as a SYSTEM scheduled task.
|
||||
|
||||
.DESCRIPTION
|
||||
- Copies Send-VeeamBackupStatus.ps1 to Program Files (only admins can modify it, which
|
||||
matters because it runs as SYSTEM).
|
||||
- Writes the config to ProgramData\BackupDashboard and restricts that folder to SYSTEM and
|
||||
Administrators, since it contains the API key.
|
||||
- Registers a scheduled task that runs every IntervalMinutes under PowerShell 7 (pwsh.exe),
|
||||
which the Veeam module on VBR 13 expects. Install PowerShell 7 machine-wide (MSI or winget)
|
||||
first: Microsoft Store installs can't be run by SYSTEM.
|
||||
Safe to re-run to change settings or update the script.
|
||||
|
||||
.EXAMPLE
|
||||
.\Install-VeeamCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp"
|
||||
(prompts for the API key)
|
||||
#>
|
||||
#Requires -RunAsAdministrator
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)][string]$ApiUrl,
|
||||
[Parameter(Mandatory)][string]$Tenant,
|
||||
[string]$SourceSystem = 'Veeam B&R',
|
||||
[ValidateRange(5, 1440)][int]$IntervalMinutes = 15,
|
||||
[ValidateRange(1, 720)][int]$LookbackHours = 48,
|
||||
[string]$InstallDir = (Join-Path $env:ProgramFiles 'BackupDashboard'),
|
||||
# Defaults to the machine-wide PowerShell 7 install
|
||||
[string]$PwshPath
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$TaskName = 'Backup Dashboard - Veeam collector'
|
||||
$DataDir = Join-Path $env:ProgramData 'BackupDashboard'
|
||||
$ConfigPath = Join-Path $DataDir 'veeam-collector.json'
|
||||
|
||||
# Find PowerShell 7 before changing anything, so a missing install fails cleanly
|
||||
if (-not $PwshPath) {
|
||||
$PwshPath = Join-Path $env:ProgramFiles 'PowerShell\7\pwsh.exe'
|
||||
if (-not (Test-Path $PwshPath)) {
|
||||
$onPath = Get-Command pwsh.exe -ErrorAction SilentlyContinue
|
||||
if ($onPath) { $PwshPath = $onPath.Source }
|
||||
}
|
||||
}
|
||||
if (-not (Test-Path $PwshPath)) {
|
||||
throw "PowerShell 7 not found at '$PwshPath'. Install it machine-wide (e.g. winget install Microsoft.PowerShell) or pass -PwshPath."
|
||||
}
|
||||
if ($PwshPath -match '\\WindowsApps\\') {
|
||||
throw "'$PwshPath' is a Microsoft Store install, which the SYSTEM account can't run. Install the MSI or winget version instead."
|
||||
}
|
||||
$pwshVersion = & $PwshPath -NoProfile -NonInteractive -Command '$PSVersionTable.PSVersion.ToString()'
|
||||
$pwshMajor = & $PwshPath -NoProfile -NonInteractive -Command '$PSVersionTable.PSVersion.Major'
|
||||
if ($LASTEXITCODE -or ($pwshMajor -as [int]) -lt 7) {
|
||||
throw "'$PwshPath' isn't a working PowerShell 7 (reported '$pwshVersion')."
|
||||
}
|
||||
|
||||
if ($ApiUrl -notmatch '^https://') {
|
||||
Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.'
|
||||
}
|
||||
|
||||
# Read the key as a SecureString so it isn't echoed to the console or kept in shell history
|
||||
$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key <name>)'
|
||||
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
||||
try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
||||
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
||||
if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." }
|
||||
|
||||
# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows)
|
||||
New-Item -ItemType Directory -Path $DataDir -Force | Out-Null
|
||||
& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
|
||||
if ($LASTEXITCODE) { throw "icacls failed on $DataDir" }
|
||||
|
||||
[ordered]@{
|
||||
ApiUrl = $ApiUrl.TrimEnd('/')
|
||||
ApiKey = $apiKey
|
||||
Tenant = $Tenant
|
||||
SourceSystem = $SourceSystem
|
||||
LookbackHours = $LookbackHours
|
||||
} | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8
|
||||
|
||||
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
|
||||
$scriptPath = Join-Path $InstallDir 'Send-VeeamBackupStatus.ps1'
|
||||
Copy-Item -Path (Join-Path $PSScriptRoot 'Send-VeeamBackupStatus.ps1') -Destination $scriptPath -Force
|
||||
Unblock-File -Path $scriptPath
|
||||
|
||||
$action = New-ScheduledTaskAction -Execute $PwshPath `
|
||||
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`""
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) `
|
||||
-RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes)
|
||||
$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest
|
||||
$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) `
|
||||
-MultipleInstances IgnoreNew -StartWhenAvailable
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal `
|
||||
-Settings $settings -Description 'Sends Veeam job results to the Backup Dashboard' -Force | Out-Null
|
||||
|
||||
Write-Host "Installed. Config: $ConfigPath"
|
||||
Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using PowerShell $pwshVersion ($PwshPath)."
|
||||
Write-Host ''
|
||||
Write-Host 'Next steps:'
|
||||
Write-Host " 1. Preview what will be sent: & '$PwshPath' -File '$scriptPath' -DryRun"
|
||||
Write-Host " 2. Backfill 30 days of history: & '$PwshPath' -File '$scriptPath' -LookbackHours 720"
|
||||
Write-Host " 3. Check the log: Get-Content '$DataDir\veeam-collector.log' -Tail 20"
|
||||
@@ -0,0 +1,83 @@
|
||||
# Stand-ins for the Veeam cmdlets, shaped like the real objects' properties the collector reads.
|
||||
# Dot-source this before running the collector to try it without a VBR server:
|
||||
#
|
||||
# . ./tests/MockVeeam.ps1
|
||||
# ./Send-VeeamBackupStatus.ps1 -ConfigPath ./tests/config.json -DryRun
|
||||
|
||||
# Fixed to the top of the hour, and IDs derived from names/times, so repeated runs see the same
|
||||
# jobs and sessions, like a real server
|
||||
$now = Get-Date -Minute 0 -Second 0 -Millisecond 0
|
||||
function New-StableGuid([string]$Seed) {
|
||||
[guid]::new([Security.Cryptography.MD5]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($Seed)))
|
||||
}
|
||||
$notSet = Get-Date '1900-01-01'
|
||||
|
||||
function New-MockJob($Name, $Type, $NextRun, [bool]$Enabled = $true) {
|
||||
[pscustomobject]@{
|
||||
Id = New-StableGuid $Name; Name = $Name; JobType = $Type
|
||||
IsScheduleEnabled = $Enabled
|
||||
ScheduleOptions = [pscustomobject]@{ NextRun = $NextRun }
|
||||
}
|
||||
}
|
||||
|
||||
function New-MockSession($Job, $Start, $End, $State, $Result, $Reason = '', $Tasks = @(), $LogRecords = @(), $Size = 0) {
|
||||
$s = [pscustomobject]@{
|
||||
Id = New-StableGuid "$($Job.Name)|$($Start.Ticks)"; JobId = $Job.Id; JobName = $Job.Name
|
||||
CreationTime = $Start; EndTime = $End; State = $State; Result = $Result
|
||||
IsRetryMode = $false
|
||||
Info = [pscustomobject]@{ Reason = $Reason }
|
||||
BackupStats = [pscustomobject]@{ BackupSize = $Size; DataSize = $Size * 3 }
|
||||
Logger = [pscustomobject]@{ Records = $LogRecords }
|
||||
Tasks = $Tasks
|
||||
}
|
||||
$s.Logger | Add-Member ScriptMethod GetLog { [pscustomobject]@{ UpdatedRecords = $this.Records } }
|
||||
$s | Add-Member ScriptMethod GetTaskSessions { $this.Tasks }
|
||||
$s
|
||||
}
|
||||
|
||||
function New-MockTask($Name, $Status, $Reason = '') {
|
||||
[pscustomobject]@{ Name = $Name; Status = $Status; Info = [pscustomobject]@{ Reason = $Reason } }
|
||||
}
|
||||
|
||||
$global:MockJobs = @(
|
||||
(New-MockJob 'File Server – Nightly' 'Backup' $now.Date.AddDays(1).AddHours(22)),
|
||||
# Older VBR versions return NextRun as a culture-formatted string
|
||||
(New-MockJob 'SQL01 – Logs' 'Backup' $now.AddMinutes(40).ToString()),
|
||||
(New-MockJob 'Offsite Copy' 'BackupSync' $null),
|
||||
(New-MockJob 'Old Exchange' 'Backup' $now.AddDays(1) $false),
|
||||
# VBR 12+ backup copy job (immediate mode, no schedule): the "policy" job Get-VBRJob returns
|
||||
(New-MockJob 'Cloud Copy' 'SimpleBackupCopyPolicy' $null)
|
||||
)
|
||||
$fs, $sql, $copy, $old, $cloud = $global:MockJobs
|
||||
|
||||
# Its hidden worker job, one per source job. Get-VBRJob doesn't return it, and Get-VBRBackupSession
|
||||
# returns only the worker's sessions. Shape taken from a real VBR 13 server.
|
||||
$cloudWorker = [pscustomobject]@{
|
||||
Id = New-StableGuid 'Cloud Copy\File Server – Nightly'; Name = 'Cloud Copy\File Server – Nightly'
|
||||
JobType = 'SimpleBackupCopyWorker'; PolicyJobId = $cloud.Id
|
||||
}
|
||||
function New-MockWorkerSession($Start, $End, $Result) {
|
||||
$s = New-MockSession $cloudWorker $Start $End 'Stopped' $Result -Size 40GB
|
||||
$s | Add-Member ScriptMethod GetJob { $cloudWorker }
|
||||
$s
|
||||
}
|
||||
|
||||
$global:MockSessions = @(
|
||||
(New-MockSession $fs $now.AddDays(-1).AddHours(-2) $now.AddDays(-1).AddHours(-1) 'Stopped' 'Success' -Size 128GB -Tasks @(
|
||||
(New-MockTask 'FS01' 'Success'), (New-MockTask 'FS02' 'Success'))),
|
||||
(New-MockSession $fs $now.AddMinutes(-20) $notSet 'Working' 'None' -Tasks @((New-MockTask 'FS01' 'InProgress'))),
|
||||
(New-MockSession $sql $now.AddHours(-1) $now.AddMinutes(-55) 'Stopped' 'Warning' -Size 2GB `
|
||||
-Tasks @((New-MockTask 'SQL01' 'Warning' 'Failed to truncate transaction logs for database "ERP_PROD"'))),
|
||||
(New-MockSession $copy $now.AddHours(-5) $now.AddHours(-4) 'Stopped' 'Failed' 'Error: Unable to connect to repository NAS02' `
|
||||
-LogRecords @(
|
||||
[pscustomobject]@{ Title = 'Preparing backup chain'; Status = 'ESucceeded' },
|
||||
[pscustomobject]@{ Title = 'Network path not found (0x80070035)'; Status = 'EFailed' })),
|
||||
(New-MockSession $old $now.AddDays(-10) $now.AddDays(-10).AddHours(1) 'Stopped' 'Success'), # outside 48h lookback
|
||||
(New-MockWorkerSession $now.AddDays(-1).AddMinutes(-55) $now.AddDays(-1).AddMinutes(-30) 'Success'),
|
||||
(New-MockWorkerSession $now.AddMinutes(-55) $now.AddMinutes(-35) 'Warning'),
|
||||
# A session from a job the collector doesn't know (e.g. deleted): must be ignored, not crash
|
||||
(New-MockSession ([pscustomobject]@{ Id = New-StableGuid 'gone'; Name = 'Deleted job' }) $now.AddHours(-2) $now.AddHours(-1) 'Stopped' 'Success')
|
||||
)
|
||||
|
||||
function Get-VBRJob { [CmdletBinding()] param() $global:MockJobs }
|
||||
function Get-VBRBackupSession { [CmdletBinding()] param() $global:MockSessions }
|
||||
@@ -1,102 +1,81 @@
|
||||
# Backup-Dashboard
|
||||
# Veeam Backup & Replication collector
|
||||
|
||||
Proof of concept backup dashboard, intended to display backup jobs from multiple sources.
|
||||
A PowerShell script that runs on each client's VBR server and reports job results to the dashboard.
|
||||
It covers jobs returned by `Get-VBRJob`: VM backup, replication and backup copy jobs.
|
||||
|
||||
```
|
||||
collectors ──POST /api/v1/ingest (API key)──▶ ┌──────────────┐
|
||||
│ FastAPI + │
|
||||
browser ──nginx (basic auth, TLS)──▶ /api/* ─▶│ SQLite │
|
||||
└─ static files: public/ └──────────────┘
|
||||
```
|
||||
## Install (on the VBR server, as an administrator)
|
||||
|
||||
- `public/`: static frontend (no build step). Add `?mock=1` to the URL to use built-in sample data.
|
||||
- `backend/`: FastAPI API backed by a single SQLite file.
|
||||
- `deploy/`: nginx site and systemd unit.
|
||||
- `collectors/`: scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md)).
|
||||
|
||||
## Data model
|
||||
|
||||
- **Tenant** (company) → **Job** → **Run**.
|
||||
- A job is keyed by `(tenant, source_system, job.external_id)`, so renaming a job at the source updates it instead of creating a duplicate.
|
||||
- A run has `status` (`running` | `success` | `warning` | `failed`), `started_at` and `finished_at`.
|
||||
The dashboard's **Last Run** is the most recent `finished_at`. While a job is running it shows
|
||||
**Running**, and Last Run still shows when the previous run finished.
|
||||
- Runs older than `BACKUP_DASH_RETENTION_DAYS` (default 90, minimum 30) are purged every 6 hours.
|
||||
|
||||
## Reporting results (collectors)
|
||||
|
||||
Each collector (a script on a Veeam server, a webhook relay, etc.) gets its own API key:
|
||||
Requires PowerShell 7 installed machine-wide (`winget install Microsoft.PowerShell` or the MSI),
|
||||
which VBR 13's Veeam module expects. The Microsoft Store version won't work, because the SYSTEM
|
||||
account can't run it. The installer checks for this before changing anything.
|
||||
|
||||
1. On the dashboard host, create a key for this server:
|
||||
```bash
|
||||
cd backend && python manage.py create-key veeam-acme
|
||||
python manage.py create-key veeam-acme
|
||||
```
|
||||
2. Copy this folder to the VBR server and run, from PowerShell 7 (`pwsh`):
|
||||
```powershell
|
||||
.\Install-VeeamCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp"
|
||||
```
|
||||
It prompts for the key, then:
|
||||
- writes `C:\ProgramData\BackupDashboard\veeam-collector.json` (readable only by SYSTEM and Administrators)
|
||||
- copies the script to `C:\Program Files\BackupDashboard\`
|
||||
- registers the scheduled task **Backup Dashboard - Veeam collector** (SYSTEM, every 15 minutes,
|
||||
run by `C:\Program Files\PowerShell\7\pwsh.exe`; use `-PwshPath` if it's installed elsewhere)
|
||||
3. Preview the payloads, then backfill 30 days so the history view is populated straight away
|
||||
(from `pwsh`):
|
||||
```powershell
|
||||
& 'C:\Program Files\BackupDashboard\Send-VeeamBackupStatus.ps1' -DryRun
|
||||
& 'C:\Program Files\BackupDashboard\Send-VeeamBackupStatus.ps1' -LookbackHours 720
|
||||
```
|
||||
|
||||
Then it POSTs one report per run. Send `running` when a run starts (optional) and the final status
|
||||
when it ends. Reports with the same `started_at` (or `run.external_id`) update the same run, and
|
||||
retries are safe.
|
||||
Re-run the installer to change settings or update the script. To remove the collector,
|
||||
delete the scheduled task and both `BackupDashboard` folders.
|
||||
|
||||
```bash
|
||||
curl -X POST https://backups.example.com/api/v1/ingest \
|
||||
-H "Authorization: Bearer bdk_..." -H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"tenant": "Acme Corp",
|
||||
"source_system": "Veeam B&R",
|
||||
"job": { "external_id": "4f1c…", "name": "File Server – Nightly", "next_run_at": "2026-10-05T22:00:00-04:00" },
|
||||
"run": {
|
||||
"status": "success",
|
||||
"started_at": "2026-10-04T22:00:05-04:00",
|
||||
"finished_at": "2026-10-04T22:47:12-04:00",
|
||||
"message": "Job completed successfully.",
|
||||
"size_bytes": 128849018880,
|
||||
"details": { "repository": "REPO-01" }
|
||||
}
|
||||
}'
|
||||
```
|
||||
## What it sends
|
||||
|
||||
- All timestamps must include a timezone offset (`Get-Date -Format o` in PowerShell does this).
|
||||
- `job.next_run_at` controls the **Overdue** flag. If you omit it, the stored value is kept. If you send `null`, it's cleared.
|
||||
- `run` can be omitted to only register or update a job.
|
||||
- Unknown fields are rejected (HTTP 422), so typos in collector scripts fail loudly.
|
||||
|
||||
## API (read)
|
||||
|
||||
| Endpoint | Returns |
|
||||
| Dashboard | Veeam |
|
||||
|---|---|
|
||||
| `GET /api/v1/jobs` | All jobs with current status, `running_since`, `last_run_at`, `next_run_at` |
|
||||
| `GET /api/v1/jobs/{id}` | One job, plus the last finished run's duration, size, message and details |
|
||||
| `GET /api/v1/jobs/{id}/runs?days=30` | Run history (no messages), newest first |
|
||||
| `GET /api/v1/runs/{id}` | One run including message and details |
|
||||
| `GET /api/v1/health` | `{"status": "ok"}` |
|
||||
| Company | `Tenant` from the config |
|
||||
| Job | Job name. The job's GUID is the ID, so renaming a job in Veeam doesn't duplicate it |
|
||||
| Status | Session result (Success / Warning / Failed). Any state other than Stopped is **Running**. A session that stopped with no result (e.g. cancelled) is reported as **Warning** |
|
||||
| Started / finished | Session `CreationTime` / `EndTime` |
|
||||
| Next scheduled | Job's next run. Sent as empty for disabled jobs and chained ("run after") jobs, so those are never flagged Overdue |
|
||||
| Message | Session reason, each VM or object that didn't succeed, and warning/error lines from the session log |
|
||||
| Size | `BackupStats.BackupSize` |
|
||||
|
||||
The OpenAPI schema is at `/api/v1/openapi.json`.
|
||||
**Backup copy jobs (VBR 12+):** Veeam records their sessions under hidden per-source "worker" jobs.
|
||||
The collector files each worker session under its copy job (via the worker's `PolicyJobId`), so a
|
||||
copy job gets one run per source job it copies, with the source in `copied_from`.
|
||||
|
||||
## Authentication
|
||||
Each run covers the last 48 hours (`LookbackHours`), so a few missed runs (server rebooting, API
|
||||
down) fill in automatically. Sessions already delivered are recorded in
|
||||
`veeam-collector.state.json`. Running sessions are re-sent until they finish.
|
||||
|
||||
- **Dashboard users:** nginx basic auth over HTTPS (`deploy/nginx.conf`). Every user can see every tenant.
|
||||
*Planned:* Microsoft 365 sign-in through [oauth2-proxy](https://oauth2-proxy.github.io/oauth2-proxy/)
|
||||
with the Entra ID provider and nginx `auth_request`. That's an nginx-only change. The frontend
|
||||
already tells the user to reload when a session expires (HTTP 401).
|
||||
- **Collectors:** per-collector bearer keys, checked by the app. Only a SHA-256 hash is stored.
|
||||
To revoke one, run `python manage.py revoke-key NAME`.
|
||||
## Troubleshooting
|
||||
|
||||
## Development
|
||||
- Log: `C:\ProgramData\BackupDashboard\veeam-collector.log` (rotates at 5 MB).
|
||||
- Exit codes: `0` OK, `1` some jobs failed to send (they're retried next run), `2` setup failed
|
||||
(config, Veeam module, or Veeam connection).
|
||||
- `API key rejected`: the key was revoked or mistyped. Re-run the installer with a new key.
|
||||
- A job shows **Unknown** on the dashboard: the collector sent the job but matched no sessions to it.
|
||||
Run `.\Get-VeeamCollectorDiagnostics.ps1` (read-only). It lists sessions whose job ID matches no
|
||||
job, plus the type and link properties of the jobs left without sessions.
|
||||
- `Get-VBRBackupSession` loads every session in the VBR database. On a server with years of history,
|
||||
this can make each run take a minute or more. The task's 30-minute limit and "don't start if
|
||||
already running" setting keep that safe.
|
||||
|
||||
```bash
|
||||
cd backend
|
||||
python -m venv .venv && . .venv/bin/activate
|
||||
pip install -r requirements.txt -r requirements-dev.txt
|
||||
pytest
|
||||
python manage.py seed-demo # 30 days of fake history
|
||||
BACKUP_DASH_SERVE_STATIC=1 uvicorn --factory app.main:create_app --reload
|
||||
# → http://localhost:8000
|
||||
## Testing without Veeam
|
||||
|
||||
`tests/MockVeeam.ps1` defines stand-in `Get-VBRJob` / `Get-VBRBackupSession` cmdlets with a mix of
|
||||
success, warning, failed, running and disabled jobs:
|
||||
|
||||
```powershell
|
||||
. ./tests/MockVeeam.ps1
|
||||
./Send-VeeamBackupStatus.ps1 -ConfigPath ./my-test-config.json -DryRun
|
||||
```
|
||||
|
||||
## Deployment (Linux, nginx)
|
||||
## Not covered yet
|
||||
|
||||
1. Copy the repo to `/opt/backup-dashboard`, create a venv at `/opt/backup-dashboard/venv`, and run `pip install -r backend/requirements.txt`.
|
||||
2. Create a `backupdash` system user, then install `deploy/backup-dashboard-api.service` into `/etc/systemd/system/` and enable it.
|
||||
3. Copy `public/` to `/var/www/backup-dashboard/public`, install `deploy/nginx.conf`, get a certificate with certbot, and create users with `htpasswd -B`.
|
||||
4. Run the `manage.py` commands as the service user with `BACKUP_DASH_DB=/var/lib/backup-dashboard/backup-dashboard.db`.
|
||||
|
||||
Back up the SQLite file with `sqlite3 backup-dashboard.db ".backup copy.db"`. Don't use a plain `cp`
|
||||
while the service is running.
|
||||
- Veeam Agent jobs managed by VBR (`Get-VBRComputerBackupJob`), tape jobs, and SureBackup.
|
||||
- Veeam Backup for Microsoft 365, which has its own PowerShell module.
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Reports Veeam Backup & Replication job results to the Backup Dashboard.
|
||||
|
||||
.DESCRIPTION
|
||||
Runs on the VBR server, normally as a SYSTEM scheduled task every 15 minutes
|
||||
(see Install-VeeamCollector.ps1). For every backup, replica and backup copy job it sends:
|
||||
- the job's next scheduled run (drives the dashboard's Overdue flag), and
|
||||
- every session from the last LookbackHours that the dashboard hasn't already received
|
||||
in its final state. Running sessions are re-sent each time until they finish.
|
||||
|
||||
Already-reported sessions are tracked in a state file, and the ingest API is idempotent,
|
||||
so a failed or repeated run is harmless: anything not confirmed is retried next time.
|
||||
|
||||
Compatible with Windows PowerShell 5.1 and PowerShell 7.
|
||||
|
||||
.PARAMETER ConfigPath
|
||||
JSON config with ApiUrl, ApiKey and Tenant (optional: SourceSystem, LookbackHours).
|
||||
|
||||
.PARAMETER LookbackHours
|
||||
Overrides the config value. Use 720 once to backfill 30 days of history.
|
||||
|
||||
.PARAMETER DryRun
|
||||
Print the payloads instead of sending them. Doesn't touch the state or log files.
|
||||
|
||||
.EXAMPLE
|
||||
.\Send-VeeamBackupStatus.ps1 -DryRun
|
||||
.EXAMPLE
|
||||
.\Send-VeeamBackupStatus.ps1 -LookbackHours 720
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\veeam-collector.json'),
|
||||
[int]$LookbackHours = 0,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$DataDir = Split-Path -Parent $ConfigPath
|
||||
$LogPath = Join-Path $DataDir 'veeam-collector.log'
|
||||
$StatePath = Join-Path $DataDir 'veeam-collector.state.json'
|
||||
$MaxLogBytes = 5MB
|
||||
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||
|
||||
# ---------- Helpers ----------
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message, [string]$Level = 'INFO')
|
||||
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
|
||||
Write-Host $line
|
||||
if ($DryRun) { return }
|
||||
if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) {
|
||||
Move-Item $LogPath "$LogPath.1" -Force
|
||||
}
|
||||
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
||||
}
|
||||
|
||||
function Get-ErrorText($ErrorRecord) {
|
||||
# Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
|
||||
if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
|
||||
return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
|
||||
}
|
||||
return $ErrorRecord.Exception.Message
|
||||
}
|
||||
|
||||
# The API needs an explicit UTC offset. Veeam returns local times, often with Kind=Unspecified,
|
||||
# which DateTimeOffset treats as local, exactly what we want.
|
||||
function ConvertTo-IsoTime($Value) {
|
||||
if ($null -eq $Value) { return $null }
|
||||
$dt = [datetime]$Value
|
||||
if ($dt.Year -le 1900) { return $null } # Veeam's "not set" value (e.g. EndTime while running)
|
||||
return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant)
|
||||
}
|
||||
|
||||
function Import-Veeam {
|
||||
if (Get-Command Get-VBRJob -ErrorAction SilentlyContinue) { return } # already loaded
|
||||
if (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell) {
|
||||
Import-Module Veeam.Backup.PowerShell -WarningAction SilentlyContinue # VBR 11+
|
||||
} else {
|
||||
Add-PSSnapin VeeamPSSnapin # VBR 10 and earlier
|
||||
}
|
||||
}
|
||||
|
||||
function Get-NextRun($Job) {
|
||||
try {
|
||||
if (-not $Job.IsScheduleEnabled) { return $null }
|
||||
$raw = $Job.ScheduleOptions.NextRun
|
||||
if (-not $raw) { return $null } # e.g. chained jobs that run "after" another job
|
||||
if ($raw -is [datetime]) { return ConvertTo-IsoTime $raw }
|
||||
# Older versions return a string formatted with the server's culture, so parse with it
|
||||
$parsed = [datetime]::MinValue
|
||||
if ([datetime]::TryParse([string]$raw, [Globalization.CultureInfo]::CurrentCulture,
|
||||
[Globalization.DateTimeStyles]::None, [ref]$parsed)) {
|
||||
return ConvertTo-IsoTime $parsed
|
||||
}
|
||||
Write-Log "Couldn't parse next run '$raw' for job '$($Job.Name)'" 'WARN'
|
||||
} catch {
|
||||
Write-Log "Couldn't read schedule for job '$($Job.Name)': $(Get-ErrorText $_)" 'WARN'
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-RunStatus($Session) {
|
||||
if ([string]$Session.State -ne 'Stopped') { return 'running' }
|
||||
switch ([string]$Session.Result) {
|
||||
'Success' { return 'success' }
|
||||
'Warning' { return 'warning' }
|
||||
'Failed' { return 'failed' }
|
||||
# Stopped without a result (e.g. cancelled): surface it rather than hide it
|
||||
default { return 'warning' }
|
||||
}
|
||||
}
|
||||
|
||||
# Builds a readable summary: the session's reason, any task (VM/object) that didn't succeed,
|
||||
# and warning/error lines from the session log.
|
||||
function Get-RunMessage($Session, [string]$Status, $Tasks) {
|
||||
$lines = New-Object 'System.Collections.Generic.List[string]'
|
||||
try {
|
||||
$reason = [string]$Session.Info.Reason
|
||||
if ($reason.Trim()) { $lines.Add($reason.Trim()) }
|
||||
} catch { }
|
||||
|
||||
foreach ($task in $Tasks) {
|
||||
$taskStatus = [string]$task.Status
|
||||
if ($taskStatus -and $taskStatus -ne 'Success' -and $taskStatus -ne 'InProgress' -and $taskStatus -ne 'Pending') {
|
||||
$taskReason = ''
|
||||
try { $taskReason = ([string]$task.Info.Reason).Trim() } catch { }
|
||||
$line = "$($task.Name): $taskStatus"
|
||||
if ($taskReason) { $line += " - $taskReason" }
|
||||
$lines.Add($line)
|
||||
}
|
||||
}
|
||||
|
||||
if ($Status -eq 'warning' -or $Status -eq 'failed') {
|
||||
try {
|
||||
foreach ($record in $Session.Logger.GetLog().UpdatedRecords) {
|
||||
$recordStatus = [string]$record.Status
|
||||
if ($recordStatus -eq 'EWarning' -or $recordStatus -eq 'EFailed') {
|
||||
$title = ([string]$record.Title).Trim()
|
||||
if ($title -and -not $lines.Contains($title)) { $lines.Add($title) }
|
||||
}
|
||||
}
|
||||
} catch { }
|
||||
}
|
||||
|
||||
if ($lines.Count -eq 0) {
|
||||
switch ($Status) {
|
||||
'success' { return 'Job completed successfully.' }
|
||||
'running' { return '' }
|
||||
default { return "Job finished with result '$($Session.Result)'." }
|
||||
}
|
||||
}
|
||||
return ($lines -join "`n")
|
||||
}
|
||||
|
||||
# VBR 12+ backup copy jobs: Get-VBRJob returns the copy (policy) job, but Get-VBRBackupSession
|
||||
# returns only the sessions of its hidden worker jobs, one per source job, never the policy's own.
|
||||
# Each worker job records its policy in PolicyJobId, so those sessions can be filed under it.
|
||||
function Get-PolicyJobId($Session, [hashtable]$KnownJobIds) {
|
||||
try {
|
||||
$policyId = [string]$Session.GetJob().PolicyJobId
|
||||
if ($policyId -and $KnownJobIds.ContainsKey($policyId)) { return $policyId }
|
||||
} catch { }
|
||||
return ''
|
||||
}
|
||||
|
||||
function ConvertTo-Run($Session, $Job) {
|
||||
$status = Get-RunStatus $Session
|
||||
$tasks = @()
|
||||
try { $tasks = @($Session.GetTaskSessions()) } catch { }
|
||||
|
||||
$details = [ordered]@{
|
||||
job_type = [string]$Job.JobType
|
||||
vbr_server = $env:COMPUTERNAME
|
||||
objects = $tasks.Count
|
||||
}
|
||||
$failedTasks = @($tasks | Where-Object { [string]$_.Status -eq 'Failed' }).Count
|
||||
$warningTasks = @($tasks | Where-Object { [string]$_.Status -eq 'Warning' }).Count
|
||||
if ($failedTasks) { $details.objects_failed = $failedTasks }
|
||||
if ($warningTasks) { $details.objects_warning = $warningTasks }
|
||||
try { if ($Session.IsRetryMode) { $details.retry = $true } } catch { }
|
||||
# Worker session of a backup copy job: say which source job it copied
|
||||
# (worker sessions are named "<copy job>\<source job>")
|
||||
if ($Session.JobId.ToString() -ne $Job.Id.ToString()) {
|
||||
$details.copied_from = ([string]$Session.JobName -split '\\')[-1]
|
||||
}
|
||||
|
||||
$run = [ordered]@{
|
||||
external_id = $Session.Id.ToString()
|
||||
status = $status
|
||||
started_at = ConvertTo-IsoTime $Session.CreationTime
|
||||
message = Get-RunMessage $Session $status $tasks
|
||||
details = $details
|
||||
}
|
||||
if ($status -ne 'running') {
|
||||
$finished = ConvertTo-IsoTime $Session.EndTime
|
||||
if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time
|
||||
try {
|
||||
$size = [long]$Session.BackupStats.BackupSize
|
||||
if ($size -gt 0) { $run.size_bytes = $size }
|
||||
$dataSize = [long]$Session.BackupStats.DataSize
|
||||
if ($dataSize -gt 0) { $details.data_size_bytes = $dataSize }
|
||||
} catch { }
|
||||
}
|
||||
return $run
|
||||
}
|
||||
|
||||
function Send-Report($Body) {
|
||||
$json = $Body | ConvertTo-Json -Depth 6 -Compress
|
||||
if ($DryRun) {
|
||||
Write-Host $json
|
||||
return
|
||||
}
|
||||
# Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1,
|
||||
# mangling non-ASCII job names (e.g. en dashes).
|
||||
$params = @{
|
||||
Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
|
||||
Method = 'Post'
|
||||
Headers = @{ Authorization = "Bearer $($Config.ApiKey)" }
|
||||
ContentType = 'application/json; charset=utf-8'
|
||||
Body = [Text.Encoding]::UTF8.GetBytes($json)
|
||||
TimeoutSec = 30
|
||||
}
|
||||
Invoke-RestMethod @params | Out-Null
|
||||
}
|
||||
|
||||
# ---------- Main ----------
|
||||
|
||||
try {
|
||||
$Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
|
||||
foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') {
|
||||
if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" }
|
||||
}
|
||||
$SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Veeam B&R' }
|
||||
if ($LookbackHours -le 0) {
|
||||
$LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 }
|
||||
}
|
||||
|
||||
# Windows PowerShell 5.1 may default to TLS 1.0
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
||||
|
||||
Import-Veeam
|
||||
|
||||
# Session ID -> start time (Unix seconds), for sessions already delivered in their final state.
|
||||
# Numbers rather than ISO strings: PowerShell 7's ConvertFrom-Json turns date strings into
|
||||
# DateTimes and loses the offset.
|
||||
$sent = @{}
|
||||
if (Test-Path $StatePath) {
|
||||
$saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json
|
||||
if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } }
|
||||
}
|
||||
|
||||
$since = (Get-Date).AddHours(-$LookbackHours)
|
||||
$jobs = @(Get-VBRJob -WarningAction SilentlyContinue)
|
||||
|
||||
$jobIds = @{}
|
||||
foreach ($j in $jobs) { $jobIds[$j.Id.ToString()] = $true }
|
||||
|
||||
# Get-VBRBackupSession returns every session in the database, so fetch once and bucket by job
|
||||
$sessionsByJob = @{}
|
||||
$policyByWorker = @{} # worker job ID -> policy job ID ('' if it isn't a worker of a listed job)
|
||||
foreach ($s in Get-VBRBackupSession -WarningAction SilentlyContinue) {
|
||||
if ($s.CreationTime -lt $since) { continue }
|
||||
$key = $s.JobId.ToString()
|
||||
if (-not $jobIds.ContainsKey($key)) {
|
||||
if (-not $policyByWorker.ContainsKey($key)) { $policyByWorker[$key] = Get-PolicyJobId $s $jobIds }
|
||||
if (-not $policyByWorker[$key]) { continue }
|
||||
$key = $policyByWorker[$key]
|
||||
}
|
||||
if (-not $sessionsByJob.ContainsKey($key)) { $sessionsByJob[$key] = New-Object 'System.Collections.Generic.List[object]' }
|
||||
$sessionsByJob[$key].Add($s)
|
||||
}
|
||||
} catch {
|
||||
Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR'
|
||||
exit 2
|
||||
}
|
||||
|
||||
$failures = 0
|
||||
$runsSent = 0
|
||||
foreach ($job in $jobs) {
|
||||
$jobInfo = [ordered]@{
|
||||
external_id = $job.Id.ToString()
|
||||
name = $job.Name
|
||||
# Always sent (null when unscheduled/disabled) so the dashboard's value tracks Veeam's
|
||||
next_run_at = Get-NextRun $job
|
||||
}
|
||||
$pending = @()
|
||||
if ($sessionsByJob.ContainsKey($jobInfo.external_id)) {
|
||||
$pending = @($sessionsByJob[$jobInfo.external_id] |
|
||||
Where-Object { -not $sent.ContainsKey($_.Id.ToString()) } |
|
||||
Sort-Object CreationTime)
|
||||
}
|
||||
|
||||
try {
|
||||
if ($pending.Count -eq 0) {
|
||||
# Keep next_run_at fresh even when nothing new ran
|
||||
Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo })
|
||||
}
|
||||
foreach ($session in $pending) {
|
||||
$run = ConvertTo-Run $session $job
|
||||
Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run })
|
||||
$runsSent++
|
||||
if ($run.status -ne 'running') {
|
||||
$sent[$run.external_id] = ([DateTimeOffset][datetime]$session.CreationTime).ToUnixTimeSeconds()
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
$failures++
|
||||
Write-Log "Failed to report job '$($job.Name)': $(Get-ErrorText $_)" 'ERROR'
|
||||
$httpStatus = 0
|
||||
try { $httpStatus = [int]$_.Exception.Response.StatusCode } catch { }
|
||||
if ($httpStatus -eq 401) {
|
||||
Write-Log 'API key rejected; check ApiKey in the config. Stopping.' 'ERROR'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $DryRun) {
|
||||
# Forget sessions that have aged out of the lookback window (plus a day of slack)
|
||||
$cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds()
|
||||
$keep = [ordered]@{}
|
||||
foreach ($id in $sent.Keys) {
|
||||
if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] }
|
||||
}
|
||||
try {
|
||||
$keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8
|
||||
} catch {
|
||||
Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
|
||||
}
|
||||
}
|
||||
|
||||
Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $jobs.Count, $runsSent, $failures)
|
||||
if ($failures) { exit 1 }
|
||||
Reference in New Issue
Block a user