kyle/Backup-Dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move code shared by the collectors into one module

Browse Source 
The Veeam, MARS, WSB and VB365 collectors each carried identical copies
of logging, ISO time conversion, property lookup, posting to the API,
config loading, TLS setup, the sent-runs state file and the 401 check,
and their installers repeated the same key prompt, locked-down config,
file copy and scheduled-task registration. These now live in
collectors/common/BackupDashboard.Collector.psm1:

- Initialize-Collector / Test-RunSent / Send-JobReport /
  Test-AuthRejected / Complete-Collector for collector runs
- Write-Log, Get-ErrorText, ConvertTo-IsoTime, Get-FirstValue
- Install-CollectorTask for the shared half of each installer

Collectors find the module next to themselves (installed) or in
..\common (repo); installers copy it into Program Files beside the
collector. The diagnostic scripts stay standalone single files.

No behavior change: a harness ran each collector against its mock
(12 scenarios: dry run, two real runs against a stub API, state file,
logs, exit codes, rejected key, en-GB, backfill) before and after, and
all 84 outputs are identical. Installers, run with stand-ins for the
Windows-only commands, write identical configs and tasks; the only
differences are the module being copied and the summary line naming
the shell's path.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
kyle
2026-10-08 11:09:01 -06:00
co-authored by Claude Opus 5.5
parent 118a7f978d
commit 8d0e6dca71
14 changed files with 416 additions and 633 deletions
Download Patch File Download Diff File Expand all files Collapse all files
collectors/vb365/Install-Vb365Collector.ps1
+14 -45
View File
@@ -37,9 +37,6 @@ param(
)
$ErrorActionPreference = 'Stop'
$TaskName = 'Backup Dashboard - VB365 collector'
$DataDir = Join-Path $env:ProgramData 'BackupDashboard'
$ConfigPath = Join-Path $DataDir 'vb365-collector.json'
$Shell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe'
if ($PwshPath) {
if (-not (Test-Path $PwshPath)) { throw "PowerShell 7 not found at '$PwshPath'." }
@@ -55,51 +52,23 @@ if (-not $Tenant -and $OrganizationTenants.Count -eq 0) {
Write-Warning 'No -Tenant or -OrganizationTenants: jobs will be listed under their Microsoft 365 organization name.'
}
if ($ApiUrl -notmatch '^https://') {
Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.'
# Shared installer code: in the repo it's in ..\common, so copy the whole collectors folder
$sharedModule = @(
(Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'),
(Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1')
) | Where-Object { Test-Path $_ } | Select-Object -First 1
if (-not $sharedModule) {
throw "BackupDashboard.Collector.psm1 not found. Copy the whole collectors folder, including common."
}
Import-Module $sharedModule -Force
# Read the key as a SecureString so it isn't echoed to the console or kept in shell history
$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key <name>)'
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." }
# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows)
New-Item -ItemType Directory -Path $DataDir -Force | Out-Null
& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
if ($LASTEXITCODE) { throw "icacls failed on $DataDir" }
$config = [ordered]@{
ApiUrl = $ApiUrl.TrimEnd('/')
ApiKey = $apiKey
$settings = [ordered]@{
SourceSystem = $SourceSystem
LookbackHours = $LookbackHours
OrganizationTenants = $OrganizationTenants
}
if ($Tenant) { $config.Tenant = $Tenant }
$config | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
$scriptPath = Join-Path $InstallDir 'Send-Vb365BackupStatus.ps1'
Copy-Item -Path (Join-Path $PSScriptRoot 'Send-Vb365BackupStatus.ps1') -Destination $scriptPath -Force
Unblock-File -Path $scriptPath
$action = New-ScheduledTaskAction -Execute $Shell `
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`""
$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) `
-RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes)
$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest
$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) `
-MultipleInstances IgnoreNew -StartWhenAvailable
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal `
-Settings $settings -Description 'Sends Veeam Backup for Microsoft 365 results to the Backup Dashboard' -Force | Out-Null
Write-Host "Installed. Config: $ConfigPath"
Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using $Shell."
Write-Host ''
Write-Host 'Next steps:'
Write-Host " 1. Preview what will be sent: & '$Shell' -File '$scriptPath' -DryRun"
Write-Host " 2. Backfill 30 days of history: & '$Shell' -File '$scriptPath' -LookbackHours 720"
Write-Host " 3. Check the log: Get-Content '$DataDir\vb365-collector.log' -Tail 20"
if ($Tenant) { $settings.Tenant = $Tenant }
Install-CollectorTask -Name 'vb365' -ScriptPath (Join-Path $PSScriptRoot 'Send-Vb365BackupStatus.ps1') `
-TaskName 'Backup Dashboard - VB365 collector' `
-TaskDescription 'Sends Veeam Backup for Microsoft 365 results to the Backup Dashboard' `
-Shell $Shell -ApiUrl $ApiUrl -IntervalMinutes $IntervalMinutes -InstallDir $InstallDir -Settings $settings
collectors/vb365/README.md
+4 -2
View File
@@ -27,7 +27,8 @@ one with `python manage.py delete-job <id>`.
```bash
python manage.py create-key vb365-01
```
2. Copy this folder to the server and run one of:
2. Copy the whole `collectors` folder to the server (the installer needs `common`
as well as this folder), then from `collectors\vb365` run one of:
```powershell
# VB365 server for one client
.\Install-Vb365Collector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp"
@@ -52,7 +53,8 @@ one with `python manage.py delete-job <id>`.
```
It can be installed alongside the other collectors: it uses its own config, state, log and task.
Re-run the installer to change settings (including the organization mapping) or update the script.
Re-run the installer to change settings (including the organization mapping) or update the script. It also installs
`BackupDashboard.Collector.psm1`, the code shared by all collectors, next to the script.
## What it sends
collectors/vb365/Send-Vb365BackupStatus.ps1
+19 -113
View File
@@ -41,61 +41,21 @@ param(
$ErrorActionPreference = 'Stop'
$DataDir = Split-Path -Parent $ConfigPath
$LogPath = Join-Path $DataDir 'vb365-collector.log'
$StatePath = Join-Path $DataDir 'vb365-collector.state.json'
$MaxLogBytes = 5MB
# Shared collector code: installed next to this script; in the repo, in ..\common
$sharedModule = @(
(Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'),
(Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1')
) | Where-Object { Test-Path $_ } | Select-Object -First 1
if (-not $sharedModule) {
Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer."
exit 2
}
Import-Module $sharedModule -Force
$Invariant = [Globalization.CultureInfo]::InvariantCulture
# ---------- Helpers ----------
function Write-Log {
param([string]$Message, [string]$Level = 'INFO')
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
Write-Host $line
if ($DryRun) { return }
if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) {
Move-Item $LogPath "$LogPath.1" -Force
}
Add-Content -Path $LogPath -Value $line -Encoding UTF8
}
function Get-ErrorText($ErrorRecord) {
# Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
}
return $ErrorRecord.Exception.Message
}
# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a
# Utc time as +00:00.
function ConvertTo-IsoTime($Value) {
if ($null -eq $Value) { return $null }
$dt = [datetime]$Value
if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue and friends)
return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant)
}
# First property of $Object, out of $Names, that exists and isn't empty. The VB365 object model isn't
# fully documented, so reads go through this rather than assuming one spelling.
function Get-FirstValue($Object, [string[]]$Names) {
if ($null -eq $Object) { return $null }
foreach ($name in $Names) {
try {
$prop = $Object.PSObject.Properties[$name]
if (-not $prop) { continue }
$value = $prop.Value
# Check emptiness by type: casting to [string] would make a one-item list of objects ''
if ($null -eq $value) { continue }
if ($value -is [string] -and $value -eq '') { continue }
if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue }
return $value
} catch { }
}
return $null
}
function Import-Vb365 {
if (Get-Command Get-VBOJob -ErrorAction SilentlyContinue) { return } # already loaded
if (Get-Module -ListAvailable -Name Veeam.Archiver.PowerShell) {
@@ -245,48 +205,15 @@ function ConvertTo-Run($Session, $Job, [string]$JobKind) {
return $run
}
function Send-Report($Body) {
$json = $Body | ConvertTo-Json -Depth 6 -Compress
if ($DryRun) {
Write-Host $json
return
}
# Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1
$params = @{
Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
Method = 'Post'
Headers = @{ Authorization = "Bearer $($Config.ApiKey)" }
ContentType = 'application/json; charset=utf-8'
Body = [Text.Encoding]::UTF8.GetBytes($json)
TimeoutSec = 30
}
Invoke-RestMethod @params | Out-Null
}
# ---------- Main ----------
try {
$Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
foreach ($key in 'ApiUrl', 'ApiKey') {
if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" }
}
$Config = Initialize-Collector -Name 'vb365' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun -RequiredKeys 'ApiUrl', 'ApiKey'
$SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Veeam Backup for M365' }
if ($LookbackHours -le 0) {
$LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 }
}
# Windows PowerShell 5.1 may default to TLS 1.0
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
$LookbackHours = $Config.LookbackHours
Import-Vb365
# Session ID -> start time (Unix seconds), for sessions already delivered in their final state
$sent = @{}
if (Test-Path $StatePath) {
$saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json
if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } }
}
$since = (Get-Date).AddHours(-$LookbackHours)
# (job, kind, sessions) for backup jobs, plus backup copy jobs where the cmdlets exist
@@ -315,7 +242,6 @@ try {
}
$failures = 0
$runsSent = 0
foreach ($item in $work) {
$job = $item.Job
$tenant = Get-TenantForJob $job
@@ -326,47 +252,27 @@ foreach ($item in $work) {
next_run_at = Get-NextRun $job
}
$pending = @($item.Sessions |
Where-Object { $_ -and -not $sent.ContainsKey([string](Get-FirstValue $_ @('Id', 'SessionId'))) } |
Where-Object { $_ -and -not (Test-RunSent ([string](Get-FirstValue $_ @('Id', 'SessionId')))) } |
Sort-Object { Get-Time $_ 'CreationTime' })
try {
if ($pending.Count -eq 0) {
# Keep next_run_at fresh even when nothing new ran
Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo })
Send-JobReport -Tenant $tenant -SourceSystem $SourceSystem -Job $jobInfo
}
foreach ($session in $pending) {
$run = ConvertTo-Run $session $job $item.Kind
Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo; run = $run })
$runsSent++
if ($run.status -ne 'running') {
$sent[$run.external_id] = ([DateTimeOffset](Get-Time $session 'CreationTime')).ToUnixTimeSeconds()
}
Send-JobReport -Tenant $tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.external_id -StartedAt ((Get-Time $session 'CreationTime'))
}
} catch {
$failures++
Write-Log "Failed to report job '$($job.Name)': $(Get-ErrorText $_)" 'ERROR'
$httpStatus = 0
try { $httpStatus = [int]$_.Exception.Response.StatusCode } catch { }
if ($httpStatus -eq 401) {
if (Test-AuthRejected $_) {
Write-Log 'API key rejected; check ApiKey in the config. Stopping.' 'ERROR'
break
}
}
}
if (-not $DryRun) {
# Forget sessions that have aged out of the lookback window (plus a day of slack)
$cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds()
$keep = [ordered]@{}
foreach ($id in $sent.Keys) {
if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] }
}
try {
$keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8
} catch {
Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
}
}
Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $work.Count, $runsSent, $failures)
if ($failures) { exit 1 }
$exitCode = Complete-Collector -JobCount $work.Count -Failures $failures
if ($exitCode) { exit $exitCode }