Move code shared by the collectors into one module
The Veeam, MARS, WSB and VB365 collectors each carried identical copies of logging, ISO time conversion, property lookup, posting to the API, config loading, TLS setup, the sent-runs state file and the 401 check, and their installers repeated the same key prompt, locked-down config, file copy and scheduled-task registration. These now live in collectors/common/BackupDashboard.Collector.psm1: - Initialize-Collector / Test-RunSent / Send-JobReport / Test-AuthRejected / Complete-Collector for collector runs - Write-Log, Get-ErrorText, ConvertTo-IsoTime, Get-FirstValue - Install-CollectorTask for the shared half of each installer Collectors find the module next to themselves (installed) or in ..\common (repo); installers copy it into Program Files beside the collector. The diagnostic scripts stay standalone single files. No behavior change: a harness ran each collector against its mock (12 scenarios: dry run, two real runs against a stub API, state file, logs, exit codes, rejected key, en-GB, backfill) before and after, and all 84 outputs are identical. Installers, run with stand-ins for the Windows-only commands, write identical configs and tasks; the only differences are the module being copied and the summary line naming the shell's path. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -29,9 +29,6 @@ param(
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$TaskName = 'Backup Dashboard - WSB collector'
|
||||
$DataDir = Join-Path $env:ProgramData 'BackupDashboard'
|
||||
$ConfigPath = Join-Path $DataDir 'wsb-collector.json'
|
||||
$WindowsPowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe'
|
||||
|
||||
# Check Windows Server Backup is here before changing anything
|
||||
@@ -39,51 +36,19 @@ if (-not (Get-Module -ListAvailable -Name WindowsServerBackup)) {
|
||||
throw 'The WindowsServerBackup PowerShell module is not available. Install the Windows Server Backup feature (Install-WindowsFeature Windows-Server-Backup).'
|
||||
}
|
||||
|
||||
if ($ApiUrl -notmatch '^https://') {
|
||||
Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.'
|
||||
# Shared installer code: in the repo it's in ..\common, so copy the whole collectors folder
|
||||
$sharedModule = @(
|
||||
(Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'),
|
||||
(Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1')
|
||||
) | Where-Object { Test-Path $_ } | Select-Object -First 1
|
||||
if (-not $sharedModule) {
|
||||
throw "BackupDashboard.Collector.psm1 not found. Copy the whole collectors folder, including common."
|
||||
}
|
||||
Import-Module $sharedModule -Force
|
||||
|
||||
# Read the key as a SecureString so it isn't echoed to the console or kept in shell history
|
||||
$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key <name>)'
|
||||
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
||||
try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
||||
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
||||
if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." }
|
||||
|
||||
# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows)
|
||||
New-Item -ItemType Directory -Path $DataDir -Force | Out-Null
|
||||
& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
|
||||
if ($LASTEXITCODE) { throw "icacls failed on $DataDir" }
|
||||
|
||||
$config = [ordered]@{
|
||||
ApiUrl = $ApiUrl.TrimEnd('/')
|
||||
ApiKey = $apiKey
|
||||
Tenant = $Tenant
|
||||
SourceSystem = $SourceSystem
|
||||
LookbackHours = $LookbackHours
|
||||
}
|
||||
if ($JobName) { $config.JobName = $JobName }
|
||||
$config | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8
|
||||
|
||||
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
|
||||
$scriptPath = Join-Path $InstallDir 'Send-WsbBackupStatus.ps1'
|
||||
Copy-Item -Path (Join-Path $PSScriptRoot 'Send-WsbBackupStatus.ps1') -Destination $scriptPath -Force
|
||||
Unblock-File -Path $scriptPath
|
||||
|
||||
$action = New-ScheduledTaskAction -Execute $WindowsPowerShell `
|
||||
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`""
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) `
|
||||
-RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes)
|
||||
$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest
|
||||
$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) `
|
||||
-MultipleInstances IgnoreNew -StartWhenAvailable
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal `
|
||||
-Settings $settings -Description 'Sends Windows Server Backup results to the Backup Dashboard' -Force | Out-Null
|
||||
|
||||
Write-Host "Installed. Config: $ConfigPath"
|
||||
Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using Windows PowerShell."
|
||||
Write-Host ''
|
||||
Write-Host 'Next steps:'
|
||||
Write-Host " 1. Preview what will be sent: & '$WindowsPowerShell' -File '$scriptPath' -DryRun"
|
||||
Write-Host " 2. Backfill 30 days of history: & '$WindowsPowerShell' -File '$scriptPath' -LookbackHours 720"
|
||||
Write-Host " 3. Check the log: Get-Content '$DataDir\wsb-collector.log' -Tail 20"
|
||||
$settings = [ordered]@{ Tenant = $Tenant; SourceSystem = $SourceSystem; LookbackHours = $LookbackHours }
|
||||
if ($JobName) { $settings.JobName = $JobName }
|
||||
Install-CollectorTask -Name 'wsb' -ScriptPath (Join-Path $PSScriptRoot 'Send-WsbBackupStatus.ps1') `
|
||||
-TaskName 'Backup Dashboard - WSB collector' `
|
||||
-TaskDescription 'Sends Windows Server Backup results to the Backup Dashboard' `
|
||||
-Shell $WindowsPowerShell -ApiUrl $ApiUrl -IntervalMinutes $IntervalMinutes -InstallDir $InstallDir -Settings $settings
|
||||
|
||||
@@ -19,7 +19,8 @@ Requires the Windows Server Backup feature with its PowerShell module
|
||||
```bash
|
||||
python manage.py create-key wsb-acme-dc01
|
||||
```
|
||||
2. Copy this folder to the server and run:
|
||||
2. Copy the whole `collectors` folder to the server (the installer needs `common`
|
||||
as well as this folder), then from `collectors\wsb` run:
|
||||
```powershell
|
||||
.\Install-WsbCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp"
|
||||
```
|
||||
@@ -35,7 +36,8 @@ Requires the Windows Server Backup feature with its PowerShell module
|
||||
```
|
||||
|
||||
It can be installed alongside the other collectors: it uses its own config, state, log and task.
|
||||
Re-run the installer to change settings or update the script.
|
||||
Re-run the installer to change settings or update the script. It also installs
|
||||
`BackupDashboard.Collector.psm1`, the code shared by all collectors, next to the script.
|
||||
|
||||
## What it sends
|
||||
|
||||
|
||||
@@ -40,64 +40,23 @@ param(
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$DataDir = Split-Path -Parent $ConfigPath
|
||||
$LogPath = Join-Path $DataDir 'wsb-collector.log'
|
||||
$StatePath = Join-Path $DataDir 'wsb-collector.state.json'
|
||||
$MaxLogBytes = 5MB
|
||||
# Shared collector code: installed next to this script; in the repo, in ..\common
|
||||
$sharedModule = @(
|
||||
(Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'),
|
||||
(Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1')
|
||||
) | Where-Object { Test-Path $_ } | Select-Object -First 1
|
||||
if (-not $sharedModule) {
|
||||
Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer."
|
||||
exit 2
|
||||
}
|
||||
Import-Module $sharedModule -Force
|
||||
|
||||
# Get-WBJob -Previous accepts at most 100. That covers a 30-day backfill at up to 3 backups a day;
|
||||
# servers backing up more often only get their latest 100 runs backfilled.
|
||||
$MaxPreviousJobs = 100
|
||||
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||
|
||||
# ---------- Helpers ----------
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message, [string]$Level = 'INFO')
|
||||
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
|
||||
Write-Host $line
|
||||
if ($DryRun) { return }
|
||||
if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) {
|
||||
Move-Item $LogPath "$LogPath.1" -Force
|
||||
}
|
||||
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
||||
}
|
||||
|
||||
function Get-ErrorText($ErrorRecord) {
|
||||
# Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
|
||||
if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
|
||||
return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
|
||||
}
|
||||
return $ErrorRecord.Exception.Message
|
||||
}
|
||||
|
||||
# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a
|
||||
# Utc time as +00:00.
|
||||
function ConvertTo-IsoTime($Value) {
|
||||
if ($null -eq $Value) { return $null }
|
||||
$dt = [datetime]$Value
|
||||
if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue and friends)
|
||||
return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant)
|
||||
}
|
||||
|
||||
# First property of $Object, out of $Names, that exists and isn't empty. The WSB object model isn't
|
||||
# fully documented, so reads go through this rather than assuming one spelling.
|
||||
function Get-FirstValue($Object, [string[]]$Names) {
|
||||
if ($null -eq $Object) { return $null }
|
||||
foreach ($name in $Names) {
|
||||
try {
|
||||
$prop = $Object.PSObject.Properties[$name]
|
||||
if (-not $prop) { continue }
|
||||
$value = $prop.Value
|
||||
# Check emptiness by type: casting to [string] would make a one-item list of objects ''
|
||||
if ($null -eq $value) { continue }
|
||||
if ($value -is [string] -and $value -eq '') { continue }
|
||||
if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue }
|
||||
return $value
|
||||
} catch { }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Import-Wsb {
|
||||
if (Get-Command Get-WBJob -ErrorAction SilentlyContinue) { return } # already loaded
|
||||
# Present only when the Windows Server Backup feature (with its command-line tools) is installed
|
||||
@@ -288,49 +247,16 @@ function Get-NextRun {
|
||||
}
|
||||
}
|
||||
|
||||
function Send-Report($Body) {
|
||||
$json = $Body | ConvertTo-Json -Depth 6 -Compress
|
||||
if ($DryRun) {
|
||||
Write-Host $json
|
||||
return
|
||||
}
|
||||
# Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1
|
||||
$params = @{
|
||||
Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
|
||||
Method = 'Post'
|
||||
Headers = @{ Authorization = "Bearer $($Config.ApiKey)" }
|
||||
ContentType = 'application/json; charset=utf-8'
|
||||
Body = [Text.Encoding]::UTF8.GetBytes($json)
|
||||
TimeoutSec = 30
|
||||
}
|
||||
Invoke-RestMethod @params | Out-Null
|
||||
}
|
||||
|
||||
# ---------- Main ----------
|
||||
|
||||
try {
|
||||
$Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
|
||||
foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') {
|
||||
if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" }
|
||||
}
|
||||
$Config = Initialize-Collector -Name 'wsb' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun
|
||||
$SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Windows Server Backup' }
|
||||
$JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Windows Server Backup" }
|
||||
if ($LookbackHours -le 0) {
|
||||
$LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 }
|
||||
}
|
||||
|
||||
# Windows PowerShell 5.1 may default to TLS 1.0
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
||||
$LookbackHours = $Config.LookbackHours
|
||||
|
||||
Import-Wsb
|
||||
|
||||
# Run start time (ISO, as sent) -> Unix seconds, for runs already delivered in their final state
|
||||
$sent = @{}
|
||||
if (Test-Path $StatePath) {
|
||||
$saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json
|
||||
if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } }
|
||||
}
|
||||
|
||||
$since = (Get-Date).AddHours(-$LookbackHours)
|
||||
# Get-WBJob with no arguments returns the job in progress (if any); -Previous lists finished ones
|
||||
$jobs = @(@(Get-WBJob) + @(Get-WBJob -Previous $MaxPreviousJobs) | Where-Object {
|
||||
@@ -342,7 +268,7 @@ try {
|
||||
if ($start -lt $since) { continue }
|
||||
$byStart[(ConvertTo-IsoTime $start)] = $job # a job in both lists: the finished entry (later) wins
|
||||
}
|
||||
$pending = @($byStart.Keys | Sort-Object | Where-Object { -not $sent.ContainsKey($_) } |
|
||||
$pending = @($byStart.Keys | Sort-Object | Where-Object { -not (Test-RunSent ($_)) } |
|
||||
ForEach-Object { $byStart[$_] })
|
||||
|
||||
$jobInfo = [ordered]@{
|
||||
@@ -358,38 +284,19 @@ try {
|
||||
}
|
||||
|
||||
$failures = 0
|
||||
$runsSent = 0
|
||||
try {
|
||||
if ($pending.Count -eq 0) {
|
||||
# Keep next_run_at fresh even when nothing new ran
|
||||
Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo })
|
||||
Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo
|
||||
}
|
||||
foreach ($job in $pending) {
|
||||
$run = ConvertTo-Run $job
|
||||
Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run })
|
||||
$runsSent++
|
||||
if ($run.status -ne 'running') {
|
||||
$sent[$run.started_at] = ([DateTimeOffset](Get-JobTime $job 'StartTime')).ToUnixTimeSeconds()
|
||||
}
|
||||
Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.started_at -StartedAt ((Get-JobTime $job 'StartTime'))
|
||||
}
|
||||
} catch {
|
||||
$failures++
|
||||
Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR'
|
||||
}
|
||||
|
||||
if (-not $DryRun) {
|
||||
# Forget runs that have aged out of the lookback window (plus a day of slack)
|
||||
$cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds()
|
||||
$keep = [ordered]@{}
|
||||
foreach ($key in $sent.Keys) {
|
||||
if ($sent[$key] -ge $cutoff) { $keep[$key] = $sent[$key] }
|
||||
}
|
||||
try {
|
||||
$keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8
|
||||
} catch {
|
||||
Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
|
||||
}
|
||||
}
|
||||
|
||||
Write-Log ("Reported {0} runs, {1} failures" -f $runsSent, $failures)
|
||||
if ($failures) { exit 1 }
|
||||
$exitCode = Complete-Collector -Failures $failures
|
||||
if ($exitCode) { exit $exitCode }
|
||||
|
||||
Reference in New Issue
Block a user