Move code shared by the collectors into one module
The Veeam, MARS, WSB and VB365 collectors each carried identical copies of logging, ISO time conversion, property lookup, posting to the API, config loading, TLS setup, the sent-runs state file and the 401 check, and their installers repeated the same key prompt, locked-down config, file copy and scheduled-task registration. These now live in collectors/common/BackupDashboard.Collector.psm1: - Initialize-Collector / Test-RunSent / Send-JobReport / Test-AuthRejected / Complete-Collector for collector runs - Write-Log, Get-ErrorText, ConvertTo-IsoTime, Get-FirstValue - Install-CollectorTask for the shared half of each installer Collectors find the module next to themselves (installed) or in ..\common (repo); installers copy it into Program Files beside the collector. The diagnostic scripts stay standalone single files. No behavior change: a harness ran each collector against its mock (12 scenarios: dry run, two real runs against a stub API, state file, logs, exit codes, rejected key, en-GB, backfill) before and after, and all 84 outputs are identical. Installers, run with stand-ins for the Windows-only commands, write identical configs and tasks; the only differences are the module being copied and the summary line naming the shell's path. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,270 @@
|
||||
<#
|
||||
Shared code for the Backup Dashboard collectors: config and state files, logging, posting to
|
||||
the ingest API, and installing a collector as a scheduled task.
|
||||
|
||||
Each collector imports this from next to itself (installed: C:\Program Files\BackupDashboard)
|
||||
or from ..\common (in the repo). Collectors on the same server share one installed copy, so
|
||||
keep these functions backward compatible.
|
||||
|
||||
Works on Windows PowerShell 5.1 and PowerShell 7.
|
||||
#>
|
||||
|
||||
$script:Config = $null
|
||||
$script:DryRun = $false
|
||||
$script:LogPath = $null
|
||||
$script:StatePath = $null
|
||||
$script:LookbackHours = 48
|
||||
$script:Sent = @{}
|
||||
$script:RunsSent = 0
|
||||
$script:MaxLogBytes = 5MB
|
||||
$script:Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||
|
||||
# ---------- Logging and values ----------
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message, [string]$Level = 'INFO')
|
||||
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
|
||||
Write-Host $line
|
||||
if ($script:DryRun -or -not $script:LogPath) { return }
|
||||
if ((Test-Path $script:LogPath) -and (Get-Item $script:LogPath).Length -gt $script:MaxLogBytes) {
|
||||
Move-Item $script:LogPath "$($script:LogPath).1" -Force
|
||||
}
|
||||
Add-Content -Path $script:LogPath -Value $line -Encoding UTF8
|
||||
}
|
||||
|
||||
function Get-ErrorText($ErrorRecord) {
|
||||
# Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
|
||||
if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
|
||||
return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
|
||||
}
|
||||
return $ErrorRecord.Exception.Message
|
||||
}
|
||||
|
||||
# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a
|
||||
# Utc time as +00:00, so convert UTC-valued times to Kind Utc before calling this.
|
||||
function ConvertTo-IsoTime($Value) {
|
||||
if ($null -eq $Value) { return $null }
|
||||
$dt = [datetime]$Value
|
||||
if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue, MARS's 1601-01-01, ...)
|
||||
return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $script:Invariant)
|
||||
}
|
||||
|
||||
# First property of $Object, out of $Names, that exists and isn't empty. Backup products' object
|
||||
# models aren't fully documented, so reads go through this rather than assuming one spelling.
|
||||
function Get-FirstValue($Object, [string[]]$Names) {
|
||||
if ($null -eq $Object) { return $null }
|
||||
foreach ($name in $Names) {
|
||||
try {
|
||||
$prop = $Object.PSObject.Properties[$name]
|
||||
if (-not $prop) { continue }
|
||||
$value = $prop.Value
|
||||
# Check emptiness by type: casting to [string] would make a one-item list of objects ''
|
||||
if ($null -eq $value) { continue }
|
||||
if ($value -is [string] -and $value -eq '') { continue }
|
||||
if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue }
|
||||
return $value
|
||||
} catch { }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
# ---------- Collector run ----------
|
||||
|
||||
<#
|
||||
Loads the config ($ConfigPath, JSON) and the record of runs already sent, and sets up logging
|
||||
and TLS. Log and state files sit next to the config: <Name>-collector.log / .state.json.
|
||||
Returns the config object; its LookbackHours is resolved (the parameter, else the config, else
|
||||
48). Throws if the config can't be read or lacks one of $RequiredKeys.
|
||||
#>
|
||||
function Initialize-Collector {
|
||||
param(
|
||||
[Parameter(Mandatory)][string]$Name,
|
||||
[Parameter(Mandatory)][string]$ConfigPath,
|
||||
[int]$LookbackHours = 0,
|
||||
[switch]$DryRun,
|
||||
[string[]]$RequiredKeys = @('ApiUrl', 'ApiKey', 'Tenant')
|
||||
)
|
||||
# Logging first, so a bad config can still be logged
|
||||
$dataDir = Split-Path -Parent $ConfigPath
|
||||
$script:LogPath = Join-Path $dataDir "$Name-collector.log"
|
||||
$script:StatePath = Join-Path $dataDir "$Name-collector.state.json"
|
||||
$script:DryRun = [bool]$DryRun
|
||||
$script:RunsSent = 0
|
||||
$script:Sent = @{}
|
||||
|
||||
$config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
|
||||
foreach ($key in $RequiredKeys) {
|
||||
if (-not $config.$key) { throw "Config $ConfigPath is missing '$key'" }
|
||||
}
|
||||
if ($LookbackHours -le 0) {
|
||||
$LookbackHours = if ($config.LookbackHours) { [int]$config.LookbackHours } else { 48 }
|
||||
}
|
||||
$config | Add-Member -NotePropertyName LookbackHours -NotePropertyValue $LookbackHours -Force
|
||||
$script:LookbackHours = $LookbackHours
|
||||
$script:Config = $config
|
||||
|
||||
# Windows PowerShell 5.1 may default to TLS 1.0
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
||||
|
||||
# Run key -> start time (Unix seconds), for runs already delivered in their final state.
|
||||
# Numbers rather than ISO strings: PowerShell 7's ConvertFrom-Json turns date strings into
|
||||
# DateTimes and loses the offset.
|
||||
if (Test-Path $script:StatePath) {
|
||||
$saved = Get-Content -Path $script:StatePath -Raw | ConvertFrom-Json
|
||||
if ($saved) { foreach ($p in $saved.PSObject.Properties) { $script:Sent[$p.Name] = [long]$p.Value } }
|
||||
}
|
||||
return $config
|
||||
}
|
||||
|
||||
function Test-RunSent([string]$Key) {
|
||||
return $script:Sent.ContainsKey($Key)
|
||||
}
|
||||
|
||||
<#
|
||||
Posts one report to the ingest API: the job alone (to refresh next_run_at) or the job with a
|
||||
run. A run that has finished is then recorded as sent under $SentKey, so later collector runs
|
||||
skip it; a running one isn't, so it's re-sent until it finishes. Throws on an HTTP error.
|
||||
#>
|
||||
function Send-JobReport {
|
||||
param(
|
||||
[Parameter(Mandatory)][string]$Tenant,
|
||||
[Parameter(Mandatory)][string]$SourceSystem,
|
||||
[Parameter(Mandatory)]$Job,
|
||||
$Run,
|
||||
[string]$SentKey,
|
||||
$StartedAt
|
||||
)
|
||||
$body = [ordered]@{ tenant = $Tenant; source_system = $SourceSystem; job = $Job }
|
||||
if ($Run) { $body.run = $Run }
|
||||
$json = $body | ConvertTo-Json -Depth 6 -Compress
|
||||
if ($script:DryRun) {
|
||||
Write-Host $json
|
||||
} else {
|
||||
# Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1,
|
||||
# mangling non-ASCII job names (e.g. en dashes)
|
||||
$params = @{
|
||||
Uri = "$($script:Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
|
||||
Method = 'Post'
|
||||
Headers = @{ Authorization = "Bearer $($script:Config.ApiKey)" }
|
||||
ContentType = 'application/json; charset=utf-8'
|
||||
Body = [Text.Encoding]::UTF8.GetBytes($json)
|
||||
TimeoutSec = 30
|
||||
}
|
||||
Invoke-RestMethod @params | Out-Null
|
||||
}
|
||||
if ($Run) {
|
||||
$script:RunsSent++
|
||||
if ($Run.status -ne 'running' -and $SentKey) {
|
||||
$script:Sent[$SentKey] = ([DateTimeOffset][datetime]$StartedAt).ToUnixTimeSeconds()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# True when a failed Send-JobReport was the API rejecting the key: no point trying more jobs
|
||||
function Test-AuthRejected($ErrorRecord) {
|
||||
$status = 0
|
||||
try { $status = [int]$ErrorRecord.Exception.Response.StatusCode } catch { }
|
||||
return $status -eq 401
|
||||
}
|
||||
|
||||
<#
|
||||
Saves the sent-runs record (dropping runs older than the lookback window plus a day) and logs
|
||||
the summary line. Doesn't exit: the collector does that with the returned exit code.
|
||||
#>
|
||||
function Complete-Collector {
|
||||
param([int]$JobCount = -1, [int]$Failures = 0)
|
||||
if (-not $script:DryRun) {
|
||||
$cutoff = [DateTimeOffset]::Now.AddHours(-$script:LookbackHours - 24).ToUnixTimeSeconds()
|
||||
$keep = [ordered]@{}
|
||||
foreach ($key in $script:Sent.Keys) {
|
||||
if ($script:Sent[$key] -ge $cutoff) { $keep[$key] = $script:Sent[$key] }
|
||||
}
|
||||
try {
|
||||
$keep | ConvertTo-Json -Compress | Set-Content -Path $script:StatePath -Encoding UTF8
|
||||
} catch {
|
||||
Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
|
||||
}
|
||||
}
|
||||
if ($JobCount -ge 0) {
|
||||
Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $JobCount, $script:RunsSent, $Failures)
|
||||
} else {
|
||||
Write-Log ("Reported {0} runs, {1} failures" -f $script:RunsSent, $Failures)
|
||||
}
|
||||
if ($Failures) { return 1 }
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---------- Installing ----------
|
||||
|
||||
<#
|
||||
The part of every collector installer after its own checks: prompt for the API key, write the
|
||||
config to ProgramData\BackupDashboard (locked to SYSTEM and Administrators, since it holds the
|
||||
key), copy the collector script and this module to $InstallDir (Program Files: only admins can
|
||||
modify what runs as SYSTEM), and register a scheduled task running it as SYSTEM.
|
||||
#>
|
||||
function Install-CollectorTask {
|
||||
param(
|
||||
[Parameter(Mandatory)][string]$Name, # 'veeam' -> veeam-collector.json / .log
|
||||
[Parameter(Mandatory)][string]$ScriptPath, # the collector script to install
|
||||
[Parameter(Mandatory)][string]$TaskName,
|
||||
[Parameter(Mandatory)][string]$TaskDescription,
|
||||
[Parameter(Mandatory)][string]$Shell, # powershell.exe or pwsh.exe, full path
|
||||
[Parameter(Mandatory)][string]$ApiUrl,
|
||||
[Parameter(Mandatory)][int]$IntervalMinutes,
|
||||
[Parameter(Mandatory)][string]$InstallDir,
|
||||
[System.Collections.IDictionary]$Settings = @{} # extra config: Tenant, LookbackHours, ...
|
||||
)
|
||||
$dataDir = Join-Path $env:ProgramData 'BackupDashboard'
|
||||
$configPath = Join-Path $dataDir "$Name-collector.json"
|
||||
|
||||
if ($ApiUrl -notmatch '^https://') {
|
||||
Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.'
|
||||
}
|
||||
|
||||
# Read the key as a SecureString so it isn't echoed to the console or kept in shell history
|
||||
$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key <name>)'
|
||||
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
||||
try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
||||
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
||||
if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." }
|
||||
|
||||
# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows)
|
||||
New-Item -ItemType Directory -Path $dataDir -Force | Out-Null
|
||||
& icacls.exe $dataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
|
||||
if ($LASTEXITCODE) { throw "icacls failed on $dataDir" }
|
||||
|
||||
$config = [ordered]@{ ApiUrl = $ApiUrl.TrimEnd('/'); ApiKey = $apiKey }
|
||||
foreach ($key in $Settings.Keys) { $config[$key] = $Settings[$key] }
|
||||
$config | ConvertTo-Json -Depth 4 | Set-Content -Path $configPath -Encoding UTF8
|
||||
|
||||
# The collector and this module, side by side (the collector looks for the module next to itself)
|
||||
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
|
||||
$installedScript = Join-Path $InstallDir (Split-Path -Leaf $ScriptPath)
|
||||
foreach ($file in $ScriptPath, $PSCommandPath) {
|
||||
$target = Join-Path $InstallDir (Split-Path -Leaf $file)
|
||||
Copy-Item -Path $file -Destination $target -Force
|
||||
Unblock-File -Path $target
|
||||
}
|
||||
|
||||
$action = New-ScheduledTaskAction -Execute $Shell `
|
||||
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$installedScript`""
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) `
|
||||
-RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes)
|
||||
$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest
|
||||
$taskSettings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) `
|
||||
-MultipleInstances IgnoreNew -StartWhenAvailable
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal `
|
||||
-Settings $taskSettings -Description $TaskDescription -Force | Out-Null
|
||||
|
||||
Write-Host "Installed. Config: $configPath"
|
||||
Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using $Shell."
|
||||
Write-Host ''
|
||||
Write-Host 'Next steps:'
|
||||
Write-Host " 1. Preview what will be sent: & '$Shell' -File '$installedScript' -DryRun"
|
||||
Write-Host " 2. Backfill 30 days of history: & '$Shell' -File '$installedScript' -LookbackHours 720"
|
||||
Write-Host " 3. Check the log: Get-Content '$dataDir\$Name-collector.log' -Tail 20"
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function Write-Log, Get-ErrorText, ConvertTo-IsoTime, Get-FirstValue,
|
||||
Initialize-Collector, Test-RunSent, Send-JobReport, Test-AuthRejected, Complete-Collector,
|
||||
Install-CollectorTask
|
||||
Reference in New Issue
Block a user