From 8d0e6dca71923ede594deb13e0fa26818805adb8 Mon Sep 17 00:00:00 2001 From: kyle Date: Thu, 8 Oct 2026 11:09:01 -0600 Subject: [PATCH] Move code shared by the collectors into one module The Veeam, MARS, WSB and VB365 collectors each carried identical copies of logging, ISO time conversion, property lookup, posting to the API, config loading, TLS setup, the sent-runs state file and the 401 check, and their installers repeated the same key prompt, locked-down config, file copy and scheduled-task registration. These now live in collectors/common/BackupDashboard.Collector.psm1: - Initialize-Collector / Test-RunSent / Send-JobReport / Test-AuthRejected / Complete-Collector for collector runs - Write-Log, Get-ErrorText, ConvertTo-IsoTime, Get-FirstValue - Install-CollectorTask for the shared half of each installer Collectors find the module next to themselves (installed) or in ..\common (repo); installers copy it into Program Files beside the collector. The diagnostic scripts stay standalone single files. No behavior change: a harness ran each collector against its mock (12 scenarios: dry run, two real runs against a stub API, state file, logs, exit codes, rejected key, en-GB, backfill) before and after, and all 84 outputs are identical. Installers, run with stand-ins for the Windows-only commands, write identical configs and tasks; the only differences are the module being copied and the summary line naming the shell's path. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- .../common/BackupDashboard.Collector.psm1 | 270 ++++++++++++++++++ collectors/mars/Install-MarsCollector.ps1 | 63 +--- collectors/mars/README.md | 8 +- collectors/mars/Send-MarsBackupStatus.ps1 | 129 ++------- collectors/vb365/Install-Vb365Collector.ps1 | 59 +--- collectors/vb365/README.md | 6 +- collectors/vb365/Send-Vb365BackupStatus.ps1 | 132 ++------- collectors/veeam/Install-VeeamCollector.ps1 | 60 +--- collectors/veeam/README.md | 6 +- collectors/veeam/Send-VeeamBackupStatus.ps1 | 116 ++------ collectors/wsb/Install-WsbCollector.ps1 | 63 +--- collectors/wsb/README.md | 6 +- collectors/wsb/Send-WsbBackupStatus.ps1 | 129 ++------- 14 files changed, 416 insertions(+), 633 deletions(-) create mode 100644 collectors/common/BackupDashboard.Collector.psm1 diff --git a/README.md b/README.md index 830d162..c042c30 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ browser ──nginx (basic auth, TLS)──▶ /api/* ─▶│ SQLite │ - `public/`: static frontend (no build step). Add `?mock=1` to the URL to use built-in sample data. - `backend/`: FastAPI API backed by a single SQLite file. - `deploy/`: nginx site and systemd unit. -- `collectors/`: scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md), [Azure Backup / MARS](collectors/mars/README.md), [Windows Server Backup](collectors/wsb/README.md), [Veeam Backup for M365](collectors/vb365/README.md)). +- `collectors/`: PowerShell scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md), [Azure Backup / MARS](collectors/mars/README.md), [Windows Server Backup](collectors/wsb/README.md), [Veeam Backup for M365](collectors/vb365/README.md)). Code they share (config, logging, posting, state, installing) is in `collectors/common/BackupDashboard.Collector.psm1`. ## Data model diff --git a/collectors/common/BackupDashboard.Collector.psm1 b/collectors/common/BackupDashboard.Collector.psm1 new file mode 100644 index 0000000..76e6014 --- /dev/null +++ b/collectors/common/BackupDashboard.Collector.psm1 @@ -0,0 +1,270 @@ +<# + Shared code for the Backup Dashboard collectors: config and state files, logging, posting to + the ingest API, and installing a collector as a scheduled task. + + Each collector imports this from next to itself (installed: C:\Program Files\BackupDashboard) + or from ..\common (in the repo). Collectors on the same server share one installed copy, so + keep these functions backward compatible. + + Works on Windows PowerShell 5.1 and PowerShell 7. +#> + +$script:Config = $null +$script:DryRun = $false +$script:LogPath = $null +$script:StatePath = $null +$script:LookbackHours = 48 +$script:Sent = @{} +$script:RunsSent = 0 +$script:MaxLogBytes = 5MB +$script:Invariant = [Globalization.CultureInfo]::InvariantCulture + +# ---------- Logging and values ---------- + +function Write-Log { + param([string]$Message, [string]$Level = 'INFO') + $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message + Write-Host $line + if ($script:DryRun -or -not $script:LogPath) { return } + if ((Test-Path $script:LogPath) -and (Get-Item $script:LogPath).Length -gt $script:MaxLogBytes) { + Move-Item $script:LogPath "$($script:LogPath).1" -Force + } + Add-Content -Path $script:LogPath -Value $line -Encoding UTF8 +} + +function Get-ErrorText($ErrorRecord) { + # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here + if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { + return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)" + } + return $ErrorRecord.Exception.Message +} + +# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a +# Utc time as +00:00, so convert UTC-valued times to Kind Utc before calling this. +function ConvertTo-IsoTime($Value) { + if ($null -eq $Value) { return $null } + $dt = [datetime]$Value + if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue, MARS's 1601-01-01, ...) + return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $script:Invariant) +} + +# First property of $Object, out of $Names, that exists and isn't empty. Backup products' object +# models aren't fully documented, so reads go through this rather than assuming one spelling. +function Get-FirstValue($Object, [string[]]$Names) { + if ($null -eq $Object) { return $null } + foreach ($name in $Names) { + try { + $prop = $Object.PSObject.Properties[$name] + if (-not $prop) { continue } + $value = $prop.Value + # Check emptiness by type: casting to [string] would make a one-item list of objects '' + if ($null -eq $value) { continue } + if ($value -is [string] -and $value -eq '') { continue } + if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue } + return $value + } catch { } + } + return $null +} + +# ---------- Collector run ---------- + +<# + Loads the config ($ConfigPath, JSON) and the record of runs already sent, and sets up logging + and TLS. Log and state files sit next to the config: -collector.log / .state.json. + Returns the config object; its LookbackHours is resolved (the parameter, else the config, else + 48). Throws if the config can't be read or lacks one of $RequiredKeys. +#> +function Initialize-Collector { + param( + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$ConfigPath, + [int]$LookbackHours = 0, + [switch]$DryRun, + [string[]]$RequiredKeys = @('ApiUrl', 'ApiKey', 'Tenant') + ) + # Logging first, so a bad config can still be logged + $dataDir = Split-Path -Parent $ConfigPath + $script:LogPath = Join-Path $dataDir "$Name-collector.log" + $script:StatePath = Join-Path $dataDir "$Name-collector.state.json" + $script:DryRun = [bool]$DryRun + $script:RunsSent = 0 + $script:Sent = @{} + + $config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json + foreach ($key in $RequiredKeys) { + if (-not $config.$key) { throw "Config $ConfigPath is missing '$key'" } + } + if ($LookbackHours -le 0) { + $LookbackHours = if ($config.LookbackHours) { [int]$config.LookbackHours } else { 48 } + } + $config | Add-Member -NotePropertyName LookbackHours -NotePropertyValue $LookbackHours -Force + $script:LookbackHours = $LookbackHours + $script:Config = $config + + # Windows PowerShell 5.1 may default to TLS 1.0 + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + + # Run key -> start time (Unix seconds), for runs already delivered in their final state. + # Numbers rather than ISO strings: PowerShell 7's ConvertFrom-Json turns date strings into + # DateTimes and loses the offset. + if (Test-Path $script:StatePath) { + $saved = Get-Content -Path $script:StatePath -Raw | ConvertFrom-Json + if ($saved) { foreach ($p in $saved.PSObject.Properties) { $script:Sent[$p.Name] = [long]$p.Value } } + } + return $config +} + +function Test-RunSent([string]$Key) { + return $script:Sent.ContainsKey($Key) +} + +<# + Posts one report to the ingest API: the job alone (to refresh next_run_at) or the job with a + run. A run that has finished is then recorded as sent under $SentKey, so later collector runs + skip it; a running one isn't, so it's re-sent until it finishes. Throws on an HTTP error. +#> +function Send-JobReport { + param( + [Parameter(Mandatory)][string]$Tenant, + [Parameter(Mandatory)][string]$SourceSystem, + [Parameter(Mandatory)]$Job, + $Run, + [string]$SentKey, + $StartedAt + ) + $body = [ordered]@{ tenant = $Tenant; source_system = $SourceSystem; job = $Job } + if ($Run) { $body.run = $Run } + $json = $body | ConvertTo-Json -Depth 6 -Compress + if ($script:DryRun) { + Write-Host $json + } else { + # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1, + # mangling non-ASCII job names (e.g. en dashes) + $params = @{ + Uri = "$($script:Config.ApiUrl.TrimEnd('/'))/api/v1/ingest" + Method = 'Post' + Headers = @{ Authorization = "Bearer $($script:Config.ApiKey)" } + ContentType = 'application/json; charset=utf-8' + Body = [Text.Encoding]::UTF8.GetBytes($json) + TimeoutSec = 30 + } + Invoke-RestMethod @params | Out-Null + } + if ($Run) { + $script:RunsSent++ + if ($Run.status -ne 'running' -and $SentKey) { + $script:Sent[$SentKey] = ([DateTimeOffset][datetime]$StartedAt).ToUnixTimeSeconds() + } + } +} + +# True when a failed Send-JobReport was the API rejecting the key: no point trying more jobs +function Test-AuthRejected($ErrorRecord) { + $status = 0 + try { $status = [int]$ErrorRecord.Exception.Response.StatusCode } catch { } + return $status -eq 401 +} + +<# + Saves the sent-runs record (dropping runs older than the lookback window plus a day) and logs + the summary line. Doesn't exit: the collector does that with the returned exit code. +#> +function Complete-Collector { + param([int]$JobCount = -1, [int]$Failures = 0) + if (-not $script:DryRun) { + $cutoff = [DateTimeOffset]::Now.AddHours(-$script:LookbackHours - 24).ToUnixTimeSeconds() + $keep = [ordered]@{} + foreach ($key in $script:Sent.Keys) { + if ($script:Sent[$key] -ge $cutoff) { $keep[$key] = $script:Sent[$key] } + } + try { + $keep | ConvertTo-Json -Compress | Set-Content -Path $script:StatePath -Encoding UTF8 + } catch { + Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN' + } + } + if ($JobCount -ge 0) { + Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $JobCount, $script:RunsSent, $Failures) + } else { + Write-Log ("Reported {0} runs, {1} failures" -f $script:RunsSent, $Failures) + } + if ($Failures) { return 1 } + return 0 +} + +# ---------- Installing ---------- + +<# + The part of every collector installer after its own checks: prompt for the API key, write the + config to ProgramData\BackupDashboard (locked to SYSTEM and Administrators, since it holds the + key), copy the collector script and this module to $InstallDir (Program Files: only admins can + modify what runs as SYSTEM), and register a scheduled task running it as SYSTEM. +#> +function Install-CollectorTask { + param( + [Parameter(Mandatory)][string]$Name, # 'veeam' -> veeam-collector.json / .log + [Parameter(Mandatory)][string]$ScriptPath, # the collector script to install + [Parameter(Mandatory)][string]$TaskName, + [Parameter(Mandatory)][string]$TaskDescription, + [Parameter(Mandatory)][string]$Shell, # powershell.exe or pwsh.exe, full path + [Parameter(Mandatory)][string]$ApiUrl, + [Parameter(Mandatory)][int]$IntervalMinutes, + [Parameter(Mandatory)][string]$InstallDir, + [System.Collections.IDictionary]$Settings = @{} # extra config: Tenant, LookbackHours, ... + ) + $dataDir = Join-Path $env:ProgramData 'BackupDashboard' + $configPath = Join-Path $dataDir "$Name-collector.json" + + if ($ApiUrl -notmatch '^https://') { + Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.' + } + + # Read the key as a SecureString so it isn't echoed to the console or kept in shell history + $secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key )' + $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) + try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) } + finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) } + if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." } + + # Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows) + New-Item -ItemType Directory -Path $dataDir -Force | Out-Null + & icacls.exe $dataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null + if ($LASTEXITCODE) { throw "icacls failed on $dataDir" } + + $config = [ordered]@{ ApiUrl = $ApiUrl.TrimEnd('/'); ApiKey = $apiKey } + foreach ($key in $Settings.Keys) { $config[$key] = $Settings[$key] } + $config | ConvertTo-Json -Depth 4 | Set-Content -Path $configPath -Encoding UTF8 + + # The collector and this module, side by side (the collector looks for the module next to itself) + New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null + $installedScript = Join-Path $InstallDir (Split-Path -Leaf $ScriptPath) + foreach ($file in $ScriptPath, $PSCommandPath) { + $target = Join-Path $InstallDir (Split-Path -Leaf $file) + Copy-Item -Path $file -Destination $target -Force + Unblock-File -Path $target + } + + $action = New-ScheduledTaskAction -Execute $Shell ` + -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$installedScript`"" + $trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) + $principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest + $taskSettings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` + -MultipleInstances IgnoreNew -StartWhenAvailable + Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal ` + -Settings $taskSettings -Description $TaskDescription -Force | Out-Null + + Write-Host "Installed. Config: $configPath" + Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using $Shell." + Write-Host '' + Write-Host 'Next steps:' + Write-Host " 1. Preview what will be sent: & '$Shell' -File '$installedScript' -DryRun" + Write-Host " 2. Backfill 30 days of history: & '$Shell' -File '$installedScript' -LookbackHours 720" + Write-Host " 3. Check the log: Get-Content '$dataDir\$Name-collector.log' -Tail 20" +} + +Export-ModuleMember -Function Write-Log, Get-ErrorText, ConvertTo-IsoTime, Get-FirstValue, + Initialize-Collector, Test-RunSent, Send-JobReport, Test-AuthRejected, Complete-Collector, + Install-CollectorTask diff --git a/collectors/mars/Install-MarsCollector.ps1 b/collectors/mars/Install-MarsCollector.ps1 index 0be60d0..d4f802f 100644 --- a/collectors/mars/Install-MarsCollector.ps1 +++ b/collectors/mars/Install-MarsCollector.ps1 @@ -29,9 +29,6 @@ param( ) $ErrorActionPreference = 'Stop' -$TaskName = 'Backup Dashboard - MARS collector' -$DataDir = Join-Path $env:ProgramData 'BackupDashboard' -$ConfigPath = Join-Path $DataDir 'mars-collector.json' $WindowsPowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' # Check the agent is here before changing anything @@ -40,51 +37,19 @@ if (-not (Get-Module -ListAvailable -Name MSOnlineBackup) -and -not (Test-Path $ throw 'The MARS agent (MSOnlineBackup PowerShell module) is not installed on this server.' } -if ($ApiUrl -notmatch '^https://') { - Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.' +# Shared installer code: in the repo it's in ..\common, so copy the whole collectors folder +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + throw "BackupDashboard.Collector.psm1 not found. Copy the whole collectors folder, including common." } +Import-Module $sharedModule -Force -# Read the key as a SecureString so it isn't echoed to the console or kept in shell history -$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key )' -$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) -try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) } -finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) } -if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." } - -# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows) -New-Item -ItemType Directory -Path $DataDir -Force | Out-Null -& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null -if ($LASTEXITCODE) { throw "icacls failed on $DataDir" } - -$config = [ordered]@{ - ApiUrl = $ApiUrl.TrimEnd('/') - ApiKey = $apiKey - Tenant = $Tenant - SourceSystem = $SourceSystem - LookbackHours = $LookbackHours -} -if ($JobName) { $config.JobName = $JobName } -$config | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8 - -New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null -$scriptPath = Join-Path $InstallDir 'Send-MarsBackupStatus.ps1' -Copy-Item -Path (Join-Path $PSScriptRoot 'Send-MarsBackupStatus.ps1') -Destination $scriptPath -Force -Unblock-File -Path $scriptPath - -$action = New-ScheduledTaskAction -Execute $WindowsPowerShell ` - -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`"" -$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` - -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) -$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest -$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` - -MultipleInstances IgnoreNew -StartWhenAvailable -Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal ` - -Settings $settings -Description 'Sends Azure Backup (MARS) results to the Backup Dashboard' -Force | Out-Null - -Write-Host "Installed. Config: $ConfigPath" -Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using Windows PowerShell." -Write-Host '' -Write-Host 'Next steps:' -Write-Host " 1. Preview what will be sent: & '$WindowsPowerShell' -File '$scriptPath' -DryRun" -Write-Host " 2. Backfill 30 days of history: & '$WindowsPowerShell' -File '$scriptPath' -LookbackHours 720" -Write-Host " 3. Check the log: Get-Content '$DataDir\mars-collector.log' -Tail 20" +$settings = [ordered]@{ Tenant = $Tenant; SourceSystem = $SourceSystem; LookbackHours = $LookbackHours } +if ($JobName) { $settings.JobName = $JobName } +Install-CollectorTask -Name 'mars' -ScriptPath (Join-Path $PSScriptRoot 'Send-MarsBackupStatus.ps1') ` + -TaskName 'Backup Dashboard - MARS collector' ` + -TaskDescription 'Sends Azure Backup (MARS) results to the Backup Dashboard' ` + -Shell $WindowsPowerShell -ApiUrl $ApiUrl -IntervalMinutes $IntervalMinutes -InstallDir $InstallDir -Settings $settings diff --git a/collectors/mars/README.md b/collectors/mars/README.md index 9134852..6c854eb 100644 --- a/collectors/mars/README.md +++ b/collectors/mars/README.md @@ -18,7 +18,8 @@ targets and which every supported Windows Server has. ```bash python manage.py create-key mars-acme-fs01 ``` -2. Copy this folder to the server and run: +2. Copy the whole `collectors` folder to the server (the installer needs `common` + as well as this folder), then from `collectors\mars` run: ```powershell .\Install-MarsCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" ``` @@ -33,8 +34,9 @@ targets and which every supported Windows Server has. powershell.exe -File 'C:\Program Files\BackupDashboard\Send-MarsBackupStatus.ps1' -LookbackHours 720 ``` -It can be installed alongside the Veeam collector: it uses its own config, state, log and task. -Re-run the installer to change settings or update the script. +It can be installed alongside the other collectors: it uses its own config, state, log and task. +Re-run the installer to change settings or update the script. It also installs +`BackupDashboard.Collector.psm1`, the code shared by all collectors, next to the script. ## What it sends diff --git a/collectors/mars/Send-MarsBackupStatus.ps1 b/collectors/mars/Send-MarsBackupStatus.ps1 index 8ebaeb8..109d76b 100644 --- a/collectors/mars/Send-MarsBackupStatus.ps1 +++ b/collectors/mars/Send-MarsBackupStatus.ps1 @@ -38,63 +38,22 @@ param( $ErrorActionPreference = 'Stop' -$DataDir = Split-Path -Parent $ConfigPath -$LogPath = Join-Path $DataDir 'mars-collector.log' -$StatePath = Join-Path $DataDir 'mars-collector.state.json' -$MaxLogBytes = 5MB +# Shared collector code: installed next to this script; in the repo, in ..\common +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer." + exit 2 +} +Import-Module $sharedModule -Force + # MARS keeps a limited job history; this comfortably covers 30 days at 3 backups a day $MaxPreviousJobs = 200 -$Invariant = [Globalization.CultureInfo]::InvariantCulture # ---------- Helpers ---------- -function Write-Log { - param([string]$Message, [string]$Level = 'INFO') - $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message - Write-Host $line - if ($DryRun) { return } - if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) { - Move-Item $LogPath "$LogPath.1" -Force - } - Add-Content -Path $LogPath -Value $line -Encoding UTF8 -} - -function Get-ErrorText($ErrorRecord) { - # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here - if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { - return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)" - } - return $ErrorRecord.Exception.Message -} - -# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a -# Utc time as +00:00. -function ConvertTo-IsoTime($Value) { - if ($null -eq $Value) { return $null } - $dt = [datetime]$Value - if ($dt.Year -le 1900) { return $null } # "not set" (MARS uses 1601-01-01) - return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant) -} - -# First property of $Object, out of $Names, that exists and isn't empty. MARS object property names -# aren't documented, so reads go through this rather than assuming one spelling. -function Get-FirstValue($Object, [string[]]$Names) { - if ($null -eq $Object) { return $null } - foreach ($name in $Names) { - try { - $prop = $Object.PSObject.Properties[$name] - if (-not $prop) { continue } - $value = $prop.Value - # Check emptiness by type: casting to [string] would make a one-item list of objects '' - if ($null -eq $value) { continue } - if ($value -is [string] -and $value -eq '') { continue } - if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue } - return $value - } catch { } - } - return $null -} - function Import-Mars { if (Get-Command Get-OBJob -ErrorAction SilentlyContinue) { return } # already loaded if (Get-Module -ListAvailable -Name MSOnlineBackup) { @@ -305,49 +264,16 @@ function Test-IsBackupJob($Job) { return (-not $type) -or $type -match 'Backup' } -function Send-Report($Body) { - $json = $Body | ConvertTo-Json -Depth 6 -Compress - if ($DryRun) { - Write-Host $json - return - } - # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1 - $params = @{ - Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest" - Method = 'Post' - Headers = @{ Authorization = "Bearer $($Config.ApiKey)" } - ContentType = 'application/json; charset=utf-8' - Body = [Text.Encoding]::UTF8.GetBytes($json) - TimeoutSec = 30 - } - Invoke-RestMethod @params | Out-Null -} - # ---------- Main ---------- try { - $Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json - foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') { - if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" } - } + $Config = Initialize-Collector -Name 'mars' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Azure Backup (MARS)' } $JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Azure Backup" } - if ($LookbackHours -le 0) { - $LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 } - } - - # Windows PowerShell 5.1 may default to TLS 1.0 - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + $LookbackHours = $Config.LookbackHours Import-Mars - # Job ID -> start time (Unix seconds), for jobs already delivered in their final state - $sent = @{} - if (Test-Path $StatePath) { - $saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json - if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } } - } - $sinceUtc = (Get-Date).ToUniversalTime().AddHours(-$LookbackHours) # Get-OBJob with no arguments lists jobs in progress; -Previous lists finished ones $jobs = @(@(Get-OBJob) + @(Get-OBJob -Previous $MaxPreviousJobs) | Where-Object { @@ -360,7 +286,7 @@ try { $byId[[string]$job.JobId] = $job # a job in both lists: the finished entry (later) wins } $pending = @($byId.Values | - Where-Object { -not $sent.ContainsKey([string]$_.JobId) } | + Where-Object { -not (Test-RunSent ([string]$_.JobId)) } | Sort-Object { Get-JobTimeUtc $_ 'StartTime' }) $jobInfo = [ordered]@{ @@ -376,38 +302,19 @@ try { } $failures = 0 -$runsSent = 0 try { if ($pending.Count -eq 0) { # Keep next_run_at fresh even when nothing new ran - Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo }) + Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo } foreach ($job in $pending) { $run = ConvertTo-Run $job - Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run }) - $runsSent++ - if ($run.status -ne 'running') { - $sent[$run.external_id] = ([DateTimeOffset](Get-JobTimeUtc $job 'StartTime')).ToUnixTimeSeconds() - } + Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.external_id -StartedAt ((Get-JobTimeUtc $job 'StartTime')) } } catch { $failures++ Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR' } -if (-not $DryRun) { - # Forget jobs that have aged out of the lookback window (plus a day of slack) - $cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds() - $keep = [ordered]@{} - foreach ($id in $sent.Keys) { - if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] } - } - try { - $keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8 - } catch { - Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN' - } -} - -Write-Log ("Reported {0} runs, {1} failures" -f $runsSent, $failures) -if ($failures) { exit 1 } +$exitCode = Complete-Collector -Failures $failures +if ($exitCode) { exit $exitCode } diff --git a/collectors/vb365/Install-Vb365Collector.ps1 b/collectors/vb365/Install-Vb365Collector.ps1 index b11e1fd..c6a9891 100644 --- a/collectors/vb365/Install-Vb365Collector.ps1 +++ b/collectors/vb365/Install-Vb365Collector.ps1 @@ -37,9 +37,6 @@ param( ) $ErrorActionPreference = 'Stop' -$TaskName = 'Backup Dashboard - VB365 collector' -$DataDir = Join-Path $env:ProgramData 'BackupDashboard' -$ConfigPath = Join-Path $DataDir 'vb365-collector.json' $Shell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' if ($PwshPath) { if (-not (Test-Path $PwshPath)) { throw "PowerShell 7 not found at '$PwshPath'." } @@ -55,51 +52,23 @@ if (-not $Tenant -and $OrganizationTenants.Count -eq 0) { Write-Warning 'No -Tenant or -OrganizationTenants: jobs will be listed under their Microsoft 365 organization name.' } -if ($ApiUrl -notmatch '^https://') { - Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.' +# Shared installer code: in the repo it's in ..\common, so copy the whole collectors folder +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + throw "BackupDashboard.Collector.psm1 not found. Copy the whole collectors folder, including common." } +Import-Module $sharedModule -Force -# Read the key as a SecureString so it isn't echoed to the console or kept in shell history -$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key )' -$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) -try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) } -finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) } -if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." } - -# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows) -New-Item -ItemType Directory -Path $DataDir -Force | Out-Null -& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null -if ($LASTEXITCODE) { throw "icacls failed on $DataDir" } - -$config = [ordered]@{ - ApiUrl = $ApiUrl.TrimEnd('/') - ApiKey = $apiKey +$settings = [ordered]@{ SourceSystem = $SourceSystem LookbackHours = $LookbackHours OrganizationTenants = $OrganizationTenants } -if ($Tenant) { $config.Tenant = $Tenant } -$config | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8 - -New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null -$scriptPath = Join-Path $InstallDir 'Send-Vb365BackupStatus.ps1' -Copy-Item -Path (Join-Path $PSScriptRoot 'Send-Vb365BackupStatus.ps1') -Destination $scriptPath -Force -Unblock-File -Path $scriptPath - -$action = New-ScheduledTaskAction -Execute $Shell ` - -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`"" -$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` - -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) -$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest -$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` - -MultipleInstances IgnoreNew -StartWhenAvailable -Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal ` - -Settings $settings -Description 'Sends Veeam Backup for Microsoft 365 results to the Backup Dashboard' -Force | Out-Null - -Write-Host "Installed. Config: $ConfigPath" -Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using $Shell." -Write-Host '' -Write-Host 'Next steps:' -Write-Host " 1. Preview what will be sent: & '$Shell' -File '$scriptPath' -DryRun" -Write-Host " 2. Backfill 30 days of history: & '$Shell' -File '$scriptPath' -LookbackHours 720" -Write-Host " 3. Check the log: Get-Content '$DataDir\vb365-collector.log' -Tail 20" +if ($Tenant) { $settings.Tenant = $Tenant } +Install-CollectorTask -Name 'vb365' -ScriptPath (Join-Path $PSScriptRoot 'Send-Vb365BackupStatus.ps1') ` + -TaskName 'Backup Dashboard - VB365 collector' ` + -TaskDescription 'Sends Veeam Backup for Microsoft 365 results to the Backup Dashboard' ` + -Shell $Shell -ApiUrl $ApiUrl -IntervalMinutes $IntervalMinutes -InstallDir $InstallDir -Settings $settings diff --git a/collectors/vb365/README.md b/collectors/vb365/README.md index 9c005d4..2ab330e 100644 --- a/collectors/vb365/README.md +++ b/collectors/vb365/README.md @@ -27,7 +27,8 @@ one with `python manage.py delete-job `. ```bash python manage.py create-key vb365-01 ``` -2. Copy this folder to the server and run one of: +2. Copy the whole `collectors` folder to the server (the installer needs `common` + as well as this folder), then from `collectors\vb365` run one of: ```powershell # VB365 server for one client .\Install-Vb365Collector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" @@ -52,7 +53,8 @@ one with `python manage.py delete-job `. ``` It can be installed alongside the other collectors: it uses its own config, state, log and task. -Re-run the installer to change settings (including the organization mapping) or update the script. +Re-run the installer to change settings (including the organization mapping) or update the script. It also installs +`BackupDashboard.Collector.psm1`, the code shared by all collectors, next to the script. ## What it sends diff --git a/collectors/vb365/Send-Vb365BackupStatus.ps1 b/collectors/vb365/Send-Vb365BackupStatus.ps1 index ce4b701..3f34945 100644 --- a/collectors/vb365/Send-Vb365BackupStatus.ps1 +++ b/collectors/vb365/Send-Vb365BackupStatus.ps1 @@ -41,61 +41,21 @@ param( $ErrorActionPreference = 'Stop' -$DataDir = Split-Path -Parent $ConfigPath -$LogPath = Join-Path $DataDir 'vb365-collector.log' -$StatePath = Join-Path $DataDir 'vb365-collector.state.json' -$MaxLogBytes = 5MB +# Shared collector code: installed next to this script; in the repo, in ..\common +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer." + exit 2 +} +Import-Module $sharedModule -Force + $Invariant = [Globalization.CultureInfo]::InvariantCulture # ---------- Helpers ---------- -function Write-Log { - param([string]$Message, [string]$Level = 'INFO') - $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message - Write-Host $line - if ($DryRun) { return } - if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) { - Move-Item $LogPath "$LogPath.1" -Force - } - Add-Content -Path $LogPath -Value $line -Encoding UTF8 -} - -function Get-ErrorText($ErrorRecord) { - # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here - if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { - return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)" - } - return $ErrorRecord.Exception.Message -} - -# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a -# Utc time as +00:00. -function ConvertTo-IsoTime($Value) { - if ($null -eq $Value) { return $null } - $dt = [datetime]$Value - if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue and friends) - return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant) -} - -# First property of $Object, out of $Names, that exists and isn't empty. The VB365 object model isn't -# fully documented, so reads go through this rather than assuming one spelling. -function Get-FirstValue($Object, [string[]]$Names) { - if ($null -eq $Object) { return $null } - foreach ($name in $Names) { - try { - $prop = $Object.PSObject.Properties[$name] - if (-not $prop) { continue } - $value = $prop.Value - # Check emptiness by type: casting to [string] would make a one-item list of objects '' - if ($null -eq $value) { continue } - if ($value -is [string] -and $value -eq '') { continue } - if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue } - return $value - } catch { } - } - return $null -} - function Import-Vb365 { if (Get-Command Get-VBOJob -ErrorAction SilentlyContinue) { return } # already loaded if (Get-Module -ListAvailable -Name Veeam.Archiver.PowerShell) { @@ -245,48 +205,15 @@ function ConvertTo-Run($Session, $Job, [string]$JobKind) { return $run } -function Send-Report($Body) { - $json = $Body | ConvertTo-Json -Depth 6 -Compress - if ($DryRun) { - Write-Host $json - return - } - # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1 - $params = @{ - Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest" - Method = 'Post' - Headers = @{ Authorization = "Bearer $($Config.ApiKey)" } - ContentType = 'application/json; charset=utf-8' - Body = [Text.Encoding]::UTF8.GetBytes($json) - TimeoutSec = 30 - } - Invoke-RestMethod @params | Out-Null -} - # ---------- Main ---------- try { - $Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json - foreach ($key in 'ApiUrl', 'ApiKey') { - if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" } - } + $Config = Initialize-Collector -Name 'vb365' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun -RequiredKeys 'ApiUrl', 'ApiKey' $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Veeam Backup for M365' } - if ($LookbackHours -le 0) { - $LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 } - } - - # Windows PowerShell 5.1 may default to TLS 1.0 - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + $LookbackHours = $Config.LookbackHours Import-Vb365 - # Session ID -> start time (Unix seconds), for sessions already delivered in their final state - $sent = @{} - if (Test-Path $StatePath) { - $saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json - if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } } - } - $since = (Get-Date).AddHours(-$LookbackHours) # (job, kind, sessions) for backup jobs, plus backup copy jobs where the cmdlets exist @@ -315,7 +242,6 @@ try { } $failures = 0 -$runsSent = 0 foreach ($item in $work) { $job = $item.Job $tenant = Get-TenantForJob $job @@ -326,47 +252,27 @@ foreach ($item in $work) { next_run_at = Get-NextRun $job } $pending = @($item.Sessions | - Where-Object { $_ -and -not $sent.ContainsKey([string](Get-FirstValue $_ @('Id', 'SessionId'))) } | + Where-Object { $_ -and -not (Test-RunSent ([string](Get-FirstValue $_ @('Id', 'SessionId')))) } | Sort-Object { Get-Time $_ 'CreationTime' }) try { if ($pending.Count -eq 0) { # Keep next_run_at fresh even when nothing new ran - Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo }) + Send-JobReport -Tenant $tenant -SourceSystem $SourceSystem -Job $jobInfo } foreach ($session in $pending) { $run = ConvertTo-Run $session $job $item.Kind - Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo; run = $run }) - $runsSent++ - if ($run.status -ne 'running') { - $sent[$run.external_id] = ([DateTimeOffset](Get-Time $session 'CreationTime')).ToUnixTimeSeconds() - } + Send-JobReport -Tenant $tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.external_id -StartedAt ((Get-Time $session 'CreationTime')) } } catch { $failures++ Write-Log "Failed to report job '$($job.Name)': $(Get-ErrorText $_)" 'ERROR' - $httpStatus = 0 - try { $httpStatus = [int]$_.Exception.Response.StatusCode } catch { } - if ($httpStatus -eq 401) { + if (Test-AuthRejected $_) { Write-Log 'API key rejected; check ApiKey in the config. Stopping.' 'ERROR' break } } } -if (-not $DryRun) { - # Forget sessions that have aged out of the lookback window (plus a day of slack) - $cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds() - $keep = [ordered]@{} - foreach ($id in $sent.Keys) { - if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] } - } - try { - $keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8 - } catch { - Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN' - } -} - -Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $work.Count, $runsSent, $failures) -if ($failures) { exit 1 } +$exitCode = Complete-Collector -JobCount $work.Count -Failures $failures +if ($exitCode) { exit $exitCode } diff --git a/collectors/veeam/Install-VeeamCollector.ps1 b/collectors/veeam/Install-VeeamCollector.ps1 index e7ddb24..b57ed41 100644 --- a/collectors/veeam/Install-VeeamCollector.ps1 +++ b/collectors/veeam/Install-VeeamCollector.ps1 @@ -30,9 +30,6 @@ param( ) $ErrorActionPreference = 'Stop' -$TaskName = 'Backup Dashboard - Veeam collector' -$DataDir = Join-Path $env:ProgramData 'BackupDashboard' -$ConfigPath = Join-Path $DataDir 'veeam-collector.json' # Find PowerShell 7 before changing anything, so a missing install fails cleanly if (-not $PwshPath) { @@ -54,49 +51,18 @@ if ($LASTEXITCODE -or ($pwshMajor -as [int]) -lt 7) { throw "'$PwshPath' isn't a working PowerShell 7 (reported '$pwshVersion')." } -if ($ApiUrl -notmatch '^https://') { - Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.' +# Shared installer code: in the repo it's in ..\common, so copy the whole collectors folder +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + throw "BackupDashboard.Collector.psm1 not found. Copy the whole collectors folder, including common." } +Import-Module $sharedModule -Force -# Read the key as a SecureString so it isn't echoed to the console or kept in shell history -$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key )' -$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) -try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) } -finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) } -if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." } - -# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows) -New-Item -ItemType Directory -Path $DataDir -Force | Out-Null -& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null -if ($LASTEXITCODE) { throw "icacls failed on $DataDir" } - -[ordered]@{ - ApiUrl = $ApiUrl.TrimEnd('/') - ApiKey = $apiKey - Tenant = $Tenant - SourceSystem = $SourceSystem - LookbackHours = $LookbackHours -} | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8 - -New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null -$scriptPath = Join-Path $InstallDir 'Send-VeeamBackupStatus.ps1' -Copy-Item -Path (Join-Path $PSScriptRoot 'Send-VeeamBackupStatus.ps1') -Destination $scriptPath -Force -Unblock-File -Path $scriptPath - -$action = New-ScheduledTaskAction -Execute $PwshPath ` - -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`"" -$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` - -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) -$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest -$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` - -MultipleInstances IgnoreNew -StartWhenAvailable -Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal ` - -Settings $settings -Description 'Sends Veeam job results to the Backup Dashboard' -Force | Out-Null - -Write-Host "Installed. Config: $ConfigPath" -Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using PowerShell $pwshVersion ($PwshPath)." -Write-Host '' -Write-Host 'Next steps:' -Write-Host " 1. Preview what will be sent: & '$PwshPath' -File '$scriptPath' -DryRun" -Write-Host " 2. Backfill 30 days of history: & '$PwshPath' -File '$scriptPath' -LookbackHours 720" -Write-Host " 3. Check the log: Get-Content '$DataDir\veeam-collector.log' -Tail 20" +$settings = [ordered]@{ Tenant = $Tenant; SourceSystem = $SourceSystem; LookbackHours = $LookbackHours } +Install-CollectorTask -Name 'veeam' -ScriptPath (Join-Path $PSScriptRoot 'Send-VeeamBackupStatus.ps1') ` + -TaskName 'Backup Dashboard - Veeam collector' ` + -TaskDescription 'Sends Veeam job results to the Backup Dashboard' ` + -Shell $PwshPath -ApiUrl $ApiUrl -IntervalMinutes $IntervalMinutes -InstallDir $InstallDir -Settings $settings diff --git a/collectors/veeam/README.md b/collectors/veeam/README.md index 5b3dede..52162a7 100644 --- a/collectors/veeam/README.md +++ b/collectors/veeam/README.md @@ -13,7 +13,8 @@ account can't run it. The installer checks for this before changing anything. ```bash python manage.py create-key veeam-acme ``` -2. Copy this folder to the VBR server and run, from PowerShell 7 (`pwsh`): +2. Copy the whole `collectors` folder to the VBR server (the installer needs `common` + as well as this folder), then from `collectors\veeam` run, from PowerShell 7 (`pwsh`): ```powershell .\Install-VeeamCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" ``` @@ -29,7 +30,8 @@ account can't run it. The installer checks for this before changing anything. & 'C:\Program Files\BackupDashboard\Send-VeeamBackupStatus.ps1' -LookbackHours 720 ``` -Re-run the installer to change settings or update the script. To remove the collector, +Re-run the installer to change settings or update the script. It also installs +`BackupDashboard.Collector.psm1`, the code shared by all collectors, next to the script. To remove the collector, delete the scheduled task and both `BackupDashboard` folders. ## What it sends diff --git a/collectors/veeam/Send-VeeamBackupStatus.ps1 b/collectors/veeam/Send-VeeamBackupStatus.ps1 index 36c6dcb..c216c45 100644 --- a/collectors/veeam/Send-VeeamBackupStatus.ps1 +++ b/collectors/veeam/Send-VeeamBackupStatus.ps1 @@ -37,42 +37,19 @@ param( $ErrorActionPreference = 'Stop' -$DataDir = Split-Path -Parent $ConfigPath -$LogPath = Join-Path $DataDir 'veeam-collector.log' -$StatePath = Join-Path $DataDir 'veeam-collector.state.json' -$MaxLogBytes = 5MB -$Invariant = [Globalization.CultureInfo]::InvariantCulture +# Shared collector code: installed next to this script; in the repo, in ..\common +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer." + exit 2 +} +Import-Module $sharedModule -Force # ---------- Helpers ---------- -function Write-Log { - param([string]$Message, [string]$Level = 'INFO') - $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message - Write-Host $line - if ($DryRun) { return } - if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) { - Move-Item $LogPath "$LogPath.1" -Force - } - Add-Content -Path $LogPath -Value $line -Encoding UTF8 -} - -function Get-ErrorText($ErrorRecord) { - # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here - if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { - return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)" - } - return $ErrorRecord.Exception.Message -} - -# The API needs an explicit UTC offset. Veeam returns local times, often with Kind=Unspecified, -# which DateTimeOffset treats as local, exactly what we want. -function ConvertTo-IsoTime($Value) { - if ($null -eq $Value) { return $null } - $dt = [datetime]$Value - if ($dt.Year -le 1900) { return $null } # Veeam's "not set" value (e.g. EndTime while running) - return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant) -} - function Import-Veeam { if (Get-Command Get-VBRJob -ErrorAction SilentlyContinue) { return } # already loaded if (Get-Module -ListAvailable -Name Veeam.Backup.PowerShell) { @@ -217,51 +194,15 @@ function ConvertTo-Run($Session, $Job) { return $run } -function Send-Report($Body) { - $json = $Body | ConvertTo-Json -Depth 6 -Compress - if ($DryRun) { - Write-Host $json - return - } - # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1, - # mangling non-ASCII job names (e.g. en dashes). - $params = @{ - Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest" - Method = 'Post' - Headers = @{ Authorization = "Bearer $($Config.ApiKey)" } - ContentType = 'application/json; charset=utf-8' - Body = [Text.Encoding]::UTF8.GetBytes($json) - TimeoutSec = 30 - } - Invoke-RestMethod @params | Out-Null -} - # ---------- Main ---------- try { - $Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json - foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') { - if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" } - } + $Config = Initialize-Collector -Name 'veeam' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Veeam B&R' } - if ($LookbackHours -le 0) { - $LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 } - } - - # Windows PowerShell 5.1 may default to TLS 1.0 - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + $LookbackHours = $Config.LookbackHours Import-Veeam - # Session ID -> start time (Unix seconds), for sessions already delivered in their final state. - # Numbers rather than ISO strings: PowerShell 7's ConvertFrom-Json turns date strings into - # DateTimes and loses the offset. - $sent = @{} - if (Test-Path $StatePath) { - $saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json - if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } } - } - $since = (Get-Date).AddHours(-$LookbackHours) $jobs = @(Get-VBRJob -WarningAction SilentlyContinue) @@ -288,7 +229,6 @@ try { } $failures = 0 -$runsSent = 0 foreach ($job in $jobs) { $jobInfo = [ordered]@{ external_id = $job.Id.ToString() @@ -299,48 +239,28 @@ foreach ($job in $jobs) { $pending = @() if ($sessionsByJob.ContainsKey($jobInfo.external_id)) { $pending = @($sessionsByJob[$jobInfo.external_id] | - Where-Object { -not $sent.ContainsKey($_.Id.ToString()) } | + Where-Object { -not (Test-RunSent ($_.Id.ToString())) } | Sort-Object CreationTime) } try { if ($pending.Count -eq 0) { # Keep next_run_at fresh even when nothing new ran - Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo }) + Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo } foreach ($session in $pending) { $run = ConvertTo-Run $session $job - Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run }) - $runsSent++ - if ($run.status -ne 'running') { - $sent[$run.external_id] = ([DateTimeOffset][datetime]$session.CreationTime).ToUnixTimeSeconds() - } + Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.external_id -StartedAt ([datetime]$session.CreationTime) } } catch { $failures++ Write-Log "Failed to report job '$($job.Name)': $(Get-ErrorText $_)" 'ERROR' - $httpStatus = 0 - try { $httpStatus = [int]$_.Exception.Response.StatusCode } catch { } - if ($httpStatus -eq 401) { + if (Test-AuthRejected $_) { Write-Log 'API key rejected; check ApiKey in the config. Stopping.' 'ERROR' break } } } -if (-not $DryRun) { - # Forget sessions that have aged out of the lookback window (plus a day of slack) - $cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds() - $keep = [ordered]@{} - foreach ($id in $sent.Keys) { - if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] } - } - try { - $keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8 - } catch { - Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN' - } -} - -Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $jobs.Count, $runsSent, $failures) -if ($failures) { exit 1 } +$exitCode = Complete-Collector -JobCount $jobs.Count -Failures $failures +if ($exitCode) { exit $exitCode } diff --git a/collectors/wsb/Install-WsbCollector.ps1 b/collectors/wsb/Install-WsbCollector.ps1 index 963d74c..4bb2938 100644 --- a/collectors/wsb/Install-WsbCollector.ps1 +++ b/collectors/wsb/Install-WsbCollector.ps1 @@ -29,9 +29,6 @@ param( ) $ErrorActionPreference = 'Stop' -$TaskName = 'Backup Dashboard - WSB collector' -$DataDir = Join-Path $env:ProgramData 'BackupDashboard' -$ConfigPath = Join-Path $DataDir 'wsb-collector.json' $WindowsPowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' # Check Windows Server Backup is here before changing anything @@ -39,51 +36,19 @@ if (-not (Get-Module -ListAvailable -Name WindowsServerBackup)) { throw 'The WindowsServerBackup PowerShell module is not available. Install the Windows Server Backup feature (Install-WindowsFeature Windows-Server-Backup).' } -if ($ApiUrl -notmatch '^https://') { - Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.' +# Shared installer code: in the repo it's in ..\common, so copy the whole collectors folder +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + throw "BackupDashboard.Collector.psm1 not found. Copy the whole collectors folder, including common." } +Import-Module $sharedModule -Force -# Read the key as a SecureString so it isn't echoed to the console or kept in shell history -$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key )' -$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) -try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) } -finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) } -if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." } - -# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows) -New-Item -ItemType Directory -Path $DataDir -Force | Out-Null -& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null -if ($LASTEXITCODE) { throw "icacls failed on $DataDir" } - -$config = [ordered]@{ - ApiUrl = $ApiUrl.TrimEnd('/') - ApiKey = $apiKey - Tenant = $Tenant - SourceSystem = $SourceSystem - LookbackHours = $LookbackHours -} -if ($JobName) { $config.JobName = $JobName } -$config | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8 - -New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null -$scriptPath = Join-Path $InstallDir 'Send-WsbBackupStatus.ps1' -Copy-Item -Path (Join-Path $PSScriptRoot 'Send-WsbBackupStatus.ps1') -Destination $scriptPath -Force -Unblock-File -Path $scriptPath - -$action = New-ScheduledTaskAction -Execute $WindowsPowerShell ` - -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`"" -$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` - -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) -$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest -$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` - -MultipleInstances IgnoreNew -StartWhenAvailable -Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal ` - -Settings $settings -Description 'Sends Windows Server Backup results to the Backup Dashboard' -Force | Out-Null - -Write-Host "Installed. Config: $ConfigPath" -Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using Windows PowerShell." -Write-Host '' -Write-Host 'Next steps:' -Write-Host " 1. Preview what will be sent: & '$WindowsPowerShell' -File '$scriptPath' -DryRun" -Write-Host " 2. Backfill 30 days of history: & '$WindowsPowerShell' -File '$scriptPath' -LookbackHours 720" -Write-Host " 3. Check the log: Get-Content '$DataDir\wsb-collector.log' -Tail 20" +$settings = [ordered]@{ Tenant = $Tenant; SourceSystem = $SourceSystem; LookbackHours = $LookbackHours } +if ($JobName) { $settings.JobName = $JobName } +Install-CollectorTask -Name 'wsb' -ScriptPath (Join-Path $PSScriptRoot 'Send-WsbBackupStatus.ps1') ` + -TaskName 'Backup Dashboard - WSB collector' ` + -TaskDescription 'Sends Windows Server Backup results to the Backup Dashboard' ` + -Shell $WindowsPowerShell -ApiUrl $ApiUrl -IntervalMinutes $IntervalMinutes -InstallDir $InstallDir -Settings $settings diff --git a/collectors/wsb/README.md b/collectors/wsb/README.md index c31e699..e983d7f 100644 --- a/collectors/wsb/README.md +++ b/collectors/wsb/README.md @@ -19,7 +19,8 @@ Requires the Windows Server Backup feature with its PowerShell module ```bash python manage.py create-key wsb-acme-dc01 ``` -2. Copy this folder to the server and run: +2. Copy the whole `collectors` folder to the server (the installer needs `common` + as well as this folder), then from `collectors\wsb` run: ```powershell .\Install-WsbCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" ``` @@ -35,7 +36,8 @@ Requires the Windows Server Backup feature with its PowerShell module ``` It can be installed alongside the other collectors: it uses its own config, state, log and task. -Re-run the installer to change settings or update the script. +Re-run the installer to change settings or update the script. It also installs +`BackupDashboard.Collector.psm1`, the code shared by all collectors, next to the script. ## What it sends diff --git a/collectors/wsb/Send-WsbBackupStatus.ps1 b/collectors/wsb/Send-WsbBackupStatus.ps1 index 89f87d3..c6b76c3 100644 --- a/collectors/wsb/Send-WsbBackupStatus.ps1 +++ b/collectors/wsb/Send-WsbBackupStatus.ps1 @@ -40,64 +40,23 @@ param( $ErrorActionPreference = 'Stop' -$DataDir = Split-Path -Parent $ConfigPath -$LogPath = Join-Path $DataDir 'wsb-collector.log' -$StatePath = Join-Path $DataDir 'wsb-collector.state.json' -$MaxLogBytes = 5MB +# Shared collector code: installed next to this script; in the repo, in ..\common +$sharedModule = @( + (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), + (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') +) | Where-Object { Test-Path $_ } | Select-Object -First 1 +if (-not $sharedModule) { + Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer." + exit 2 +} +Import-Module $sharedModule -Force + # Get-WBJob -Previous accepts at most 100. That covers a 30-day backfill at up to 3 backups a day; # servers backing up more often only get their latest 100 runs backfilled. $MaxPreviousJobs = 100 -$Invariant = [Globalization.CultureInfo]::InvariantCulture # ---------- Helpers ---------- -function Write-Log { - param([string]$Message, [string]$Level = 'INFO') - $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message - Write-Host $line - if ($DryRun) { return } - if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) { - Move-Item $LogPath "$LogPath.1" -Force - } - Add-Content -Path $LogPath -Value $line -Encoding UTF8 -} - -function Get-ErrorText($ErrorRecord) { - # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here - if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { - return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)" - } - return $ErrorRecord.Exception.Message -} - -# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a -# Utc time as +00:00. -function ConvertTo-IsoTime($Value) { - if ($null -eq $Value) { return $null } - $dt = [datetime]$Value - if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue and friends) - return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant) -} - -# First property of $Object, out of $Names, that exists and isn't empty. The WSB object model isn't -# fully documented, so reads go through this rather than assuming one spelling. -function Get-FirstValue($Object, [string[]]$Names) { - if ($null -eq $Object) { return $null } - foreach ($name in $Names) { - try { - $prop = $Object.PSObject.Properties[$name] - if (-not $prop) { continue } - $value = $prop.Value - # Check emptiness by type: casting to [string] would make a one-item list of objects '' - if ($null -eq $value) { continue } - if ($value -is [string] -and $value -eq '') { continue } - if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue } - return $value - } catch { } - } - return $null -} - function Import-Wsb { if (Get-Command Get-WBJob -ErrorAction SilentlyContinue) { return } # already loaded # Present only when the Windows Server Backup feature (with its command-line tools) is installed @@ -288,49 +247,16 @@ function Get-NextRun { } } -function Send-Report($Body) { - $json = $Body | ConvertTo-Json -Depth 6 -Compress - if ($DryRun) { - Write-Host $json - return - } - # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1 - $params = @{ - Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest" - Method = 'Post' - Headers = @{ Authorization = "Bearer $($Config.ApiKey)" } - ContentType = 'application/json; charset=utf-8' - Body = [Text.Encoding]::UTF8.GetBytes($json) - TimeoutSec = 30 - } - Invoke-RestMethod @params | Out-Null -} - # ---------- Main ---------- try { - $Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json - foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') { - if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" } - } + $Config = Initialize-Collector -Name 'wsb' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Windows Server Backup' } $JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Windows Server Backup" } - if ($LookbackHours -le 0) { - $LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 } - } - - # Windows PowerShell 5.1 may default to TLS 1.0 - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + $LookbackHours = $Config.LookbackHours Import-Wsb - # Run start time (ISO, as sent) -> Unix seconds, for runs already delivered in their final state - $sent = @{} - if (Test-Path $StatePath) { - $saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json - if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } } - } - $since = (Get-Date).AddHours(-$LookbackHours) # Get-WBJob with no arguments returns the job in progress (if any); -Previous lists finished ones $jobs = @(@(Get-WBJob) + @(Get-WBJob -Previous $MaxPreviousJobs) | Where-Object { @@ -342,7 +268,7 @@ try { if ($start -lt $since) { continue } $byStart[(ConvertTo-IsoTime $start)] = $job # a job in both lists: the finished entry (later) wins } - $pending = @($byStart.Keys | Sort-Object | Where-Object { -not $sent.ContainsKey($_) } | + $pending = @($byStart.Keys | Sort-Object | Where-Object { -not (Test-RunSent ($_)) } | ForEach-Object { $byStart[$_] }) $jobInfo = [ordered]@{ @@ -358,38 +284,19 @@ try { } $failures = 0 -$runsSent = 0 try { if ($pending.Count -eq 0) { # Keep next_run_at fresh even when nothing new ran - Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo }) + Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo } foreach ($job in $pending) { $run = ConvertTo-Run $job - Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run }) - $runsSent++ - if ($run.status -ne 'running') { - $sent[$run.started_at] = ([DateTimeOffset](Get-JobTime $job 'StartTime')).ToUnixTimeSeconds() - } + Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.started_at -StartedAt ((Get-JobTime $job 'StartTime')) } } catch { $failures++ Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR' } -if (-not $DryRun) { - # Forget runs that have aged out of the lookback window (plus a day of slack) - $cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds() - $keep = [ordered]@{} - foreach ($key in $sent.Keys) { - if ($sent[$key] -ge $cutoff) { $keep[$key] = $sent[$key] } - } - try { - $keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8 - } catch { - Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN' - } -} - -Write-Log ("Reported {0} runs, {1} failures" -f $runsSent, $failures) -if ($failures) { exit 1 } +$exitCode = Complete-Collector -Failures $failures +if ($exitCode) { exit $exitCode }