From 10181e014b37b9576194493b537ad7471c459066 Mon Sep 17 00:00:00 2001 From: kyle Date: Wed, 7 Oct 2026 15:23:46 -0600 Subject: [PATCH] Add Azure Backup (MARS agent) collector PowerShell script for servers running the MARS agent, run as a SYSTEM scheduled task (hourly) under Windows PowerShell 5.1, which the agent's MSOnlineBackup module targets. Each server is one dashboard job. - Backup jobs from Get-OBJob (in progress and -Previous) within the lookback window; restores are ignored. Completed with failed bytes or an error reports as warning - Next run computed from the backup and System State policy schedules; left empty for every-N-weeks schedules to avoid false Overdue alarms - Property reads tolerate unknown names, since the agent's object model isn't documented; Get-MarsCollectorDiagnostics.ps1 dumps the real structure for checking - Install-MarsCollector.ps1 mirrors the Veeam installer; tests/MockMars.ps1 provides stand-in cmdlets Not yet run against a real MARS agent. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- .../mars/Get-MarsCollectorDiagnostics.ps1 | 99 +++++ collectors/mars/Install-MarsCollector.ps1 | 90 +++++ collectors/mars/README.md | 66 ++++ collectors/mars/Send-MarsBackupStatus.ps1 | 369 ++++++++++++++++++ collectors/mars/tests/MockMars.ps1 | 56 +++ 6 files changed, 681 insertions(+), 1 deletion(-) create mode 100644 collectors/mars/Get-MarsCollectorDiagnostics.ps1 create mode 100644 collectors/mars/Install-MarsCollector.ps1 create mode 100644 collectors/mars/README.md create mode 100644 collectors/mars/Send-MarsBackupStatus.ps1 create mode 100644 collectors/mars/tests/MockMars.ps1 diff --git a/README.md b/README.md index 1afaefb..79cfab7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ browser ──nginx (basic auth, TLS)──▶ /api/* ─▶│ SQLite │ - `public/`: static frontend (no build step). Add `?mock=1` to the URL to use built-in sample data. - `backend/`: FastAPI API backed by a single SQLite file. - `deploy/`: nginx site and systemd unit. -- `collectors/`: scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md)). +- `collectors/`: scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md), [Azure Backup / MARS](collectors/mars/README.md)). ## Data model diff --git a/collectors/mars/Get-MarsCollectorDiagnostics.ps1 b/collectors/mars/Get-MarsCollectorDiagnostics.ps1 new file mode 100644 index 0000000..f918a8e --- /dev/null +++ b/collectors/mars/Get-MarsCollectorDiagnostics.ps1 @@ -0,0 +1,99 @@ +<# +.SYNOPSIS + Read-only dump of what the MARS agent's cmdlets return, for troubleshooting the collector. + +.DESCRIPTION + The collector reads properties of Get-OBJob / Get-OBPolicy / Get-OBSchedule results whose names + aren't documented. This prints those objects' real structure (property names, types and values, + a few levels deep) so mismatches can be fixed. Nothing is changed. Output is also saved to a + text file. + +.EXAMPLE + .\Get-MarsCollectorDiagnostics.ps1 +#> +[CmdletBinding()] +param( + [int]$Jobs = 3, + [string]$OutFile = (Join-Path $env:TEMP 'mars-collector-diagnostics.txt') +) + +$ErrorActionPreference = 'Stop' +$MaxDepth = 6 + +if (-not (Get-Command Get-OBJob -ErrorAction SilentlyContinue)) { + if (Get-Module -ListAvailable -Name MSOnlineBackup) { + Import-Module MSOnlineBackup + } else { + Import-Module (Join-Path $env:ProgramFiles 'Microsoft Azure Recovery Services Agent\bin\Modules\MSOnlineBackup') + } +} + +function Test-Scalar($Value) { + return $null -eq $Value -or $Value -is [string] -or $Value -is [ValueType] -or $Value -is [guid] +} + +# Recursive "name (type) = value" listing. Collections show their first 3 items. +function Show-Object($Value, [string]$Indent = ' ', [int]$Depth = 0) { + if (Test-Scalar $Value) { return } + if ($Depth -ge $MaxDepth) { "$Indent..."; return } + if ($Value -is [Collections.IEnumerable]) { + $items = @($Value) + "$Indent[$($items.Count) item(s)]" + $i = 0 + foreach ($item in $items | Select-Object -First 3) { + "$Indent[$i] ($($item.GetType().FullName))$(if (Test-Scalar $item) { " = $item" })" + Show-Object $item "$Indent " ($Depth + 1) + $i++ + } + return + } + foreach ($p in $Value.PSObject.Properties) { + try { $v = $p.Value } catch { "$Indent$($p.Name) = "; continue } + $type = if ($null -eq $v) { 'null' } else { $v.GetType().FullName } + if (Test-Scalar $v) { + "$Indent$($p.Name) ($type) = $v" + } else { + $text = [string]$v + if ($text.Length -gt 100) { $text = $text.Substring(0, 97) + '...' } + "$Indent$($p.Name) ($type) = $text" + Show-Object $v "$Indent " ($Depth + 1) + } + } +} + +function Show-Section([string]$Title, [scriptblock]$Get) { + "== $Title" + try { + $result = @(& $Get) + if ($result.Count -eq 0 -or ($result.Count -eq 1 -and $null -eq $result[0])) { ' (nothing returned)' } + $n = 0 + foreach ($item in $result) { + if ($null -eq $item) { continue } + " [$n] ($($item.GetType().FullName))" + Show-Object $item ' ' + $n++ + } + } catch { + " error: $($_.Exception.Message)" + } + '' +} + +$report = & { + $agent = try { + (Get-Item (Join-Path $env:ProgramFiles 'Microsoft Azure Recovery Services Agent\bin\cbengine.exe')).VersionInfo.ProductVersion + } catch { 'unknown' } + "Generated $(Get-Date -Format o) on $env:COMPUTERNAME, MARS agent $agent, PowerShell $($PSVersionTable.PSVersion)" + '' + Show-Section 'Get-OBJob (in progress)' { Get-OBJob } + Show-Section "Get-OBJob -Previous $Jobs" { Get-OBJob -Previous $Jobs } + Show-Section 'Get-OBPolicy' { Get-OBPolicy } + Show-Section 'Get-OBPolicy | Get-OBSchedule' { Get-OBSchedule -Policy (Get-OBPolicy) } + if (Get-Command Get-OBSystemStatePolicy -ErrorAction SilentlyContinue) { + Show-Section 'Get-OBSystemStatePolicy | Get-OBSchedule' { Get-OBSchedule -Policy (Get-OBSystemStatePolicy) } + } +} | Out-String -Width 300 + +$report +$report | Set-Content -Path $OutFile -Encoding UTF8 +Write-Host "Saved to $OutFile" diff --git a/collectors/mars/Install-MarsCollector.ps1 b/collectors/mars/Install-MarsCollector.ps1 new file mode 100644 index 0000000..0be60d0 --- /dev/null +++ b/collectors/mars/Install-MarsCollector.ps1 @@ -0,0 +1,90 @@ +<# +.SYNOPSIS + Installs the MARS (Azure Backup agent) collector as a SYSTEM scheduled task. + +.DESCRIPTION + - Copies Send-MarsBackupStatus.ps1 to Program Files (only admins can modify it, which + matters because it runs as SYSTEM). + - Writes the config to ProgramData\BackupDashboard and restricts that folder to SYSTEM and + Administrators, since it contains the API key. + - Registers a scheduled task that runs every IntervalMinutes under Windows PowerShell 5.1 + (powershell.exe), which the agent's MSOnlineBackup module targets. + Safe to re-run to change settings or update the script. Can sit alongside the Veeam collector. + +.EXAMPLE + .\Install-MarsCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" + (prompts for the API key) +#> +#Requires -RunAsAdministrator +[CmdletBinding()] +param( + [Parameter(Mandatory)][string]$ApiUrl, + [Parameter(Mandatory)][string]$Tenant, + # Name shown on the dashboard; defaults to " - Azure Backup" + [string]$JobName, + [string]$SourceSystem = 'Azure Backup (MARS)', + [ValidateRange(5, 1440)][int]$IntervalMinutes = 60, + [ValidateRange(1, 720)][int]$LookbackHours = 48, + [string]$InstallDir = (Join-Path $env:ProgramFiles 'BackupDashboard') +) + +$ErrorActionPreference = 'Stop' +$TaskName = 'Backup Dashboard - MARS collector' +$DataDir = Join-Path $env:ProgramData 'BackupDashboard' +$ConfigPath = Join-Path $DataDir 'mars-collector.json' +$WindowsPowerShell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe' + +# Check the agent is here before changing anything +$agentModule = Join-Path $env:ProgramFiles 'Microsoft Azure Recovery Services Agent\bin\Modules\MSOnlineBackup' +if (-not (Get-Module -ListAvailable -Name MSOnlineBackup) -and -not (Test-Path $agentModule)) { + throw 'The MARS agent (MSOnlineBackup PowerShell module) is not installed on this server.' +} + +if ($ApiUrl -notmatch '^https://') { + Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.' +} + +# Read the key as a SecureString so it isn't echoed to the console or kept in shell history +$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key )' +$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure) +try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) } +finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) } +if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." } + +# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows) +New-Item -ItemType Directory -Path $DataDir -Force | Out-Null +& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null +if ($LASTEXITCODE) { throw "icacls failed on $DataDir" } + +$config = [ordered]@{ + ApiUrl = $ApiUrl.TrimEnd('/') + ApiKey = $apiKey + Tenant = $Tenant + SourceSystem = $SourceSystem + LookbackHours = $LookbackHours +} +if ($JobName) { $config.JobName = $JobName } +$config | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8 + +New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null +$scriptPath = Join-Path $InstallDir 'Send-MarsBackupStatus.ps1' +Copy-Item -Path (Join-Path $PSScriptRoot 'Send-MarsBackupStatus.ps1') -Destination $scriptPath -Force +Unblock-File -Path $scriptPath + +$action = New-ScheduledTaskAction -Execute $WindowsPowerShell ` + -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`"" +$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes) +$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) ` + -MultipleInstances IgnoreNew -StartWhenAvailable +Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal ` + -Settings $settings -Description 'Sends Azure Backup (MARS) results to the Backup Dashboard' -Force | Out-Null + +Write-Host "Installed. Config: $ConfigPath" +Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using Windows PowerShell." +Write-Host '' +Write-Host 'Next steps:' +Write-Host " 1. Preview what will be sent: & '$WindowsPowerShell' -File '$scriptPath' -DryRun" +Write-Host " 2. Backfill 30 days of history: & '$WindowsPowerShell' -File '$scriptPath' -LookbackHours 720" +Write-Host " 3. Check the log: Get-Content '$DataDir\mars-collector.log' -Tail 20" diff --git a/collectors/mars/README.md b/collectors/mars/README.md new file mode 100644 index 0000000..3ac6f15 --- /dev/null +++ b/collectors/mars/README.md @@ -0,0 +1,66 @@ +# Azure Backup (MARS agent) collector + +A PowerShell script that runs on each server with the Microsoft Azure Recovery Services (MARS) +agent and reports its backups to the dashboard. Each server appears as one job, +named ` - Azure Backup` by default. + +> **Not yet checked against a real MARS agent.** The cmdlets are the agent's own (`Get-OBJob`, +> `Get-OBPolicy`, `Get-OBSchedule`), but the property names of the objects they return aren't +> documented. Before installing on the first server, run `Get-MarsCollectorDiagnostics.ps1` and +> the collector with `-DryRun` (see below) to confirm what it reads. + +## Install (on the server, as an administrator) + +Runs under Windows PowerShell 5.1 (`powershell.exe`), which the agent's `MSOnlineBackup` module +targets and which every supported Windows Server has. + +1. On the dashboard host, create a key for this server: + ```bash + python manage.py create-key mars-acme-fs01 + ``` +2. Copy this folder to the server and run: + ```powershell + .\Install-MarsCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp" + ``` + Add `-JobName "File Server"` to choose the name shown on the dashboard. It prompts for the + key, then: + - writes `C:\ProgramData\BackupDashboard\mars-collector.json` (readable only by SYSTEM and Administrators) + - copies the script to `C:\Program Files\BackupDashboard\` + - registers the scheduled task **Backup Dashboard - MARS collector** (SYSTEM, every 60 minutes) +3. Preview the payloads, then backfill 30 days so the history view is populated straight away: + ```powershell + powershell.exe -File 'C:\Program Files\BackupDashboard\Send-MarsBackupStatus.ps1' -DryRun + powershell.exe -File 'C:\Program Files\BackupDashboard\Send-MarsBackupStatus.ps1' -LookbackHours 720 + ``` + +It can be installed alongside the Veeam collector: it uses its own config, state, log and task. +Re-run the installer to change settings or update the script. + +## What it sends + +| Dashboard | MARS | +|---|---| +| Company | `Tenant` from the config | +| Job | One per server, ID `mars:`, so changing `JobName` renames rather than duplicates it | +| Status | Backup job state: Failed → **Failed**. Completed with failed bytes or an error → **Warning**. Aborted/cancelled or unrecognised → **Warning**. In progress → **Running**. Restore jobs are ignored | +| Started / finished | Job start / end time | +| Next scheduled | Next slot from the backup policy schedule, or the System State policy's if that's sooner. Empty (so never Overdue) for schedules that run every 2+ weeks, since which week is next can't be told | +| Message | Job and per-datasource errors, and datasources with failed bytes | +| Size | Bytes transferred, summed across datasources | + +Each run covers the last 48 hours (`LookbackHours`). Jobs already delivered are recorded in +`mars-collector.state.json`. + +## Troubleshooting + +- Log: `C:\ProgramData\BackupDashboard\mars-collector.log`. Exit codes: `0` OK, `1` some runs + failed to send (retried next run), `2` setup failed (config or MARS module). +- Wrong status, sizes or messages: run `.\Get-MarsCollectorDiagnostics.ps1` (read-only). It prints + the real structure of the objects the agent's cmdlets return. + +## Testing without MARS + +```powershell +. ./tests/MockMars.ps1 +./Send-MarsBackupStatus.ps1 -ConfigPath ./my-test-config.json -DryRun +``` diff --git a/collectors/mars/Send-MarsBackupStatus.ps1 b/collectors/mars/Send-MarsBackupStatus.ps1 new file mode 100644 index 0000000..a58caa9 --- /dev/null +++ b/collectors/mars/Send-MarsBackupStatus.ps1 @@ -0,0 +1,369 @@ +<# +.SYNOPSIS + Reports Microsoft Azure Recovery Services (MARS) agent backups to the Backup Dashboard. + +.DESCRIPTION + Runs on a server with the MARS agent, normally as a SYSTEM scheduled task every 60 minutes + (see Install-MarsCollector.ps1). The server appears on the dashboard as one job. Each run sends: + - the next scheduled backup, worked out from the backup (and System State) policy schedules, + which drives the dashboard's Overdue flag, and + - every backup job from the last LookbackHours that the dashboard hasn't already received in + its final state. A backup in progress is sent as Running until it finishes. + + Already-reported jobs are tracked in a state file, and the ingest API is idempotent, so a + failed or repeated run is harmless: anything not confirmed is retried next time. + + Written for Windows PowerShell 5.1, which the MARS agent's MSOnlineBackup module targets. + +.PARAMETER ConfigPath + JSON config with ApiUrl, ApiKey and Tenant (optional: JobName, SourceSystem, LookbackHours). + +.PARAMETER LookbackHours + Overrides the config value. Use 720 once to backfill 30 days of history. + +.PARAMETER DryRun + Print the payloads instead of sending them. Doesn't touch the state or log files. + +.EXAMPLE + .\Send-MarsBackupStatus.ps1 -DryRun +.EXAMPLE + .\Send-MarsBackupStatus.ps1 -LookbackHours 720 +#> +[CmdletBinding()] +param( + [string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\mars-collector.json'), + [int]$LookbackHours = 0, + [switch]$DryRun +) + +$ErrorActionPreference = 'Stop' + +$DataDir = Split-Path -Parent $ConfigPath +$LogPath = Join-Path $DataDir 'mars-collector.log' +$StatePath = Join-Path $DataDir 'mars-collector.state.json' +$MaxLogBytes = 5MB +# MARS keeps a limited job history; this comfortably covers 30 days at 3 backups a day +$MaxPreviousJobs = 200 +$Invariant = [Globalization.CultureInfo]::InvariantCulture + +# ---------- Helpers ---------- + +function Write-Log { + param([string]$Message, [string]$Level = 'INFO') + $line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message + Write-Host $line + if ($DryRun) { return } + if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) { + Move-Item $LogPath "$LogPath.1" -Force + } + Add-Content -Path $LogPath -Value $line -Encoding UTF8 +} + +function Get-ErrorText($ErrorRecord) { + # Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here + if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) { + return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)" + } + return $ErrorRecord.Exception.Message +} + +# The API needs an explicit UTC offset. MARS returns local times, which DateTimeOffset keeps as local. +function ConvertTo-IsoTime($Value) { + if ($null -eq $Value) { return $null } + $dt = [datetime]$Value + if ($dt.Year -le 1900) { return $null } # "not set" + return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant) +} + +# First property of $Object, out of $Names, that exists and isn't empty. MARS object property names +# aren't documented, so reads go through this rather than assuming one spelling. +function Get-FirstValue($Object, [string[]]$Names) { + if ($null -eq $Object) { return $null } + foreach ($name in $Names) { + try { + $prop = $Object.PSObject.Properties[$name] + if (-not $prop) { continue } + $value = $prop.Value + # Check emptiness by type: casting to [string] would make a one-item list of objects '' + if ($null -eq $value) { continue } + if ($value -is [string] -and $value -eq '') { continue } + if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue } + return $value + } catch { } + } + return $null +} + +function Import-Mars { + if (Get-Command Get-OBJob -ErrorAction SilentlyContinue) { return } # already loaded + if (Get-Module -ListAvailable -Name MSOnlineBackup) { + Import-Module MSOnlineBackup + return + } + # The agent normally adds its module folder to PSModulePath; fall back to the default install path + $agentModule = Join-Path $env:ProgramFiles 'Microsoft Azure Recovery Services Agent\bin\Modules\MSOnlineBackup' + Import-Module $agentModule +} + +# ---------- Schedule ---------- + +# Next slot strictly after $After for a MARS schedule (days of week + times of day). +# Returns $null when there's no schedule, or when it runs every N>1 weeks: we can't tell which +# week is next, and a wrong guess would raise a false Overdue alarm. +function Get-NextSlot($Schedule, [datetime]$After) { + if ($null -eq $Schedule) { return $null } + $weeks = Get-FirstValue $Schedule @('ScheduleWeeklyFrequency', 'WeeksInterval') + if ($weeks -and [int]$weeks -gt 1) { return $null } + + $days = @(Get-FirstValue $Schedule @('ScheduleRunDays', 'DaysOfWeek') | ForEach-Object { [string]$_ }) + $times = @(Get-FirstValue $Schedule @('ScheduleRunTimes', 'TimesOfDay') | + ForEach-Object { ([datetime]$_).TimeOfDay } | Sort-Object) + if ($days.Count -eq 0 -or $times.Count -eq 0) { return $null } + + for ($offset = 0; $offset -le 7; $offset++) { + $date = $After.Date.AddDays($offset) + if ($days -notcontains [string]$date.DayOfWeek) { continue } + foreach ($time in $times) { + $candidate = $date + $time + if ($candidate -gt $After) { return $candidate } + } + } + return $null +} + +function Get-NextRun { + $next = $null + $policies = @() + try { $policies += @(Get-OBPolicy) } catch { Write-Log "Couldn't read the backup policy: $(Get-ErrorText $_)" 'WARN' } + # Windows Server can have a separate System State policy with its own schedule + if (Get-Command Get-OBSystemStatePolicy -ErrorAction SilentlyContinue) { + try { $policies += @(Get-OBSystemStatePolicy) } catch { } + } + foreach ($policy in $policies) { + if ($null -eq $policy) { continue } + try { + $slot = Get-NextSlot (Get-OBSchedule -Policy $policy) (Get-Date) + if ($slot -and (-not $next -or $slot -lt $next)) { $next = $slot } + } catch { + Write-Log "Couldn't read a policy schedule: $(Get-ErrorText $_)" 'WARN' + } + } + return ConvertTo-IsoTime $next +} + +# ---------- Jobs ---------- + +function Get-JobStatus($Job) { + $status = $Job.JobStatus + $end = Get-FirstValue $status @('EndTime') + $state = [string](Get-FirstValue $status @('JobState', 'State')) + if (-not (ConvertTo-IsoTime $end) -and $state -notmatch 'Fail|Abort|Cancel|Complete|Success|Succeed|Warn') { + return 'running' + } + if ($state -match 'Fail') { return 'failed' } + if ($state -match 'Warn|Abort|Cancel|Stop') { return 'warning' } + if ($state -match 'Complete|Success|Succeed') { + # MARS can report Completed while individual files failed or an error was recorded + if ((Get-FailedBytes $Job) -gt 0 -or (Get-FirstValue $status @('ErrorInfo'))) { return 'warning' } + return 'success' + } + return 'warning' # finished in a state we don't recognise: surface it rather than hide it +} + +function Get-DatasourceStatuses($Job) { + return @(Get-FirstValue $Job.JobStatus @('DatasourceStatus', 'DataSourceStatus') | Where-Object { $_ }) +} + +function Get-ByteTotal($Job, [string]$Field) { + $sum = [long]0 + foreach ($ds in Get-DatasourceStatuses $Job) { + $progress = Get-FirstValue $ds @('ByteProgress') + $value = Get-FirstValue $progress @($Field) + if ($value) { $sum += [long]$value } + } + return $sum +} + +function Get-FailedBytes($Job) { return Get-ByteTotal $Job 'Failed' } + +function Format-ErrorInfo($ErrorInfo) { + if (-not $ErrorInfo) { return $null } + $text = Get-FirstValue $ErrorInfo @('ErrorMessage', 'Message', 'Description') + if ($text) { return ([string]$text).Trim() } + $code = Get-FirstValue $ErrorInfo @('ErrorCode', 'DetailedErrorCode') + $params = @(Get-FirstValue $ErrorInfo @('ErrorParamList') | Where-Object { $_ } | ForEach-Object { + $name = Get-FirstValue $_ @('Name') + $value = Get-FirstValue $_ @('Value') + if ($name) { "$name=$value" } else { [string]$_ } + } | Where-Object { $_ }) + $line = if ($code) { "Error code $code" } else { [string]$ErrorInfo } + if ($params.Count) { $line += " ($($params -join ', '))" } + return $line +} + +function Get-RunMessage($Job, [string]$Status) { + $lines = New-Object 'System.Collections.Generic.List[string]' + $state = [string](Get-FirstValue $Job.JobStatus @('JobState', 'State')) + $jobError = Format-ErrorInfo (Get-FirstValue $Job.JobStatus @('ErrorInfo')) + if ($jobError) { $lines.Add($jobError) } + + foreach ($ds in Get-DatasourceStatuses $Job) { + $name = Get-FirstValue $ds @('Datasource', 'DataSource', 'DatasourceName', 'Name') + $dsError = Format-ErrorInfo (Get-FirstValue $ds @('ErrorInfo')) + $failed = Get-FirstValue (Get-FirstValue $ds @('ByteProgress')) @('Failed') + if ($dsError) { $lines.Add("${name}: $dsError") } + elseif ($failed -and [long]$failed -gt 0) { $lines.Add("${name}: $failed bytes failed to back up") } + } + + if ($lines.Count -eq 0) { + switch ($Status) { + 'success' { return 'Backup completed successfully.' } + 'running' { return '' } + default { return "Backup finished with state '$state'." } + } + } + if ($state -and $Status -ne 'success') { $lines.Insert(0, "Job state: $state") } + return ($lines -join "`n") +} + +function ConvertTo-Run($Job) { + $status = Get-JobStatus $Job + $datasources = @(Get-DatasourceStatuses $Job | ForEach-Object { + [string](Get-FirstValue $_ @('Datasource', 'DataSource', 'DatasourceName', 'Name')) + } | Where-Object { $_ }) + + $details = [ordered]@{ server = $env:COMPUTERNAME } + if ($datasources.Count) { $details.datasources = $datasources } + $agentState = Get-FirstValue $Job.JobStatus @('JobState', 'State') + if ($agentState) { $details.mars_state = [string]$agentState } + + $run = [ordered]@{ + external_id = [string]$Job.JobId + status = $status + started_at = ConvertTo-IsoTime (Get-FirstValue $Job.JobStatus @('StartTime')) + message = Get-RunMessage $Job $status + details = $details + } + if ($status -ne 'running') { + $finished = ConvertTo-IsoTime (Get-FirstValue $Job.JobStatus @('EndTime')) + if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time + $transferred = Get-ByteTotal $Job 'Progress' + if ($transferred -gt 0) { $run.size_bytes = $transferred } + $failedBytes = Get-FailedBytes $Job + if ($failedBytes -gt 0) { $details.failed_bytes = $failedBytes } + } + return $run +} + +function Test-IsBackupJob($Job) { + # Get-OBJob also lists restores (JobType Recovery); only backups belong on the dashboard + $type = [string](Get-FirstValue $Job @('JobType', 'Type')) + return (-not $type) -or $type -match 'Backup' +} + +function Send-Report($Body) { + $json = $Body | ConvertTo-Json -Depth 6 -Compress + if ($DryRun) { + Write-Host $json + return + } + # Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1 + $params = @{ + Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest" + Method = 'Post' + Headers = @{ Authorization = "Bearer $($Config.ApiKey)" } + ContentType = 'application/json; charset=utf-8' + Body = [Text.Encoding]::UTF8.GetBytes($json) + TimeoutSec = 30 + } + Invoke-RestMethod @params | Out-Null +} + +# ---------- Main ---------- + +try { + $Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json + foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') { + if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" } + } + $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Azure Backup (MARS)' } + $JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Azure Backup" } + if ($LookbackHours -le 0) { + $LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 } + } + + # Windows PowerShell 5.1 may default to TLS 1.0 + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + + Import-Mars + + # Job ID -> start time (Unix seconds), for jobs already delivered in their final state + $sent = @{} + if (Test-Path $StatePath) { + $saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json + if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } } + } + + $since = (Get-Date).AddHours(-$LookbackHours) + # Get-OBJob with no arguments lists jobs in progress; -Previous lists finished ones + $jobs = @(@(Get-OBJob) + @(Get-OBJob -Previous $MaxPreviousJobs) | Where-Object { + $_ -and (Test-IsBackupJob $_) + }) + $byId = [ordered]@{} + foreach ($job in $jobs) { + $start = Get-FirstValue $job.JobStatus @('StartTime') + if (-not $start -or [datetime]$start -lt $since) { continue } + $byId[[string]$job.JobId] = $job # a job in both lists: the finished entry (later) wins + } + $pending = @($byId.Values | + Where-Object { -not $sent.ContainsKey([string]$_.JobId) } | + Sort-Object { [datetime](Get-FirstValue $_.JobStatus @('StartTime')) }) + + $jobInfo = [ordered]@{ + # Stable per server, so renaming the dashboard job via JobName doesn't create a new one + external_id = "mars:$($env:COMPUTERNAME.ToLowerInvariant())" + name = $JobName + # Always sent (null when unscheduled) so the dashboard's value tracks the agent's + next_run_at = Get-NextRun + } +} catch { + Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR' + exit 2 +} + +$failures = 0 +$runsSent = 0 +try { + if ($pending.Count -eq 0) { + # Keep next_run_at fresh even when nothing new ran + Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo }) + } + foreach ($job in $pending) { + $run = ConvertTo-Run $job + Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run }) + $runsSent++ + if ($run.status -ne 'running') { + $sent[$run.external_id] = ([DateTimeOffset][datetime](Get-FirstValue $job.JobStatus @('StartTime'))).ToUnixTimeSeconds() + } + } +} catch { + $failures++ + Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR' +} + +if (-not $DryRun) { + # Forget jobs that have aged out of the lookback window (plus a day of slack) + $cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds() + $keep = [ordered]@{} + foreach ($id in $sent.Keys) { + if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] } + } + try { + $keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8 + } catch { + Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN' + } +} + +Write-Log ("Reported {0} runs, {1} failures" -f $runsSent, $failures) +if ($failures) { exit 1 } diff --git a/collectors/mars/tests/MockMars.ps1 b/collectors/mars/tests/MockMars.ps1 new file mode 100644 index 0000000..06edade --- /dev/null +++ b/collectors/mars/tests/MockMars.ps1 @@ -0,0 +1,56 @@ +# Stand-ins for the MARS agent cmdlets, shaped like the properties the collector reads. +# Property names are best guesses until checked against Get-MarsCollectorDiagnostics.ps1 output. +# +# . ./tests/MockMars.ps1 +# ./Send-MarsBackupStatus.ps1 -ConfigPath ./my-test-config.json -DryRun + +# Fixed to the top of the hour, and IDs derived from times, so repeated runs see the same jobs +$now = Get-Date -Minute 0 -Second 0 -Millisecond 0 +function New-StableGuid([string]$Seed) { + [guid]::new([Security.Cryptography.MD5]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($Seed))) +} + +function New-MockDatasource($Name, [long]$Done, [long]$Failed = 0, $ErrorInfo = $null) { + [pscustomobject]@{ + Datasource = $Name + ByteProgress = [pscustomobject]@{ Total = $Done + $Failed; Progress = $Done; Failed = $Failed } + ErrorInfo = $ErrorInfo + } +} + +function New-MockJob($Type, $Start, $End, $State, $Datasources = @(), $ErrorInfo = $null) { + [pscustomobject]@{ + JobId = New-StableGuid "$Type|$($Start.Ticks)" + JobType = $Type + JobStatus = [pscustomobject]@{ + JobState = $State; StartTime = $Start; EndTime = $End + DatasourceStatus = $Datasources; ErrorInfo = $ErrorInfo + } + } +} + +$global:MockMarsRunning = @( + (New-MockJob 'Backup' $now.AddMinutes(-10) $null 'InProgress' @((New-MockDatasource 'D:\Shares' 2GB))) +) +$global:MockMarsPrevious = @( + (New-MockJob 'Backup' $now.AddHours(-12) $now.AddHours(-11) 'Completed' @( + (New-MockDatasource 'D:\Shares' 12GB), (New-MockDatasource 'System State' 8GB))), + (New-MockJob 'Backup' $now.AddHours(-36) $now.AddHours(-35) 'Completed' @( + (New-MockDatasource 'D:\Shares' 3GB 40MB))), # completed, but some files failed + (New-MockJob 'Backup' $now.AddHours(-30) $now.AddHours(-29.5) 'Failed' @() ` + ([pscustomobject]@{ ErrorCode = 100050; ErrorParamList = @([pscustomobject]@{ Name = 'Volume'; Value = 'D:' }) })), + (New-MockJob 'Recovery' $now.AddHours(-5) $now.AddHours(-4) 'Completed'), # a restore: must be ignored + (New-MockJob 'Backup' $now.AddDays(-5) $now.AddDays(-5).AddHours(1) 'Completed') # outside 48h lookback +) + +# Daily at 21:00 and 03:00, weekdays only +$global:MockMarsPolicy = [pscustomobject]@{ PolicyName = 'Files' } +$global:MockMarsSchedule = [pscustomobject]@{ + ScheduleRunDays = @([DayOfWeek]::Monday, [DayOfWeek]::Tuesday, [DayOfWeek]::Wednesday, [DayOfWeek]::Thursday, [DayOfWeek]::Friday) + ScheduleRunTimes = @($now.Date.AddHours(21), $now.Date.AddHours(3)) + ScheduleWeeklyFrequency = 1 +} + +function Get-OBJob { param([int]$Previous) if ($PSBoundParameters.ContainsKey('Previous')) { $global:MockMarsPrevious } else { $global:MockMarsRunning } } +function Get-OBPolicy { $global:MockMarsPolicy } +function Get-OBSchedule { param($Policy) $global:MockMarsSchedule }