kyleandClaude Opus 5.5
5982fc9434
Checked with Get-MarsCollectorDiagnostics.ps1 output from MARS agent 2.0.9274 (VA-FS-01): - JobStatus.StartTime/EndTime are UTC values not marked as UTC (DisplayTime is the local one); they were read as local, putting runs 6 hours off. Lookback filtering now compares in UTC too - ErrorInfo is always present with ErrorCode 0 on success; every completed backup would have been a warning. Only non-zero error codes count - Schedule times are TimeSpans; casting them to DateTime failed, so next_run_at was always empty - DatasourceStatus.Datasource is an object; use its DataSourceName - Also report FileProgress.Failed file counts and FailedFileLog - Mock rebuilt on the real shapes; diagnostics handle no System State policy Co-Authored-By: Claude Opus 5.5 <[email protected]>
414 lines
17 KiB
PowerShell
414 lines
17 KiB
PowerShell
<#
|
|
.SYNOPSIS
|
|
Reports Microsoft Azure Recovery Services (MARS) agent backups to the Backup Dashboard.
|
|
|
|
.DESCRIPTION
|
|
Runs on a server with the MARS agent, normally as a SYSTEM scheduled task every 60 minutes
|
|
(see Install-MarsCollector.ps1). The server appears on the dashboard as one job. Each run sends:
|
|
- the next scheduled backup, worked out from the backup (and System State) policy schedules,
|
|
which drives the dashboard's Overdue flag, and
|
|
- every backup job from the last LookbackHours that the dashboard hasn't already received in
|
|
its final state. A backup in progress is sent as Running until it finishes.
|
|
|
|
Already-reported jobs are tracked in a state file, and the ingest API is idempotent, so a
|
|
failed or repeated run is harmless: anything not confirmed is retried next time.
|
|
|
|
Written for Windows PowerShell 5.1, which the MARS agent's MSOnlineBackup module targets.
|
|
|
|
.PARAMETER ConfigPath
|
|
JSON config with ApiUrl, ApiKey and Tenant (optional: JobName, SourceSystem, LookbackHours).
|
|
|
|
.PARAMETER LookbackHours
|
|
Overrides the config value. Use 720 once to backfill 30 days of history.
|
|
|
|
.PARAMETER DryRun
|
|
Print the payloads instead of sending them. Doesn't touch the state or log files.
|
|
|
|
.EXAMPLE
|
|
.\Send-MarsBackupStatus.ps1 -DryRun
|
|
.EXAMPLE
|
|
.\Send-MarsBackupStatus.ps1 -LookbackHours 720
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\mars-collector.json'),
|
|
[int]$LookbackHours = 0,
|
|
[switch]$DryRun
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
$DataDir = Split-Path -Parent $ConfigPath
|
|
$LogPath = Join-Path $DataDir 'mars-collector.log'
|
|
$StatePath = Join-Path $DataDir 'mars-collector.state.json'
|
|
$MaxLogBytes = 5MB
|
|
# MARS keeps a limited job history; this comfortably covers 30 days at 3 backups a day
|
|
$MaxPreviousJobs = 200
|
|
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
|
|
|
# ---------- Helpers ----------
|
|
|
|
function Write-Log {
|
|
param([string]$Message, [string]$Level = 'INFO')
|
|
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
|
|
Write-Host $line
|
|
if ($DryRun) { return }
|
|
if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) {
|
|
Move-Item $LogPath "$LogPath.1" -Force
|
|
}
|
|
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
|
}
|
|
|
|
function Get-ErrorText($ErrorRecord) {
|
|
# Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
|
|
if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
|
|
return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
|
|
}
|
|
return $ErrorRecord.Exception.Message
|
|
}
|
|
|
|
# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a
|
|
# Utc time as +00:00.
|
|
function ConvertTo-IsoTime($Value) {
|
|
if ($null -eq $Value) { return $null }
|
|
$dt = [datetime]$Value
|
|
if ($dt.Year -le 1900) { return $null } # "not set" (MARS uses 1601-01-01)
|
|
return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant)
|
|
}
|
|
|
|
# First property of $Object, out of $Names, that exists and isn't empty. MARS object property names
|
|
# aren't documented, so reads go through this rather than assuming one spelling.
|
|
function Get-FirstValue($Object, [string[]]$Names) {
|
|
if ($null -eq $Object) { return $null }
|
|
foreach ($name in $Names) {
|
|
try {
|
|
$prop = $Object.PSObject.Properties[$name]
|
|
if (-not $prop) { continue }
|
|
$value = $prop.Value
|
|
# Check emptiness by type: casting to [string] would make a one-item list of objects ''
|
|
if ($null -eq $value) { continue }
|
|
if ($value -is [string] -and $value -eq '') { continue }
|
|
if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue }
|
|
return $value
|
|
} catch { }
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Import-Mars {
|
|
if (Get-Command Get-OBJob -ErrorAction SilentlyContinue) { return } # already loaded
|
|
if (Get-Module -ListAvailable -Name MSOnlineBackup) {
|
|
Import-Module MSOnlineBackup
|
|
return
|
|
}
|
|
# The agent normally adds its module folder to PSModulePath; fall back to the default install path
|
|
$agentModule = Join-Path $env:ProgramFiles 'Microsoft Azure Recovery Services Agent\bin\Modules\MSOnlineBackup'
|
|
Import-Module $agentModule
|
|
}
|
|
|
|
# ---------- Schedule ----------
|
|
|
|
# Next slot strictly after $After for a MARS schedule (days of week + times of day).
|
|
# Returns $null when there's no schedule, or when it runs every N>1 weeks: we can't tell which
|
|
# week is next, and a wrong guess would raise a false Overdue alarm.
|
|
function Get-NextSlot($Schedule, [datetime]$After) {
|
|
if ($null -eq $Schedule) { return $null }
|
|
$weeks = Get-FirstValue $Schedule @('ScheduleWeeklyFrequency', 'WeeksInterval')
|
|
if ($weeks -and [int]$weeks -gt 1) { return $null }
|
|
|
|
$days = @(Get-FirstValue $Schedule @('ScheduleRunDays', 'DaysOfWeek') | ForEach-Object { [string]$_ })
|
|
# TimeSpans (time of day, local) on real agents; accept DateTimes too
|
|
$times = @(Get-FirstValue $Schedule @('ScheduleRunTimes', 'TimesOfDay') | ForEach-Object {
|
|
if ($_ -is [timespan]) { $_ } else { ([datetime]$_).TimeOfDay }
|
|
} | Sort-Object)
|
|
if ($days.Count -eq 0 -or $times.Count -eq 0) { return $null }
|
|
|
|
for ($offset = 0; $offset -le 7; $offset++) {
|
|
$date = $After.Date.AddDays($offset)
|
|
if ($days -notcontains [string]$date.DayOfWeek) { continue }
|
|
foreach ($time in $times) {
|
|
$candidate = $date + $time
|
|
if ($candidate -gt $After) { return $candidate }
|
|
}
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Get-NextRun {
|
|
$next = $null
|
|
$policies = @()
|
|
try { $policies += @(Get-OBPolicy) } catch { Write-Log "Couldn't read the backup policy: $(Get-ErrorText $_)" 'WARN' }
|
|
# Windows Server can have a separate System State policy with its own schedule
|
|
if (Get-Command Get-OBSystemStatePolicy -ErrorAction SilentlyContinue) {
|
|
try { $policies += @(Get-OBSystemStatePolicy) } catch { }
|
|
}
|
|
foreach ($policy in $policies) {
|
|
if ($null -eq $policy) { continue }
|
|
try {
|
|
$slot = Get-NextSlot (Get-OBSchedule -Policy $policy) (Get-Date)
|
|
if ($slot -and (-not $next -or $slot -lt $next)) { $next = $slot }
|
|
} catch {
|
|
Write-Log "Couldn't read a policy schedule: $(Get-ErrorText $_)" 'WARN'
|
|
}
|
|
}
|
|
return ConvertTo-IsoTime $next
|
|
}
|
|
|
|
# ---------- Jobs ----------
|
|
|
|
# A job's StartTime/EndTime are UTC values, though not necessarily marked as UTC: on a real agent
|
|
# StartTime matched the "StartTimeUTC" in the status text, while DisplayTime held local time.
|
|
# Returns $null for "not set" (1601-01-01), e.g. EndTime while a backup is running.
|
|
function Get-JobTimeUtc($Job, [string]$Field) {
|
|
$value = Get-FirstValue $Job.JobStatus @($Field)
|
|
if ($null -eq $value) { return $null }
|
|
$dt = [datetime]$value
|
|
if ($dt.Year -le 1900) { return $null }
|
|
return [datetime]::SpecifyKind($dt, [DateTimeKind]::Utc)
|
|
}
|
|
|
|
# MARS always includes an ErrorInfo; ErrorCode 0 means no error
|
|
function Test-HasError($ErrorInfo) {
|
|
if (-not $ErrorInfo) { return $false }
|
|
$code = Get-FirstValue $ErrorInfo @('ErrorCode')
|
|
$detailed = Get-FirstValue $ErrorInfo @('DetailedErrorCode')
|
|
return ($code -and [long]$code -ne 0) -or ($detailed -and [long]$detailed -ne 0)
|
|
}
|
|
|
|
# DatasourceStatus.Datasource is an object whose DataSourceName is e.g. "E:\" or "System State"
|
|
function Get-DatasourceName($DatasourceStatus) {
|
|
$ds = Get-FirstValue $DatasourceStatus @('Datasource', 'DataSource')
|
|
$name = Get-FirstValue $ds @('DataSourceName', 'FriendlyNameDSPS', 'Name')
|
|
if ($name) { return [string]$name }
|
|
if ($ds -is [string]) { return $ds }
|
|
return 'Datasource'
|
|
}
|
|
|
|
function Get-FailedFiles($Job) {
|
|
$sum = [long]0
|
|
foreach ($ds in Get-DatasourceStatuses $Job) {
|
|
$value = Get-FirstValue (Get-FirstValue $ds @('FileProgress')) @('Failed')
|
|
if ($value) { $sum += [long]$value }
|
|
}
|
|
return $sum
|
|
}
|
|
|
|
function Get-JobStatus($Job) {
|
|
$status = $Job.JobStatus
|
|
$state = [string](Get-FirstValue $status @('JobState', 'State'))
|
|
if (-not (Get-JobTimeUtc $Job 'EndTime') -and $state -notmatch 'Fail|Abort|Cancel|Complete|Success|Succeed|Warn') {
|
|
return 'running'
|
|
}
|
|
if ($state -match 'Fail') { return 'failed' }
|
|
if ($state -match 'Warn|Abort|Cancel|Stop') { return 'warning' }
|
|
if ($state -match 'Complete|Success|Succeed') {
|
|
# MARS can report Completed while individual files failed or an error was recorded
|
|
if ((Get-FailedBytes $Job) -gt 0 -or (Get-FailedFiles $Job) -gt 0 -or
|
|
(Test-HasError (Get-FirstValue $status @('ErrorInfo')))) { return 'warning' }
|
|
return 'success'
|
|
}
|
|
return 'warning' # finished in a state we don't recognise: surface it rather than hide it
|
|
}
|
|
|
|
function Get-DatasourceStatuses($Job) {
|
|
return @(Get-FirstValue $Job.JobStatus @('DatasourceStatus', 'DataSourceStatus') | Where-Object { $_ })
|
|
}
|
|
|
|
function Get-ByteTotal($Job, [string]$Field) {
|
|
$sum = [long]0
|
|
foreach ($ds in Get-DatasourceStatuses $Job) {
|
|
$progress = Get-FirstValue $ds @('ByteProgress')
|
|
$value = Get-FirstValue $progress @($Field)
|
|
if ($value) { $sum += [long]$value }
|
|
}
|
|
return $sum
|
|
}
|
|
|
|
function Get-FailedBytes($Job) { return Get-ByteTotal $Job 'Failed' }
|
|
|
|
function Format-ErrorInfo($ErrorInfo) {
|
|
if (-not (Test-HasError $ErrorInfo)) { return $null }
|
|
$text = Get-FirstValue $ErrorInfo @('ErrorMessage', 'Message', 'Description')
|
|
if ($text) { return ([string]$text).Trim() }
|
|
$code = Get-FirstValue $ErrorInfo @('ErrorCode', 'DetailedErrorCode')
|
|
$params = @(Get-FirstValue $ErrorInfo @('ErrorParamList') | Where-Object { $_ } | ForEach-Object {
|
|
$name = Get-FirstValue $_ @('Name')
|
|
$value = Get-FirstValue $_ @('Value')
|
|
if ($name) { "$name=$value" } else { [string]$_ }
|
|
} | Where-Object { $_ })
|
|
$line = if ($code) { "Error code $code" } else { [string]$ErrorInfo }
|
|
if ($params.Count) { $line += " ($($params -join ', '))" }
|
|
return $line
|
|
}
|
|
|
|
function Get-RunMessage($Job, [string]$Status) {
|
|
$lines = New-Object 'System.Collections.Generic.List[string]'
|
|
$state = [string](Get-FirstValue $Job.JobStatus @('JobState', 'State'))
|
|
$jobError = Format-ErrorInfo (Get-FirstValue $Job.JobStatus @('ErrorInfo'))
|
|
if ($jobError) { $lines.Add($jobError) }
|
|
|
|
foreach ($ds in Get-DatasourceStatuses $Job) {
|
|
$name = Get-DatasourceName $ds
|
|
$dsError = Format-ErrorInfo (Get-FirstValue $ds @('ErrorInfo'))
|
|
$failedFiles = Get-FirstValue (Get-FirstValue $ds @('FileProgress')) @('Failed')
|
|
$failedBytes = Get-FirstValue (Get-FirstValue $ds @('ByteProgress')) @('Failed')
|
|
if ($dsError) { $lines.Add("${name}: $dsError") }
|
|
if ($failedFiles -and [long]$failedFiles -gt 0) { $lines.Add("${name}: $failedFiles file(s) failed to back up") }
|
|
elseif ($failedBytes -and [long]$failedBytes -gt 0) { $lines.Add("${name}: $failedBytes bytes failed to back up") }
|
|
}
|
|
$failedLog = Get-FirstValue $Job.JobStatus @('FailedFileLog')
|
|
if ($failedLog -and $lines.Count) { $lines.Add("Failed file log: $failedLog") }
|
|
|
|
if ($lines.Count -eq 0) {
|
|
switch ($Status) {
|
|
'success' { return 'Backup completed successfully.' }
|
|
'running' { return '' }
|
|
default { return "Backup finished with state '$state'." }
|
|
}
|
|
}
|
|
if ($state -and $Status -ne 'success') { $lines.Insert(0, "Job state: $state") }
|
|
return ($lines -join "`n")
|
|
}
|
|
|
|
function ConvertTo-Run($Job) {
|
|
$status = Get-JobStatus $Job
|
|
$datasources = @(Get-DatasourceStatuses $Job | ForEach-Object { Get-DatasourceName $_ })
|
|
|
|
$details = [ordered]@{ server = $env:COMPUTERNAME }
|
|
if ($datasources.Count) { $details.datasources = $datasources }
|
|
$agentState = Get-FirstValue $Job.JobStatus @('JobState', 'State')
|
|
if ($agentState) { $details.mars_state = [string]$agentState }
|
|
|
|
$run = [ordered]@{
|
|
external_id = [string]$Job.JobId
|
|
status = $status
|
|
started_at = ConvertTo-IsoTime (Get-JobTimeUtc $Job 'StartTime')
|
|
message = Get-RunMessage $Job $status
|
|
details = $details
|
|
}
|
|
if ($status -ne 'running') {
|
|
$finished = ConvertTo-IsoTime (Get-JobTimeUtc $Job 'EndTime')
|
|
if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time
|
|
$transferred = Get-ByteTotal $Job 'Progress'
|
|
if ($transferred -gt 0) { $run.size_bytes = $transferred }
|
|
$failedBytes = Get-FailedBytes $Job
|
|
if ($failedBytes -gt 0) { $details.failed_bytes = $failedBytes }
|
|
$failedFiles = Get-FailedFiles $Job
|
|
if ($failedFiles -gt 0) { $details.failed_files = $failedFiles }
|
|
}
|
|
return $run
|
|
}
|
|
|
|
function Test-IsBackupJob($Job) {
|
|
# Get-OBJob also lists restores (JobType Recovery); only backups belong on the dashboard
|
|
$type = [string](Get-FirstValue $Job @('JobType', 'Type'))
|
|
return (-not $type) -or $type -match 'Backup'
|
|
}
|
|
|
|
function Send-Report($Body) {
|
|
$json = $Body | ConvertTo-Json -Depth 6 -Compress
|
|
if ($DryRun) {
|
|
Write-Host $json
|
|
return
|
|
}
|
|
# Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1
|
|
$params = @{
|
|
Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
|
|
Method = 'Post'
|
|
Headers = @{ Authorization = "Bearer $($Config.ApiKey)" }
|
|
ContentType = 'application/json; charset=utf-8'
|
|
Body = [Text.Encoding]::UTF8.GetBytes($json)
|
|
TimeoutSec = 30
|
|
}
|
|
Invoke-RestMethod @params | Out-Null
|
|
}
|
|
|
|
# ---------- Main ----------
|
|
|
|
try {
|
|
$Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
|
|
foreach ($key in 'ApiUrl', 'ApiKey', 'Tenant') {
|
|
if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" }
|
|
}
|
|
$SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Azure Backup (MARS)' }
|
|
$JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Azure Backup" }
|
|
if ($LookbackHours -le 0) {
|
|
$LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 }
|
|
}
|
|
|
|
# Windows PowerShell 5.1 may default to TLS 1.0
|
|
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
|
|
|
Import-Mars
|
|
|
|
# Job ID -> start time (Unix seconds), for jobs already delivered in their final state
|
|
$sent = @{}
|
|
if (Test-Path $StatePath) {
|
|
$saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json
|
|
if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } }
|
|
}
|
|
|
|
$sinceUtc = (Get-Date).ToUniversalTime().AddHours(-$LookbackHours)
|
|
# Get-OBJob with no arguments lists jobs in progress; -Previous lists finished ones
|
|
$jobs = @(@(Get-OBJob) + @(Get-OBJob -Previous $MaxPreviousJobs) | Where-Object {
|
|
$_ -and (Test-IsBackupJob $_)
|
|
})
|
|
$byId = [ordered]@{}
|
|
foreach ($job in $jobs) {
|
|
$start = Get-JobTimeUtc $job 'StartTime'
|
|
if (-not $start -or $start -lt $sinceUtc) { continue }
|
|
$byId[[string]$job.JobId] = $job # a job in both lists: the finished entry (later) wins
|
|
}
|
|
$pending = @($byId.Values |
|
|
Where-Object { -not $sent.ContainsKey([string]$_.JobId) } |
|
|
Sort-Object { Get-JobTimeUtc $_ 'StartTime' })
|
|
|
|
$jobInfo = [ordered]@{
|
|
# Stable per server, so renaming the dashboard job via JobName doesn't create a new one
|
|
external_id = "mars:$($env:COMPUTERNAME.ToLowerInvariant())"
|
|
name = $JobName
|
|
# Always sent (null when unscheduled) so the dashboard's value tracks the agent's
|
|
next_run_at = Get-NextRun
|
|
}
|
|
} catch {
|
|
Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR'
|
|
exit 2
|
|
}
|
|
|
|
$failures = 0
|
|
$runsSent = 0
|
|
try {
|
|
if ($pending.Count -eq 0) {
|
|
# Keep next_run_at fresh even when nothing new ran
|
|
Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo })
|
|
}
|
|
foreach ($job in $pending) {
|
|
$run = ConvertTo-Run $job
|
|
Send-Report ([ordered]@{ tenant = $Config.Tenant; source_system = $SourceSystem; job = $jobInfo; run = $run })
|
|
$runsSent++
|
|
if ($run.status -ne 'running') {
|
|
$sent[$run.external_id] = ([DateTimeOffset](Get-JobTimeUtc $job 'StartTime')).ToUnixTimeSeconds()
|
|
}
|
|
}
|
|
} catch {
|
|
$failures++
|
|
Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR'
|
|
}
|
|
|
|
if (-not $DryRun) {
|
|
# Forget jobs that have aged out of the lookback window (plus a day of slack)
|
|
$cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds()
|
|
$keep = [ordered]@{}
|
|
foreach ($id in $sent.Keys) {
|
|
if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] }
|
|
}
|
|
try {
|
|
$keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8
|
|
} catch {
|
|
Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
|
|
}
|
|
}
|
|
|
|
Write-Log ("Reported {0} runs, {1} failures" -f $runsSent, $failures)
|
|
if ($failures) { exit 1 }
|