<# .SYNOPSIS Reports Microsoft Azure Recovery Services (MARS) agent backups to the Backup Dashboard. .DESCRIPTION Runs on a server with the MARS agent, normally as a SYSTEM scheduled task every 60 minutes (see Install-MarsCollector.ps1). The server appears on the dashboard as one job. Each run sends: - the next scheduled backup, worked out from the backup (and System State) policy schedules, which drives the dashboard's Overdue flag, and - every backup job from the last LookbackHours that the dashboard hasn't already received in its final state. A backup in progress is sent as Running until it finishes. Already-reported jobs are tracked in a state file, and the ingest API is idempotent, so a failed or repeated run is harmless: anything not confirmed is retried next time. Written for Windows PowerShell 5.1, which the MARS agent's MSOnlineBackup module targets. .PARAMETER ConfigPath JSON config with ApiUrl, ApiKey and Tenant (optional: JobName, SourceSystem, LookbackHours). .PARAMETER LookbackHours Overrides the config value. Use 720 once to backfill 30 days of history. .PARAMETER DryRun Print the payloads instead of sending them. Doesn't touch the state or log files. .EXAMPLE .\Send-MarsBackupStatus.ps1 -DryRun .EXAMPLE .\Send-MarsBackupStatus.ps1 -LookbackHours 720 #> [CmdletBinding()] param( [string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\mars-collector.json'), [int]$LookbackHours = 0, [switch]$DryRun ) $ErrorActionPreference = 'Stop' # Shared collector code: installed next to this script; in the repo, in ..\common $sharedModule = @( (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'), (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1') ) | Where-Object { Test-Path $_ } | Select-Object -First 1 if (-not $sharedModule) { Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer." exit 2 } Import-Module $sharedModule -Force # MARS keeps a limited job history; this comfortably covers 30 days at 3 backups a day $MaxPreviousJobs = 200 # ---------- Helpers ---------- function Import-Mars { if (Get-Command Get-OBJob -ErrorAction SilentlyContinue) { return } # already loaded if (Get-Module -ListAvailable -Name MSOnlineBackup) { Import-Module MSOnlineBackup return } # The agent normally adds its module folder to PSModulePath; fall back to the default install path $agentModule = Join-Path $env:ProgramFiles 'Microsoft Azure Recovery Services Agent\bin\Modules\MSOnlineBackup' Import-Module $agentModule } # ---------- Schedule ---------- # Next slot strictly after $After for a MARS schedule (days of week + times of day). # Returns $null when there's no schedule, or when it runs every N>1 weeks: we can't tell which # week is next, and a wrong guess would raise a false Overdue alarm. function Get-NextSlot($Schedule, [datetime]$After) { if ($null -eq $Schedule) { return $null } $weeks = Get-FirstValue $Schedule @('ScheduleWeeklyFrequency', 'WeeksInterval') if ($weeks -and [int]$weeks -gt 1) { return $null } $days = @(Get-FirstValue $Schedule @('ScheduleRunDays', 'DaysOfWeek') | ForEach-Object { [string]$_ }) # TimeSpans (time of day, local) on real agents; accept DateTimes too $times = @(Get-FirstValue $Schedule @('ScheduleRunTimes', 'TimesOfDay') | ForEach-Object { if ($_ -is [timespan]) { $_ } else { ([datetime]$_).TimeOfDay } } | Sort-Object) if ($days.Count -eq 0 -or $times.Count -eq 0) { return $null } for ($offset = 0; $offset -le 7; $offset++) { $date = $After.Date.AddDays($offset) if ($days -notcontains [string]$date.DayOfWeek) { continue } foreach ($time in $times) { $candidate = $date + $time if ($candidate -gt $After) { return $candidate } } } return $null } function Get-NextRun { $next = $null $policies = @() try { $policies += @(Get-OBPolicy) } catch { Write-Log "Couldn't read the backup policy: $(Get-ErrorText $_)" 'WARN' } # Windows Server can have a separate System State policy with its own schedule if (Get-Command Get-OBSystemStatePolicy -ErrorAction SilentlyContinue) { try { $policies += @(Get-OBSystemStatePolicy) } catch { } } foreach ($policy in $policies) { if ($null -eq $policy) { continue } try { $slot = Get-NextSlot (Get-OBSchedule -Policy $policy) (Get-Date) if ($slot -and (-not $next -or $slot -lt $next)) { $next = $slot } } catch { Write-Log "Couldn't read a policy schedule: $(Get-ErrorText $_)" 'WARN' } } return ConvertTo-IsoTime $next } # ---------- Jobs ---------- # A job's StartTime/EndTime are UTC values, though not necessarily marked as UTC: on a real agent # StartTime matched the "StartTimeUTC" in the status text, while DisplayTime held local time. # Returns $null for "not set" (1601-01-01), e.g. EndTime while a backup is running. function Get-JobTimeUtc($Job, [string]$Field) { $value = Get-FirstValue $Job.JobStatus @($Field) if ($null -eq $value) { return $null } $dt = [datetime]$value if ($dt.Year -le 1900) { return $null } return [datetime]::SpecifyKind($dt, [DateTimeKind]::Utc) } # MARS always includes an ErrorInfo; ErrorCode 0 means no error function Test-HasError($ErrorInfo) { if (-not $ErrorInfo) { return $false } $code = Get-FirstValue $ErrorInfo @('ErrorCode') $detailed = Get-FirstValue $ErrorInfo @('DetailedErrorCode') return ($code -and [long]$code -ne 0) -or ($detailed -and [long]$detailed -ne 0) } # DatasourceStatus.Datasource is an object whose DataSourceName is e.g. "E:\" or "System State" function Get-DatasourceName($DatasourceStatus) { $ds = Get-FirstValue $DatasourceStatus @('Datasource', 'DataSource') $name = Get-FirstValue $ds @('DataSourceName', 'FriendlyNameDSPS', 'Name') if ($name) { return [string]$name } if ($ds -is [string]) { return $ds } return 'Datasource' } function Get-FailedFiles($Job) { $sum = [long]0 foreach ($ds in Get-DatasourceStatuses $Job) { $value = Get-FirstValue (Get-FirstValue $ds @('FileProgress')) @('Failed') if ($value) { $sum += [long]$value } } return $sum } function Get-JobStatus($Job) { $status = $Job.JobStatus $state = [string](Get-FirstValue $status @('JobState', 'State')) if (-not (Get-JobTimeUtc $Job 'EndTime') -and $state -notmatch 'Fail|Abort|Cancel|Complete|Success|Succeed|Warn') { return 'running' } if ($state -match 'Fail') { return 'failed' } if ($state -match 'Warn|Abort|Cancel|Stop') { return 'warning' } if ($state -match 'Complete|Success|Succeed') { # MARS can report Completed while individual files failed or an error was recorded if ((Get-FailedBytes $Job) -gt 0 -or (Get-FailedFiles $Job) -gt 0 -or (Test-HasError (Get-FirstValue $status @('ErrorInfo')))) { return 'warning' } return 'success' } return 'warning' # finished in a state we don't recognise: surface it rather than hide it } function Get-DatasourceStatuses($Job) { return @(Get-FirstValue $Job.JobStatus @('DatasourceStatus', 'DataSourceStatus') | Where-Object { $_ }) } function Get-ByteTotal($Job, [string]$Field) { $sum = [long]0 foreach ($ds in Get-DatasourceStatuses $Job) { $progress = Get-FirstValue $ds @('ByteProgress') $value = Get-FirstValue $progress @($Field) if ($value) { $sum += [long]$value } } return $sum } function Get-FailedBytes($Job) { return Get-ByteTotal $Job 'Failed' } function Format-ErrorInfo($ErrorInfo) { if (-not (Test-HasError $ErrorInfo)) { return $null } $text = Get-FirstValue $ErrorInfo @('ErrorMessage', 'Message', 'Description') if ($text) { return ([string]$text).Trim() } $code = Get-FirstValue $ErrorInfo @('ErrorCode', 'DetailedErrorCode') $params = @(Get-FirstValue $ErrorInfo @('ErrorParamList') | Where-Object { $_ } | ForEach-Object { $name = Get-FirstValue $_ @('Name') $value = Get-FirstValue $_ @('Value') if ($name) { "$name=$value" } else { [string]$_ } } | Where-Object { $_ }) $line = if ($code) { "Error code $code" } else { [string]$ErrorInfo } if ($params.Count) { $line += " ($($params -join ', '))" } return $line } function Get-RunMessage($Job, [string]$Status) { $lines = New-Object 'System.Collections.Generic.List[string]' $state = [string](Get-FirstValue $Job.JobStatus @('JobState', 'State')) $jobError = Format-ErrorInfo (Get-FirstValue $Job.JobStatus @('ErrorInfo')) if ($jobError) { $lines.Add($jobError) } foreach ($ds in Get-DatasourceStatuses $Job) { $name = Get-DatasourceName $ds $dsError = Format-ErrorInfo (Get-FirstValue $ds @('ErrorInfo')) $failedFiles = Get-FirstValue (Get-FirstValue $ds @('FileProgress')) @('Failed') $failedBytes = Get-FirstValue (Get-FirstValue $ds @('ByteProgress')) @('Failed') if ($dsError) { $lines.Add("${name}: $dsError") } if ($failedFiles -and [long]$failedFiles -gt 0) { $lines.Add("${name}: $failedFiles file(s) failed to back up") } elseif ($failedBytes -and [long]$failedBytes -gt 0) { $lines.Add("${name}: $failedBytes bytes failed to back up") } } $failedLog = Get-FirstValue $Job.JobStatus @('FailedFileLog') if ($failedLog -and $lines.Count) { $lines.Add("Failed file log: $failedLog") } if ($lines.Count -eq 0) { switch ($Status) { 'success' { return 'Backup completed successfully.' } 'running' { return '' } default { return "Backup finished with state '$state'." } } } if ($state -and $Status -ne 'success') { $lines.Insert(0, "Job state: $state") } return ($lines -join "`n") } function ConvertTo-Run($Job) { $status = Get-JobStatus $Job $datasources = @(Get-DatasourceStatuses $Job | ForEach-Object { Get-DatasourceName $_ }) $details = [ordered]@{ server = $env:COMPUTERNAME } if ($datasources.Count) { $details.datasources = $datasources } $agentState = Get-FirstValue $Job.JobStatus @('JobState', 'State') if ($agentState) { $details.mars_state = [string]$agentState } $run = [ordered]@{ external_id = [string]$Job.JobId status = $status started_at = ConvertTo-IsoTime (Get-JobTimeUtc $Job 'StartTime') message = Get-RunMessage $Job $status details = $details } if ($status -ne 'running') { $finished = ConvertTo-IsoTime (Get-JobTimeUtc $Job 'EndTime') if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time $transferred = Get-ByteTotal $Job 'Progress' if ($transferred -gt 0) { $run.size_bytes = $transferred } $failedBytes = Get-FailedBytes $Job if ($failedBytes -gt 0) { $details.failed_bytes = $failedBytes } $failedFiles = Get-FailedFiles $Job if ($failedFiles -gt 0) { $details.failed_files = $failedFiles } } return $run } function Test-IsBackupJob($Job) { # Get-OBJob also lists restores (JobType Recovery); only backups belong on the dashboard $type = [string](Get-FirstValue $Job @('JobType', 'Type')) return (-not $type) -or $type -match 'Backup' } # ---------- Main ---------- try { $Config = Initialize-Collector -Name 'mars' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Azure Backup (MARS)' } $JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Azure Backup" } $LookbackHours = $Config.LookbackHours Import-Mars $sinceUtc = (Get-Date).ToUniversalTime().AddHours(-$LookbackHours) # Get-OBJob with no arguments lists jobs in progress; -Previous lists finished ones $jobs = @(@(Get-OBJob) + @(Get-OBJob -Previous $MaxPreviousJobs) | Where-Object { $_ -and (Test-IsBackupJob $_) }) $byId = [ordered]@{} foreach ($job in $jobs) { $start = Get-JobTimeUtc $job 'StartTime' if (-not $start -or $start -lt $sinceUtc) { continue } $byId[[string]$job.JobId] = $job # a job in both lists: the finished entry (later) wins } $pending = @($byId.Values | Where-Object { -not (Test-RunSent ([string]$_.JobId)) } | Sort-Object { Get-JobTimeUtc $_ 'StartTime' }) $jobInfo = [ordered]@{ # Stable per server, so renaming the dashboard job via JobName doesn't create a new one external_id = "mars:$($env:COMPUTERNAME.ToLowerInvariant())" name = $JobName # Always sent (null when unscheduled) so the dashboard's value tracks the agent's next_run_at = Get-NextRun } } catch { Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR' exit 2 } $failures = 0 try { if ($pending.Count -eq 0) { # Keep next_run_at fresh even when nothing new ran Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo } foreach ($job in $pending) { $run = ConvertTo-Run $job Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.external_id -StartedAt ((Get-JobTimeUtc $job 'StartTime')) } } catch { $failures++ Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR' } $exitCode = Complete-Collector -Failures $failures if ($exitCode) { exit $exitCode }