Backup-Dashboard/collectors/mars/Send-MarsBackupStatus.ps1

<#
.SYNOPSIS
    Reports Microsoft Azure Recovery Services (MARS) agent backups to the Backup Dashboard.

.DESCRIPTION
    Runs on a server with the MARS agent, normally as a SYSTEM scheduled task every 60 minutes
    (see Install-MarsCollector.ps1). The server appears on the dashboard as one job. Each run sends:
      - the next scheduled backup, worked out from the backup (and System State) policy schedules,
        which drives the dashboard's Overdue flag, and
      - every backup job from the last LookbackHours that the dashboard hasn't already received in
        its final state. A backup in progress is sent as Running until it finishes.

    Already-reported jobs are tracked in a state file, and the ingest API is idempotent, so a
    failed or repeated run is harmless: anything not confirmed is retried next time.

    Written for Windows PowerShell 5.1, which the MARS agent's MSOnlineBackup module targets.

.PARAMETER ConfigPath
    JSON config with ApiUrl, ApiKey and Tenant (optional: JobName, SourceSystem, LookbackHours).

.PARAMETER LookbackHours
    Overrides the config value. Use 720 once to backfill 30 days of history.

.PARAMETER DryRun
    Print the payloads instead of sending them. Doesn't touch the state or log files.

.EXAMPLE
    .\Send-MarsBackupStatus.ps1 -DryRun
.EXAMPLE
    .\Send-MarsBackupStatus.ps1 -LookbackHours 720
#>
[CmdletBinding()]
param(
    [string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\mars-collector.json'),
    [int]$LookbackHours = 0,
    [switch]$DryRun
)

$ErrorActionPreference = 'Stop'

# Shared collector code: installed next to this script; in the repo, in ..\common
$sharedModule = @(
    (Join-Path $PSScriptRoot 'BackupDashboard.Collector.psm1'),
    (Join-Path (Join-Path (Join-Path $PSScriptRoot '..') 'common') 'BackupDashboard.Collector.psm1')
) | Where-Object { Test-Path $_ } | Select-Object -First 1
if (-not $sharedModule) {
    Write-Host "BackupDashboard.Collector.psm1 not found next to $PSCommandPath; re-run the installer."
    exit 2
}
Import-Module $sharedModule -Force

# MARS keeps a limited job history; this comfortably covers 30 days at 3 backups a day
$MaxPreviousJobs = 200

# ---------- Helpers ----------

function Import-Mars {
    if (Get-Command Get-OBJob -ErrorAction SilentlyContinue) { return }  # already loaded
    if (Get-Module -ListAvailable -Name MSOnlineBackup) {
        Import-Module MSOnlineBackup
        return
    }
    # The agent normally adds its module folder to PSModulePath; fall back to the default install path
    $agentModule = Join-Path $env:ProgramFiles 'Microsoft Azure Recovery Services Agent\bin\Modules\MSOnlineBackup'
    Import-Module $agentModule
}

# ---------- Schedule ----------

# Next slot strictly after $After for a MARS schedule (days of week + times of day).
# Returns $null when there's no schedule, or when it runs every N>1 weeks: we can't tell which
# week is next, and a wrong guess would raise a false Overdue alarm.
function Get-NextSlot($Schedule, [datetime]$After) {
    if ($null -eq $Schedule) { return $null }
    $weeks = Get-FirstValue $Schedule @('ScheduleWeeklyFrequency', 'WeeksInterval')
    if ($weeks -and [int]$weeks -gt 1) { return $null }

    $days = @(Get-FirstValue $Schedule @('ScheduleRunDays', 'DaysOfWeek') | ForEach-Object { [string]$_ })
    # TimeSpans (time of day, local) on real agents; accept DateTimes too
    $times = @(Get-FirstValue $Schedule @('ScheduleRunTimes', 'TimesOfDay') | ForEach-Object {
        if ($_ -is [timespan]) { $_ } else { ([datetime]$_).TimeOfDay }
    } | Sort-Object)
    if ($days.Count -eq 0 -or $times.Count -eq 0) { return $null }

    for ($offset = 0; $offset -le 7; $offset++) {
        $date = $After.Date.AddDays($offset)
        if ($days -notcontains [string]$date.DayOfWeek) { continue }
        foreach ($time in $times) {
            $candidate = $date + $time
            if ($candidate -gt $After) { return $candidate }
        }
    }
    return $null
}

function Get-NextRun {
    $next = $null
    $policies = @()
    try { $policies += @(Get-OBPolicy) } catch { Write-Log "Couldn't read the backup policy: $(Get-ErrorText $_)" 'WARN' }
    # Windows Server can have a separate System State policy with its own schedule
    if (Get-Command Get-OBSystemStatePolicy -ErrorAction SilentlyContinue) {
        try { $policies += @(Get-OBSystemStatePolicy) } catch { }
    }
    foreach ($policy in $policies) {
        if ($null -eq $policy) { continue }
        try {
            $slot = Get-NextSlot (Get-OBSchedule -Policy $policy) (Get-Date)
            if ($slot -and (-not $next -or $slot -lt $next)) { $next = $slot }
        } catch {
            Write-Log "Couldn't read a policy schedule: $(Get-ErrorText $_)" 'WARN'
        }
    }
    return ConvertTo-IsoTime $next
}

# ---------- Jobs ----------

# A job's StartTime/EndTime are UTC values, though not necessarily marked as UTC: on a real agent
# StartTime matched the "StartTimeUTC" in the status text, while DisplayTime held local time.
# Returns $null for "not set" (1601-01-01), e.g. EndTime while a backup is running.
function Get-JobTimeUtc($Job, [string]$Field) {
    $value = Get-FirstValue $Job.JobStatus @($Field)
    if ($null -eq $value) { return $null }
    $dt = [datetime]$value
    if ($dt.Year -le 1900) { return $null }
    return [datetime]::SpecifyKind($dt, [DateTimeKind]::Utc)
}

# MARS always includes an ErrorInfo; ErrorCode 0 means no error
function Test-HasError($ErrorInfo) {
    if (-not $ErrorInfo) { return $false }
    $code = Get-FirstValue $ErrorInfo @('ErrorCode')
    $detailed = Get-FirstValue $ErrorInfo @('DetailedErrorCode')
    return ($code -and [long]$code -ne 0) -or ($detailed -and [long]$detailed -ne 0)
}

# DatasourceStatus.Datasource is an object whose DataSourceName is e.g. "E:\" or "System State"
function Get-DatasourceName($DatasourceStatus) {
    $ds = Get-FirstValue $DatasourceStatus @('Datasource', 'DataSource')
    $name = Get-FirstValue $ds @('DataSourceName', 'FriendlyNameDSPS', 'Name')
    if ($name) { return [string]$name }
    if ($ds -is [string]) { return $ds }
    return 'Datasource'
}

function Get-FailedFiles($Job) {
    $sum = [long]0
    foreach ($ds in Get-DatasourceStatuses $Job) {
        $value = Get-FirstValue (Get-FirstValue $ds @('FileProgress')) @('Failed')
        if ($value) { $sum += [long]$value }
    }
    return $sum
}

function Get-JobStatus($Job) {
    $status = $Job.JobStatus
    $state = [string](Get-FirstValue $status @('JobState', 'State'))
    if (-not (Get-JobTimeUtc $Job 'EndTime') -and $state -notmatch 'Fail|Abort|Cancel|Complete|Success|Succeed|Warn') {
        return 'running'
    }
    if ($state -match 'Fail') { return 'failed' }
    if ($state -match 'Warn|Abort|Cancel|Stop') { return 'warning' }
    if ($state -match 'Complete|Success|Succeed') {
        # MARS can report Completed while individual files failed or an error was recorded
        if ((Get-FailedBytes $Job) -gt 0 -or (Get-FailedFiles $Job) -gt 0 -or
            (Test-HasError (Get-FirstValue $status @('ErrorInfo')))) { return 'warning' }
        return 'success'
    }
    return 'warning'  # finished in a state we don't recognise: surface it rather than hide it
}

function Get-DatasourceStatuses($Job) {
    return @(Get-FirstValue $Job.JobStatus @('DatasourceStatus', 'DataSourceStatus') | Where-Object { $_ })
}

function Get-ByteTotal($Job, [string]$Field) {
    $sum = [long]0
    foreach ($ds in Get-DatasourceStatuses $Job) {
        $progress = Get-FirstValue $ds @('ByteProgress')
        $value = Get-FirstValue $progress @($Field)
        if ($value) { $sum += [long]$value }
    }
    return $sum
}

function Get-FailedBytes($Job) { return Get-ByteTotal $Job 'Failed' }

function Format-ErrorInfo($ErrorInfo) {
    if (-not (Test-HasError $ErrorInfo)) { return $null }
    $text = Get-FirstValue $ErrorInfo @('ErrorMessage', 'Message', 'Description')
    if ($text) { return ([string]$text).Trim() }
    $code = Get-FirstValue $ErrorInfo @('ErrorCode', 'DetailedErrorCode')
    $params = @(Get-FirstValue $ErrorInfo @('ErrorParamList') | Where-Object { $_ } | ForEach-Object {
        $name = Get-FirstValue $_ @('Name')
        $value = Get-FirstValue $_ @('Value')
        if ($name) { "$name=$value" } else { [string]$_ }
    } | Where-Object { $_ })
    $line = if ($code) { "Error code $code" } else { [string]$ErrorInfo }
    if ($params.Count) { $line += " ($($params -join ', '))" }
    return $line
}

function Get-RunMessage($Job, [string]$Status) {
    $lines = New-Object 'System.Collections.Generic.List[string]'
    $state = [string](Get-FirstValue $Job.JobStatus @('JobState', 'State'))
    $jobError = Format-ErrorInfo (Get-FirstValue $Job.JobStatus @('ErrorInfo'))
    if ($jobError) { $lines.Add($jobError) }

    foreach ($ds in Get-DatasourceStatuses $Job) {
        $name = Get-DatasourceName $ds
        $dsError = Format-ErrorInfo (Get-FirstValue $ds @('ErrorInfo'))
        $failedFiles = Get-FirstValue (Get-FirstValue $ds @('FileProgress')) @('Failed')
        $failedBytes = Get-FirstValue (Get-FirstValue $ds @('ByteProgress')) @('Failed')
        if ($dsError) { $lines.Add("${name}: $dsError") }
        if ($failedFiles -and [long]$failedFiles -gt 0) { $lines.Add("${name}: $failedFiles file(s) failed to back up") }
        elseif ($failedBytes -and [long]$failedBytes -gt 0) { $lines.Add("${name}: $failedBytes bytes failed to back up") }
    }
    $failedLog = Get-FirstValue $Job.JobStatus @('FailedFileLog')
    if ($failedLog -and $lines.Count) { $lines.Add("Failed file log: $failedLog") }

    if ($lines.Count -eq 0) {
        switch ($Status) {
            'success' { return 'Backup completed successfully.' }
            'running' { return '' }
            default   { return "Backup finished with state '$state'." }
        }
    }
    if ($state -and $Status -ne 'success') { $lines.Insert(0, "Job state: $state") }
    return ($lines -join "`n")
}

function ConvertTo-Run($Job) {
    $status = Get-JobStatus $Job
    $datasources = @(Get-DatasourceStatuses $Job | ForEach-Object { Get-DatasourceName $_ })

    $details = [ordered]@{ server = $env:COMPUTERNAME }
    if ($datasources.Count) { $details.datasources = $datasources }
    $agentState = Get-FirstValue $Job.JobStatus @('JobState', 'State')
    if ($agentState) { $details.mars_state = [string]$agentState }

    $run = [ordered]@{
        external_id = [string]$Job.JobId
        status      = $status
        started_at  = ConvertTo-IsoTime (Get-JobTimeUtc $Job 'StartTime')
        message     = Get-RunMessage $Job $status
        details     = $details
    }
    if ($status -ne 'running') {
        $finished = ConvertTo-IsoTime (Get-JobTimeUtc $Job 'EndTime')
        if ($finished) { $run.finished_at = $finished }  # otherwise the API uses receipt time
        $transferred = Get-ByteTotal $Job 'Progress'
        if ($transferred -gt 0) { $run.size_bytes = $transferred }
        $failedBytes = Get-FailedBytes $Job
        if ($failedBytes -gt 0) { $details.failed_bytes = $failedBytes }
        $failedFiles = Get-FailedFiles $Job
        if ($failedFiles -gt 0) { $details.failed_files = $failedFiles }
    }
    return $run
}

function Test-IsBackupJob($Job) {
    # Get-OBJob also lists restores (JobType Recovery); only backups belong on the dashboard
    $type = [string](Get-FirstValue $Job @('JobType', 'Type'))
    return (-not $type) -or $type -match 'Backup'
}

# ---------- Main ----------

try {
    $Config = Initialize-Collector -Name 'mars' -ConfigPath $ConfigPath -LookbackHours $LookbackHours -DryRun:$DryRun
    $SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Azure Backup (MARS)' }
    $JobName = if ($Config.JobName) { [string]$Config.JobName } else { "$env:COMPUTERNAME - Azure Backup" }
    $LookbackHours = $Config.LookbackHours

    Import-Mars

    $sinceUtc = (Get-Date).ToUniversalTime().AddHours(-$LookbackHours)
    # Get-OBJob with no arguments lists jobs in progress; -Previous lists finished ones
    $jobs = @(@(Get-OBJob) + @(Get-OBJob -Previous $MaxPreviousJobs) | Where-Object {
        $_ -and (Test-IsBackupJob $_)
    })
    $byId = [ordered]@{}
    foreach ($job in $jobs) {
        $start = Get-JobTimeUtc $job 'StartTime'
        if (-not $start -or $start -lt $sinceUtc) { continue }
        $byId[[string]$job.JobId] = $job  # a job in both lists: the finished entry (later) wins
    }
    $pending = @($byId.Values |
        Where-Object { -not (Test-RunSent ([string]$_.JobId)) } |
        Sort-Object { Get-JobTimeUtc $_ 'StartTime' })

    $jobInfo = [ordered]@{
        # Stable per server, so renaming the dashboard job via JobName doesn't create a new one
        external_id = "mars:$($env:COMPUTERNAME.ToLowerInvariant())"
        name        = $JobName
        # Always sent (null when unscheduled) so the dashboard's value tracks the agent's
        next_run_at = Get-NextRun
    }
} catch {
    Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR'
    exit 2
}

$failures = 0
try {
    if ($pending.Count -eq 0) {
        # Keep next_run_at fresh even when nothing new ran
        Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo
    }
    foreach ($job in $pending) {
        $run = ConvertTo-Run $job
        Send-JobReport -Tenant $Config.Tenant -SourceSystem $SourceSystem -Job $jobInfo -Run $run -SentKey $run.external_id -StartedAt ((Get-JobTimeUtc $job 'StartTime'))
    }
} catch {
    $failures++
    Write-Log "Failed to report: $(Get-ErrorText $_)" 'ERROR'
}

$exitCode = Complete-Collector -Failures $failures
if ($exitCode) { exit $exitCode }