Compare commits
2
Commits
dd3e2fe885
...
118a7f978d
| Author | SHA1 | Date | |
|---|---|---|---|
  kyleandClaude Opus 5.5
|
118a7f978d | ||
|
kyleandClaude Opus 5.5
|
3f407ba76e |
@@ -12,7 +12,7 @@ browser ──nginx (basic auth, TLS)──▶ /api/* ─▶│ SQLite │
|
||||
- `public/`: static frontend (no build step). Add `?mock=1` to the URL to use built-in sample data.
|
||||
- `backend/`: FastAPI API backed by a single SQLite file.
|
||||
- `deploy/`: nginx site and systemd unit.
|
||||
- `collectors/`: scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md), [Azure Backup / MARS](collectors/mars/README.md), [Windows Server Backup](collectors/wsb/README.md)).
|
||||
- `collectors/`: scripts that report results from backup products ([Veeam B&R](collectors/veeam/README.md), [Azure Backup / MARS](collectors/mars/README.md), [Windows Server Backup](collectors/wsb/README.md), [Veeam Backup for M365](collectors/vb365/README.md)).
|
||||
|
||||
## Data model
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Read-only dump of what the Veeam Backup for Microsoft 365 cmdlets return, for troubleshooting
|
||||
the collector.
|
||||
|
||||
.DESCRIPTION
|
||||
The collector reads properties of Get-VBOJob / Get-VBOJobSession results whose names aren't
|
||||
fully documented. This prints those objects' real structure (property names, types, values and
|
||||
each date's Kind, a few levels deep) for a few jobs and their latest sessions. Nothing is
|
||||
changed. Output is also saved to a text file.
|
||||
|
||||
.EXAMPLE
|
||||
.\Get-Vb365CollectorDiagnostics.ps1
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[int]$Jobs = 3,
|
||||
[int]$SessionsPerJob = 2,
|
||||
[string]$OutFile = (Join-Path $env:TEMP 'vb365-collector-diagnostics.txt')
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$MaxDepth = 4
|
||||
|
||||
if (-not (Get-Command Get-VBOJob -ErrorAction SilentlyContinue)) {
|
||||
if (Get-Module -ListAvailable -Name Veeam.Archiver.PowerShell) {
|
||||
Import-Module Veeam.Archiver.PowerShell
|
||||
} else {
|
||||
Import-Module (Join-Path $env:ProgramFiles 'Veeam\Backup365\Veeam.Archiver.PowerShell\Veeam.Archiver.PowerShell.psd1')
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Scalar($Value) {
|
||||
return $null -eq $Value -or $Value -is [string] -or $Value -is [ValueType] -or $Value -is [guid]
|
||||
}
|
||||
|
||||
# Recursive "name (type) = value" listing. Collections show their first 3 items.
|
||||
function Show-Object($Value, [string]$Indent = ' ', [int]$Depth = 0) {
|
||||
if (Test-Scalar $Value) { return }
|
||||
if ($Depth -ge $MaxDepth) { "$Indent..."; return }
|
||||
if ($Value -is [Collections.IEnumerable]) {
|
||||
$items = @($Value)
|
||||
"$Indent[$($items.Count) item(s)]"
|
||||
$i = 0
|
||||
foreach ($item in $items | Select-Object -First 3) {
|
||||
"$Indent[$i] ($($item.GetType().FullName))$(if (Test-Scalar $item) { " = $item" })"
|
||||
Show-Object $item "$Indent " ($Depth + 1)
|
||||
$i++
|
||||
}
|
||||
return
|
||||
}
|
||||
foreach ($p in $Value.PSObject.Properties) {
|
||||
try { $v = $p.Value } catch { "$Indent$($p.Name) = <error: $($_.Exception.Message)>"; continue }
|
||||
$type = if ($null -eq $v) { 'null' } else { $v.GetType().FullName }
|
||||
if (Test-Scalar $v) {
|
||||
$kind = if ($v -is [datetime]) { " [Kind=$($v.Kind)]" } else { '' }
|
||||
"$Indent$($p.Name) ($type) = $v$kind"
|
||||
} else {
|
||||
$text = [string]$v
|
||||
if ($text.Length -gt 100) { $text = $text.Substring(0, 97) + '...' }
|
||||
"$Indent$($p.Name) ($type) = $text"
|
||||
Show-Object $v "$Indent " ($Depth + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Show-Section([string]$Title, [scriptblock]$Get) {
|
||||
"== $Title"
|
||||
try {
|
||||
$result = @(& $Get)
|
||||
if ($result.Count -eq 0 -or ($result.Count -eq 1 -and $null -eq $result[0])) { ' (nothing returned)' }
|
||||
$n = 0
|
||||
foreach ($item in $result) {
|
||||
if ($null -eq $item) { continue }
|
||||
" [$n] ($($item.GetType().FullName))"
|
||||
Show-Object $item ' '
|
||||
$n++
|
||||
}
|
||||
} catch {
|
||||
" error: $($_.Exception.Message)"
|
||||
}
|
||||
''
|
||||
}
|
||||
|
||||
$report = & {
|
||||
$module = Get-Module Veeam.Archiver.PowerShell
|
||||
"Generated $(Get-Date -Format o) on $env:COMPUTERNAME, PowerShell $($PSVersionTable.PSVersion), module $($module.Version)"
|
||||
"Local time zone: $([TimeZoneInfo]::Local.Id), UTC offset now $([TimeZoneInfo]::Local.GetUtcOffset((Get-Date))), culture $([Globalization.CultureInfo]::CurrentCulture.Name)"
|
||||
''
|
||||
"== Organizations"
|
||||
try {
|
||||
Get-VBOOrganization | ForEach-Object { " $($_.Name) [$($_.Type)]" }
|
||||
} catch { " error: $($_.Exception.Message)" }
|
||||
''
|
||||
$allJobs = @(Get-VBOJob)
|
||||
"== Jobs: $($allJobs.Count) in total"
|
||||
$allJobs | ForEach-Object { " $($_.Name) [$($_.Id)] enabled=$($_.IsEnabled) next=$($_.NextRun)" }
|
||||
''
|
||||
Show-Section "Get-VBOJob (first $Jobs, full detail)" { $allJobs | Select-Object -First $Jobs }
|
||||
foreach ($j in $allJobs | Select-Object -First $Jobs) {
|
||||
Show-Section "Get-VBOJobSession -Job '$($j.Name)' (latest $SessionsPerJob)" {
|
||||
Get-VBOJobSession -Job $j | Sort-Object CreationTime -Descending | Select-Object -First $SessionsPerJob
|
||||
}
|
||||
}
|
||||
if (Get-Command Get-VBOCopyJob -ErrorAction SilentlyContinue) {
|
||||
Show-Section 'Get-VBOCopyJob (first 2)' { Get-VBOCopyJob | Select-Object -First 2 }
|
||||
if (Get-Command Get-VBOCopyJobSession -ErrorAction SilentlyContinue) {
|
||||
Show-Section 'Get-VBOCopyJobSession (latest 2)' {
|
||||
Get-VBOCopyJobSession | Sort-Object CreationTime -Descending | Select-Object -First 2
|
||||
}
|
||||
}
|
||||
} else {
|
||||
'== Get-VBOCopyJob: not available'
|
||||
}
|
||||
} | Out-String -Width 300
|
||||
|
||||
$report
|
||||
$report | Set-Content -Path $OutFile -Encoding UTF8
|
||||
Write-Host "Saved to $OutFile"
|
||||
@@ -0,0 +1,105 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Installs the Veeam Backup for Microsoft 365 collector as a SYSTEM scheduled task.
|
||||
|
||||
.DESCRIPTION
|
||||
- Copies Send-Vb365BackupStatus.ps1 to Program Files (only admins can modify it, which
|
||||
matters because it runs as SYSTEM).
|
||||
- Writes the config to ProgramData\BackupDashboard and restricts that folder to SYSTEM and
|
||||
Administrators, since it contains the API key.
|
||||
- Registers a scheduled task that runs every IntervalMinutes, under Windows PowerShell 5.1
|
||||
(powershell.exe) unless -PwshPath points at PowerShell 7.
|
||||
Safe to re-run to change settings or update the script. Can sit alongside the other collectors.
|
||||
|
||||
Dashboard company per job: -OrganizationTenants maps Microsoft 365 organization names to
|
||||
companies; jobs of unmapped organizations use -Tenant, or the organization's name if no -Tenant.
|
||||
|
||||
.EXAMPLE
|
||||
.\Install-Vb365Collector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp"
|
||||
One client per VB365 server. Prompts for the API key.
|
||||
.EXAMPLE
|
||||
.\Install-Vb365Collector.ps1 -ApiUrl https://backups.example.com -OrganizationTenants @{
|
||||
'acme.onmicrosoft.com' = 'Acme Corp'; 'bluewater.onmicrosoft.com' = 'Bluewater Dental' }
|
||||
A shared VB365 server protecting several organizations.
|
||||
#>
|
||||
#Requires -RunAsAdministrator
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)][string]$ApiUrl,
|
||||
[string]$Tenant,
|
||||
[hashtable]$OrganizationTenants = @{},
|
||||
[string]$SourceSystem = 'Veeam Backup for M365',
|
||||
[ValidateRange(5, 1440)][int]$IntervalMinutes = 60,
|
||||
[ValidateRange(1, 720)][int]$LookbackHours = 48,
|
||||
[string]$InstallDir = (Join-Path $env:ProgramFiles 'BackupDashboard'),
|
||||
# Run under PowerShell 7 instead of Windows PowerShell, e.g. 'C:\Program Files\PowerShell\7\pwsh.exe'
|
||||
[string]$PwshPath
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$TaskName = 'Backup Dashboard - VB365 collector'
|
||||
$DataDir = Join-Path $env:ProgramData 'BackupDashboard'
|
||||
$ConfigPath = Join-Path $DataDir 'vb365-collector.json'
|
||||
$Shell = Join-Path $env:SystemRoot 'System32\WindowsPowerShell\v1.0\powershell.exe'
|
||||
if ($PwshPath) {
|
||||
if (-not (Test-Path $PwshPath)) { throw "PowerShell 7 not found at '$PwshPath'." }
|
||||
$Shell = $PwshPath
|
||||
}
|
||||
|
||||
# Check VB365 is here before changing anything
|
||||
$moduleFile = Join-Path $env:ProgramFiles 'Veeam\Backup365\Veeam.Archiver.PowerShell\Veeam.Archiver.PowerShell.psd1'
|
||||
if (-not (Get-Module -ListAvailable -Name Veeam.Archiver.PowerShell) -and -not (Test-Path $moduleFile)) {
|
||||
throw 'The Veeam Backup for Microsoft 365 PowerShell module (Veeam.Archiver.PowerShell) is not installed on this server.'
|
||||
}
|
||||
if (-not $Tenant -and $OrganizationTenants.Count -eq 0) {
|
||||
Write-Warning 'No -Tenant or -OrganizationTenants: jobs will be listed under their Microsoft 365 organization name.'
|
||||
}
|
||||
|
||||
if ($ApiUrl -notmatch '^https://') {
|
||||
Write-Warning 'ApiUrl is not HTTPS: the API key would be sent in clear text.'
|
||||
}
|
||||
|
||||
# Read the key as a SecureString so it isn't echoed to the console or kept in shell history
|
||||
$secure = Read-Host -AsSecureString 'API key (from: python manage.py create-key <name>)'
|
||||
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secure)
|
||||
try { $apiKey = [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr) }
|
||||
finally { [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) }
|
||||
if (-not $apiKey.StartsWith('bdk_')) { throw "That doesn't look like a dashboard API key (expected 'bdk_...')." }
|
||||
|
||||
# Config folder: SYSTEM + Administrators only (SIDs so this works on non-English Windows)
|
||||
New-Item -ItemType Directory -Path $DataDir -Force | Out-Null
|
||||
& icacls.exe $DataDir /inheritance:r /grant:r '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null
|
||||
if ($LASTEXITCODE) { throw "icacls failed on $DataDir" }
|
||||
|
||||
$config = [ordered]@{
|
||||
ApiUrl = $ApiUrl.TrimEnd('/')
|
||||
ApiKey = $apiKey
|
||||
SourceSystem = $SourceSystem
|
||||
LookbackHours = $LookbackHours
|
||||
OrganizationTenants = $OrganizationTenants
|
||||
}
|
||||
if ($Tenant) { $config.Tenant = $Tenant }
|
||||
$config | ConvertTo-Json | Set-Content -Path $ConfigPath -Encoding UTF8
|
||||
|
||||
New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null
|
||||
$scriptPath = Join-Path $InstallDir 'Send-Vb365BackupStatus.ps1'
|
||||
Copy-Item -Path (Join-Path $PSScriptRoot 'Send-Vb365BackupStatus.ps1') -Destination $scriptPath -Force
|
||||
Unblock-File -Path $scriptPath
|
||||
|
||||
$action = New-ScheduledTaskAction -Execute $Shell `
|
||||
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$scriptPath`""
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(1) `
|
||||
-RepetitionInterval (New-TimeSpan -Minutes $IntervalMinutes)
|
||||
$principal = New-ScheduledTaskPrincipal -UserId 'S-1-5-18' -LogonType ServiceAccount -RunLevel Highest
|
||||
$settings = New-ScheduledTaskSettingsSet -ExecutionTimeLimit (New-TimeSpan -Minutes 30) `
|
||||
-MultipleInstances IgnoreNew -StartWhenAvailable
|
||||
Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal `
|
||||
-Settings $settings -Description 'Sends Veeam Backup for Microsoft 365 results to the Backup Dashboard' -Force | Out-Null
|
||||
|
||||
Write-Host "Installed. Config: $ConfigPath"
|
||||
Write-Host "Task '$TaskName' runs every $IntervalMinutes minutes as SYSTEM, using $Shell."
|
||||
Write-Host ''
|
||||
Write-Host 'Next steps:'
|
||||
Write-Host " 1. Preview what will be sent: & '$Shell' -File '$scriptPath' -DryRun"
|
||||
Write-Host " 2. Backfill 30 days of history: & '$Shell' -File '$scriptPath' -LookbackHours 720"
|
||||
Write-Host " 3. Check the log: Get-Content '$DataDir\vb365-collector.log' -Tail 20"
|
||||
@@ -0,0 +1,86 @@
|
||||
# Veeam Backup for Microsoft 365 collector
|
||||
|
||||
A PowerShell script that runs on the Veeam Backup for Microsoft 365 (VB365) server and reports its
|
||||
backup jobs (and backup copy jobs) to the dashboard, one dashboard job per VB365 job.
|
||||
|
||||
> **Checked against:** VB365 8.4 under Windows PowerShell 5.1, a shared server protecting 11
|
||||
> organizations: successful, warning and retried sessions. Not yet seen on a real server: a failed
|
||||
> session, a session in progress, or a backup copy job. If one looks wrong, run
|
||||
> `Get-Vb365CollectorDiagnostics.ps1` while it's in that state and send the output.
|
||||
|
||||
## Which company a job appears under
|
||||
|
||||
One VB365 server can protect several Microsoft 365 organizations, so the company is picked per job
|
||||
from the job's organization:
|
||||
|
||||
1. `OrganizationTenants` maps an organization name (as `Get-VBOOrganization` shows it, e.g.
|
||||
`acme.onmicrosoft.com`; not case-sensitive) to a dashboard company.
|
||||
2. Otherwise `Tenant`, for a VB365 server dedicated to one client.
|
||||
3. Otherwise the organization's own name.
|
||||
|
||||
Changing a job's company later creates a new dashboard job under the new company; delete the old
|
||||
one with `python manage.py delete-job <id>`.
|
||||
|
||||
## Install (on the VB365 server, as an administrator)
|
||||
|
||||
1. On the dashboard host, create a key for this server:
|
||||
```bash
|
||||
python manage.py create-key vb365-01
|
||||
```
|
||||
2. Copy this folder to the server and run one of:
|
||||
```powershell
|
||||
# VB365 server for one client
|
||||
.\Install-Vb365Collector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp"
|
||||
|
||||
# Shared VB365 server
|
||||
.\Install-Vb365Collector.ps1 -ApiUrl https://backups.example.com -OrganizationTenants @{
|
||||
'acme.onmicrosoft.com' = 'Acme Corp'
|
||||
'bluewater.onmicrosoft.com' = 'Bluewater Dental'
|
||||
}
|
||||
```
|
||||
It prompts for the key, then:
|
||||
- writes `C:\ProgramData\BackupDashboard\vb365-collector.json` (readable only by SYSTEM and Administrators)
|
||||
- copies the script to `C:\Program Files\BackupDashboard\`
|
||||
- registers the scheduled task **Backup Dashboard - VB365 collector** (SYSTEM, every 60 minutes)
|
||||
|
||||
It runs under Windows PowerShell 5.1. If your VB365 version's module needs PowerShell 7, add
|
||||
`-PwshPath 'C:\Program Files\PowerShell\7\pwsh.exe'`.
|
||||
3. Preview the payloads, then backfill 30 days so the history view is populated straight away:
|
||||
```powershell
|
||||
powershell.exe -File 'C:\Program Files\BackupDashboard\Send-Vb365BackupStatus.ps1' -DryRun
|
||||
powershell.exe -File 'C:\Program Files\BackupDashboard\Send-Vb365BackupStatus.ps1' -LookbackHours 720
|
||||
```
|
||||
|
||||
It can be installed alongside the other collectors: it uses its own config, state, log and task.
|
||||
Re-run the installer to change settings (including the organization mapping) or update the script.
|
||||
|
||||
## What it sends
|
||||
|
||||
| Dashboard | VB365 |
|
||||
|---|---|
|
||||
| Company | See above |
|
||||
| Job | Each backup job and backup copy job. The job's GUID is the ID, so renaming it in VB365 doesn't duplicate it |
|
||||
| Status | Session status: Success / Warning / Failed. Running or queued → **Running**. Stopped (cancelled) or unrecognised → **Warning** |
|
||||
| Started / finished | Session `CreationTime` / `EndTime` |
|
||||
| Next scheduled | Job's `NextRun`. Empty for disabled jobs |
|
||||
| Message | Warning and error lines from the session log (items titled `[Warning] ...`, `[Failed] ...`), plus "Veeam will retry this job." when a retry is pending |
|
||||
| Retries | VB365 runs a retry as a separate session, so it appears as its own run, marked `details.retry_of` (the session it retried). The retried run is marked `details.will_retry` |
|
||||
| Size | Data transferred (`Statistics.TransferredData`) |
|
||||
| Details | Organization, objects processed, job kind (`backup` / `copy`) |
|
||||
|
||||
Each run covers the last 48 hours (`LookbackHours`). Sessions already delivered are recorded in
|
||||
`vb365-collector.state.json`.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- Log: `C:\ProgramData\BackupDashboard\vb365-collector.log`. Exit codes: `0` OK, `1` some jobs
|
||||
failed to send (retried next run), `2` setup failed (config or VB365 module).
|
||||
- Wrong status, times or messages: run `.\Get-Vb365CollectorDiagnostics.ps1` (read-only). It prints
|
||||
the real structure of jobs and sessions, including whether each date is local or UTC.
|
||||
|
||||
## Testing without VB365
|
||||
|
||||
```powershell
|
||||
. ./tests/MockVb365.ps1
|
||||
./Send-Vb365BackupStatus.ps1 -ConfigPath ./my-test-config.json -DryRun
|
||||
```
|
||||
@@ -0,0 +1,372 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Reports Veeam Backup for Microsoft 365 (VB365) job results to the Backup Dashboard.
|
||||
|
||||
.DESCRIPTION
|
||||
Runs on the VB365 server, normally as a SYSTEM scheduled task every 60 minutes
|
||||
(see Install-Vb365Collector.ps1). For every backup job (and backup copy job) it sends:
|
||||
- the job's next scheduled run, which drives the dashboard's Overdue flag, and
|
||||
- every session from the last LookbackHours that the dashboard hasn't already received in
|
||||
its final state. Running sessions are re-sent each time until they finish.
|
||||
|
||||
One VB365 server can protect several Microsoft 365 organizations, so the dashboard company is
|
||||
chosen per job from its organization: OrganizationTenants in the config maps an organization
|
||||
name (e.g. "contoso.onmicrosoft.com") to a company; otherwise Tenant is used; otherwise the
|
||||
organization's own name.
|
||||
|
||||
Already-reported sessions are tracked in a state file, and the ingest API is idempotent, so a
|
||||
failed or repeated run is harmless: anything not confirmed is retried next time.
|
||||
|
||||
.PARAMETER ConfigPath
|
||||
JSON config with ApiUrl and ApiKey (optional: Tenant, OrganizationTenants, SourceSystem,
|
||||
LookbackHours).
|
||||
|
||||
.PARAMETER LookbackHours
|
||||
Overrides the config value. Use 720 once to backfill 30 days of history.
|
||||
|
||||
.PARAMETER DryRun
|
||||
Print the payloads instead of sending them. Doesn't touch the state or log files.
|
||||
|
||||
.EXAMPLE
|
||||
.\Send-Vb365BackupStatus.ps1 -DryRun
|
||||
.EXAMPLE
|
||||
.\Send-Vb365BackupStatus.ps1 -LookbackHours 720
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ConfigPath = (Join-Path $env:ProgramData 'BackupDashboard\vb365-collector.json'),
|
||||
[int]$LookbackHours = 0,
|
||||
[switch]$DryRun
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$DataDir = Split-Path -Parent $ConfigPath
|
||||
$LogPath = Join-Path $DataDir 'vb365-collector.log'
|
||||
$StatePath = Join-Path $DataDir 'vb365-collector.state.json'
|
||||
$MaxLogBytes = 5MB
|
||||
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||
|
||||
# ---------- Helpers ----------
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message, [string]$Level = 'INFO')
|
||||
$line = '{0} [{1}] {2}' -f (Get-Date).ToString('yyyy-MM-dd HH:mm:ss'), $Level, $Message
|
||||
Write-Host $line
|
||||
if ($DryRun) { return }
|
||||
if ((Test-Path $LogPath) -and (Get-Item $LogPath).Length -gt $MaxLogBytes) {
|
||||
Move-Item $LogPath "$LogPath.1" -Force
|
||||
}
|
||||
Add-Content -Path $LogPath -Value $line -Encoding UTF8
|
||||
}
|
||||
|
||||
function Get-ErrorText($ErrorRecord) {
|
||||
# Windows PowerShell puts the HTTP response body (e.g. the API's 422 explanation) here
|
||||
if ($ErrorRecord.ErrorDetails -and $ErrorRecord.ErrorDetails.Message) {
|
||||
return "$($ErrorRecord.Exception.Message) $($ErrorRecord.ErrorDetails.Message)"
|
||||
}
|
||||
return $ErrorRecord.Exception.Message
|
||||
}
|
||||
|
||||
# The API needs an explicit UTC offset. DateTimeOffset keeps a Local/Unspecified time as local and a
|
||||
# Utc time as +00:00.
|
||||
function ConvertTo-IsoTime($Value) {
|
||||
if ($null -eq $Value) { return $null }
|
||||
$dt = [datetime]$Value
|
||||
if ($dt.Year -le 1900) { return $null } # "not set" (DateTime.MinValue and friends)
|
||||
return ([DateTimeOffset]$dt).ToString("yyyy-MM-dd'T'HH:mm:sszzz", $Invariant)
|
||||
}
|
||||
|
||||
# First property of $Object, out of $Names, that exists and isn't empty. The VB365 object model isn't
|
||||
# fully documented, so reads go through this rather than assuming one spelling.
|
||||
function Get-FirstValue($Object, [string[]]$Names) {
|
||||
if ($null -eq $Object) { return $null }
|
||||
foreach ($name in $Names) {
|
||||
try {
|
||||
$prop = $Object.PSObject.Properties[$name]
|
||||
if (-not $prop) { continue }
|
||||
$value = $prop.Value
|
||||
# Check emptiness by type: casting to [string] would make a one-item list of objects ''
|
||||
if ($null -eq $value) { continue }
|
||||
if ($value -is [string] -and $value -eq '') { continue }
|
||||
if ($value -is [Collections.ICollection] -and $value.Count -eq 0) { continue }
|
||||
return $value
|
||||
} catch { }
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Import-Vb365 {
|
||||
if (Get-Command Get-VBOJob -ErrorAction SilentlyContinue) { return } # already loaded
|
||||
if (Get-Module -ListAvailable -Name Veeam.Archiver.PowerShell) {
|
||||
Import-Module Veeam.Archiver.PowerShell
|
||||
return
|
||||
}
|
||||
# Fall back to the default install path
|
||||
Import-Module (Join-Path $env:ProgramFiles 'Veeam\Backup365\Veeam.Archiver.PowerShell\Veeam.Archiver.PowerShell.psd1')
|
||||
}
|
||||
|
||||
function Get-Time($Object, [string]$Field) {
|
||||
$value = Get-FirstValue $Object @($Field)
|
||||
if ($null -eq $value) { return $null }
|
||||
$dt = [datetime]$value
|
||||
if ($dt.Year -le 1900) { return $null } # e.g. EndTime while a session is running
|
||||
return $dt
|
||||
}
|
||||
|
||||
# "1.5 GB", "512 MB", "123456" -> bytes; $null when it can't be read
|
||||
function ConvertTo-Bytes($Value) {
|
||||
if ($null -eq $Value) { return $null }
|
||||
if ($Value -is [ValueType]) { return [long]$Value }
|
||||
$match = [regex]::Match([string]$Value, '^\s*([\d.,]+)\s*([KMGTP]?B)?\s*$', 'IgnoreCase')
|
||||
if (-not $match.Success) { return $null }
|
||||
$number = 0.0
|
||||
if (-not [double]::TryParse(($match.Groups[1].Value -replace ',', '.'), [Globalization.NumberStyles]::Float, $Invariant, [ref]$number)) {
|
||||
return $null
|
||||
}
|
||||
$power = @{ '' = 0; 'B' = 0; 'KB' = 1; 'MB' = 2; 'GB' = 3; 'TB' = 4; 'PB' = 5 }[$match.Groups[2].Value.ToUpperInvariant()]
|
||||
return [long]($number * [math]::Pow(1024, $power))
|
||||
}
|
||||
|
||||
# ---------- Jobs and tenants ----------
|
||||
|
||||
function Get-OrganizationName($Job) {
|
||||
$org = Get-FirstValue $Job @('Organization')
|
||||
if ($null -eq $org) {
|
||||
# Copy jobs point at their source backup job
|
||||
$org = Get-FirstValue (Get-FirstValue $Job @('BackupJob')) @('Organization')
|
||||
}
|
||||
if ($org -is [string]) { return $org }
|
||||
$name = Get-FirstValue $org @('Name', 'OfficeName', 'DisplayName')
|
||||
if ($name) { return [string]$name }
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-TenantForJob($Job) {
|
||||
$org = Get-OrganizationName $Job
|
||||
if ($org -and $Config.OrganizationTenants) {
|
||||
$mapped = $Config.OrganizationTenants.PSObject.Properties[$org] # case-insensitive lookup
|
||||
if ($mapped -and $mapped.Value) { return [string]$mapped.Value }
|
||||
}
|
||||
if ($Config.Tenant) { return [string]$Config.Tenant }
|
||||
if ($org) { return $org }
|
||||
return $env:COMPUTERNAME
|
||||
}
|
||||
|
||||
function Get-NextRun($Job) {
|
||||
try {
|
||||
$enabled = Get-FirstValue $Job @('IsEnabled', 'Enabled')
|
||||
if ($enabled -eq $false) { return $null }
|
||||
return ConvertTo-IsoTime (Get-Time $Job 'NextRun')
|
||||
} catch {
|
||||
Write-Log "Couldn't read schedule for job '$($Job.Name)': $(Get-ErrorText $_)" 'WARN'
|
||||
return $null
|
||||
}
|
||||
}
|
||||
|
||||
# ---------- Sessions ----------
|
||||
|
||||
function Get-RunStatus($Session) {
|
||||
$status = [string](Get-FirstValue $Session @('Status', 'Result', 'State'))
|
||||
if ($status -match 'Fail|Error') { return 'failed' }
|
||||
if ($status -match 'Warn') { return 'warning' }
|
||||
if ($status -match 'Succe') { return 'success' }
|
||||
if ($status -match 'Run|Queue|Progress|Start|Wait') { return 'running' }
|
||||
if (-not (Get-Time $Session 'EndTime')) { return 'running' }
|
||||
# Stopped (cancelled), Disconnected, or a state we don't recognise: surface it rather than hide it
|
||||
return 'warning'
|
||||
}
|
||||
|
||||
function Test-EmptyGuid($Value) {
|
||||
return (-not $Value) -or [string]$Value -eq [string][guid]::Empty
|
||||
}
|
||||
|
||||
# Warning/error lines from the session log. Each log item's status is a prefix of its Title, e.g.
|
||||
# "[Warning] Item was skipped" (there's no separate status property), so match on that rather than
|
||||
# on words in the text, which would catch e.g. "[Success] Processing site: .../error-reports".
|
||||
function Get-RunMessage($Session, [string]$Status) {
|
||||
if ($Status -eq 'running') { return '' }
|
||||
$lines = New-Object 'System.Collections.Generic.List[string]'
|
||||
$sessionStatus = [string](Get-FirstValue $Session @('Status', 'Result', 'State'))
|
||||
if ($Status -ne 'success') {
|
||||
foreach ($entry in @(Get-FirstValue $Session @('Log', 'Logs') | Where-Object { $_ })) {
|
||||
$title = ([string](Get-FirstValue $entry @('Title', 'Message', 'Text'))).Trim()
|
||||
$prefix = [regex]::Match($title, '^\[(\w+)\]\s*')
|
||||
if ($prefix.Success) {
|
||||
if ($prefix.Groups[1].Value -notmatch 'Warn|Fail|Error') { continue }
|
||||
$title = $title.Substring($prefix.Length)
|
||||
} elseif ([string](Get-FirstValue $entry @('Status', 'Type', 'Severity')) -notmatch 'Warn|Fail|Error') {
|
||||
continue
|
||||
}
|
||||
if ($title -and -not $lines.Contains($title)) { $lines.Add($title) }
|
||||
}
|
||||
}
|
||||
if ($lines.Count -eq 0) {
|
||||
if ($Status -eq 'success') { $lines.Add('Job completed successfully.') }
|
||||
else { $lines.Add("Job finished with status '$sessionStatus'.") }
|
||||
} elseif ($sessionStatus -and $sessionStatus -notmatch 'Fail|Warn') {
|
||||
$lines.Insert(0, "Session status: $sessionStatus")
|
||||
}
|
||||
# A warning/failure VB365 is about to retry; the retry arrives as its own run
|
||||
if ($Status -ne 'success' -and (Get-FirstValue $Session @('JobWillBeRetried')) -eq $true) {
|
||||
$lines.Add('Veeam will retry this job.')
|
||||
}
|
||||
return ($lines -join "`n")
|
||||
}
|
||||
|
||||
function ConvertTo-Run($Session, $Job, [string]$JobKind) {
|
||||
$status = Get-RunStatus $Session
|
||||
$details = [ordered]@{ vb365_server = $env:COMPUTERNAME; job_kind = $JobKind }
|
||||
$org = Get-OrganizationName $Job
|
||||
if ($org) { $details.organization = $org }
|
||||
$sessionStatus = Get-FirstValue $Session @('Status', 'Result', 'State')
|
||||
if ($sessionStatus) { $details.vb365_status = [string]$sessionStatus }
|
||||
# Retries are separate sessions pointing at the session they retry
|
||||
$mainSession = Get-FirstValue $Session @('MainSessionId')
|
||||
if (-not (Test-EmptyGuid $mainSession)) { $details.retry_of = [string]$mainSession }
|
||||
if ((Get-FirstValue $Session @('JobWillBeRetried')) -eq $true) { $details.will_retry = $true }
|
||||
|
||||
$run = [ordered]@{
|
||||
external_id = [string](Get-FirstValue $Session @('Id', 'SessionId'))
|
||||
status = $status
|
||||
started_at = ConvertTo-IsoTime (Get-Time $Session 'CreationTime')
|
||||
message = Get-RunMessage $Session $status
|
||||
details = $details
|
||||
}
|
||||
if ($status -ne 'running') {
|
||||
$finished = ConvertTo-IsoTime (Get-Time $Session 'EndTime')
|
||||
if ($finished) { $run.finished_at = $finished } # otherwise the API uses receipt time
|
||||
$stats = Get-FirstValue $Session @('Statistics')
|
||||
$processed = Get-FirstValue $stats @('ProcessedObjects')
|
||||
if ($null -ne $processed) { $details.processed_objects = $processed }
|
||||
$transferred = ConvertTo-Bytes (Get-FirstValue $stats @('TransferredData', 'TransferredDataSize'))
|
||||
if ($transferred -gt 0) { $run.size_bytes = $transferred }
|
||||
}
|
||||
return $run
|
||||
}
|
||||
|
||||
function Send-Report($Body) {
|
||||
$json = $Body | ConvertTo-Json -Depth 6 -Compress
|
||||
if ($DryRun) {
|
||||
Write-Host $json
|
||||
return
|
||||
}
|
||||
# Send bytes: Windows PowerShell would otherwise encode a string body as Latin-1
|
||||
$params = @{
|
||||
Uri = "$($Config.ApiUrl.TrimEnd('/'))/api/v1/ingest"
|
||||
Method = 'Post'
|
||||
Headers = @{ Authorization = "Bearer $($Config.ApiKey)" }
|
||||
ContentType = 'application/json; charset=utf-8'
|
||||
Body = [Text.Encoding]::UTF8.GetBytes($json)
|
||||
TimeoutSec = 30
|
||||
}
|
||||
Invoke-RestMethod @params | Out-Null
|
||||
}
|
||||
|
||||
# ---------- Main ----------
|
||||
|
||||
try {
|
||||
$Config = Get-Content -Path $ConfigPath -Raw | ConvertFrom-Json
|
||||
foreach ($key in 'ApiUrl', 'ApiKey') {
|
||||
if (-not $Config.$key) { throw "Config $ConfigPath is missing '$key'" }
|
||||
}
|
||||
$SourceSystem = if ($Config.SourceSystem) { [string]$Config.SourceSystem } else { 'Veeam Backup for M365' }
|
||||
if ($LookbackHours -le 0) {
|
||||
$LookbackHours = if ($Config.LookbackHours) { [int]$Config.LookbackHours } else { 48 }
|
||||
}
|
||||
|
||||
# Windows PowerShell 5.1 may default to TLS 1.0
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
|
||||
|
||||
Import-Vb365
|
||||
|
||||
# Session ID -> start time (Unix seconds), for sessions already delivered in their final state
|
||||
$sent = @{}
|
||||
if (Test-Path $StatePath) {
|
||||
$saved = Get-Content -Path $StatePath -Raw | ConvertFrom-Json
|
||||
if ($saved) { foreach ($p in $saved.PSObject.Properties) { $sent[$p.Name] = [long]$p.Value } }
|
||||
}
|
||||
|
||||
$since = (Get-Date).AddHours(-$LookbackHours)
|
||||
|
||||
# (job, kind, sessions) for backup jobs, plus backup copy jobs where the cmdlets exist
|
||||
$work = New-Object 'System.Collections.Generic.List[object]'
|
||||
foreach ($job in @(Get-VBOJob)) {
|
||||
$sessions = @(Get-VBOJobSession -Job $job | Where-Object { (Get-Time $_ 'CreationTime') -ge $since })
|
||||
$work.Add([pscustomobject]@{ Job = $job; Kind = 'backup'; Sessions = $sessions })
|
||||
}
|
||||
if ((Get-Command Get-VBOCopyJob -ErrorAction SilentlyContinue) -and (Get-Command Get-VBOCopyJobSession -ErrorAction SilentlyContinue)) {
|
||||
$copyJobs = @(Get-VBOCopyJob)
|
||||
if ($copyJobs.Count) {
|
||||
# Fetch once and match by job ID, since the per-job parameter name isn't consistent
|
||||
$copySessions = @(Get-VBOCopyJobSession | Where-Object { (Get-Time $_ 'CreationTime') -ge $since })
|
||||
foreach ($job in $copyJobs) {
|
||||
$id = [string]$job.Id
|
||||
$work.Add([pscustomobject]@{
|
||||
Job = $job; Kind = 'copy'
|
||||
Sessions = @($copySessions | Where-Object { [string](Get-FirstValue $_ @('JobId')) -eq $id })
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
Write-Log "Collector setup failed: $(Get-ErrorText $_)" 'ERROR'
|
||||
exit 2
|
||||
}
|
||||
|
||||
$failures = 0
|
||||
$runsSent = 0
|
||||
foreach ($item in $work) {
|
||||
$job = $item.Job
|
||||
$tenant = Get-TenantForJob $job
|
||||
$jobInfo = [ordered]@{
|
||||
external_id = [string]$job.Id
|
||||
name = [string]$job.Name
|
||||
# Always sent (null when disabled/unscheduled) so the dashboard's value tracks VB365's
|
||||
next_run_at = Get-NextRun $job
|
||||
}
|
||||
$pending = @($item.Sessions |
|
||||
Where-Object { $_ -and -not $sent.ContainsKey([string](Get-FirstValue $_ @('Id', 'SessionId'))) } |
|
||||
Sort-Object { Get-Time $_ 'CreationTime' })
|
||||
|
||||
try {
|
||||
if ($pending.Count -eq 0) {
|
||||
# Keep next_run_at fresh even when nothing new ran
|
||||
Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo })
|
||||
}
|
||||
foreach ($session in $pending) {
|
||||
$run = ConvertTo-Run $session $job $item.Kind
|
||||
Send-Report ([ordered]@{ tenant = $tenant; source_system = $SourceSystem; job = $jobInfo; run = $run })
|
||||
$runsSent++
|
||||
if ($run.status -ne 'running') {
|
||||
$sent[$run.external_id] = ([DateTimeOffset](Get-Time $session 'CreationTime')).ToUnixTimeSeconds()
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
$failures++
|
||||
Write-Log "Failed to report job '$($job.Name)': $(Get-ErrorText $_)" 'ERROR'
|
||||
$httpStatus = 0
|
||||
try { $httpStatus = [int]$_.Exception.Response.StatusCode } catch { }
|
||||
if ($httpStatus -eq 401) {
|
||||
Write-Log 'API key rejected; check ApiKey in the config. Stopping.' 'ERROR'
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $DryRun) {
|
||||
# Forget sessions that have aged out of the lookback window (plus a day of slack)
|
||||
$cutoff = [DateTimeOffset]::Now.AddHours(-$LookbackHours - 24).ToUnixTimeSeconds()
|
||||
$keep = [ordered]@{}
|
||||
foreach ($id in $sent.Keys) {
|
||||
if ($sent[$id] -ge $cutoff) { $keep[$id] = $sent[$id] }
|
||||
}
|
||||
try {
|
||||
$keep | ConvertTo-Json -Compress | Set-Content -Path $StatePath -Encoding UTF8
|
||||
} catch {
|
||||
Write-Log "Couldn't save state file: $(Get-ErrorText $_)" 'WARN'
|
||||
}
|
||||
}
|
||||
|
||||
Write-Log ("Reported {0} jobs, {1} runs, {2} failures" -f $work.Count, $runsSent, $failures)
|
||||
if ($failures) { exit 1 }
|
||||
@@ -0,0 +1,78 @@
|
||||
# Stand-ins for the Veeam Backup for Microsoft 365 cmdlets. Object shapes follow
|
||||
# Get-Vb365CollectorDiagnostics.ps1 output from a real VB365 8.4 server: session and NextRun times
|
||||
# are local, Statistics.TransferredData is a string ("187.2 MB"), log items carry their status as a
|
||||
# Title prefix ("[Warning] ..."), and retries are separate sessions whose MainSessionId points at
|
||||
# the session they retry (which has JobWillBeRetried = True).
|
||||
#
|
||||
# . ./tests/MockVb365.ps1
|
||||
# ./Send-Vb365BackupStatus.ps1 -ConfigPath ./my-test-config.json -DryRun
|
||||
|
||||
# Fixed to the top of the hour, and IDs derived from names/times, so repeated runs see the same data
|
||||
$now = Get-Date -Minute 0 -Second 0 -Millisecond 0
|
||||
function New-StableGuid([string]$Seed) {
|
||||
[guid]::new([Security.Cryptography.MD5]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($Seed)))
|
||||
}
|
||||
|
||||
function New-MockOrg($Name) { [pscustomobject]@{ Name = $Name; OfficeName = $Name; Type = 'Office365'; IsBackedUp = $true } }
|
||||
$acme = New-MockOrg 'acme.onmicrosoft.com'
|
||||
$blue = New-MockOrg 'bluewater.onmicrosoft.com'
|
||||
|
||||
function New-MockJob($Name, $Org, $NextRun, [bool]$Enabled = $true) {
|
||||
[pscustomobject]@{
|
||||
Id = New-StableGuid $Name; Name = $Name; Organization = $Org; JobType = 'Backup'
|
||||
IsEnabled = $Enabled; NextRun = $NextRun; LastStatus = 'Success'
|
||||
LastRun = $(if ($NextRun) { $NextRun.AddDays(-1).ToUniversalTime() }) # UTC, as on the real server
|
||||
}
|
||||
}
|
||||
|
||||
function New-LogItem($Status, $Text, $At) {
|
||||
[pscustomobject]@{ Id = [guid]::NewGuid(); USN = 1; Title = "[$Status] $Text"; CreationTime = $At; EndTime = $At }
|
||||
}
|
||||
|
||||
function New-MockSession($Job, $Start, $End, $Status, $Transferred = '0 B', [long]$Objects = 0, $Log = @(),
|
||||
$MainSessionId = [guid]::Empty, [bool]$WillRetry = $false) {
|
||||
$logItems = @((New-LogItem 'Success' "Job started at: $($Start.ToString('G'))" $Start)) + @($Log)
|
||||
[pscustomobject]@{
|
||||
JobId = $Job.Id; JobName = $Job.Name; Status = $Status
|
||||
Statistics = [pscustomobject]@{ ProcessingRate = '1.6 MB/s (4 items/s)'; TransferredData = $Transferred; ProcessedObjects = $Objects; Bottleneck = 'Target' }
|
||||
JobSessionConfigType = 'Incremental'
|
||||
Id = New-StableGuid "$($Job.Name)|$($Start.Ticks)"
|
||||
CreationTime = $Start
|
||||
EndTime = $(if ($End) { $End } else { [datetime]::MinValue })
|
||||
Progress = 49; MainSessionId = $MainSessionId; JobWillBeRetried = $WillRetry
|
||||
Log = $logItems
|
||||
}
|
||||
}
|
||||
|
||||
$global:MockVboJobs = @(
|
||||
(New-MockJob 'Acme 365 Backup' $acme $now.Date.AddHours(17).AddDays($(if ($now.Hour -ge 17) { 1 } else { 0 }))),
|
||||
(New-MockJob 'Bluewater 365 Backup' $blue $now.Date.AddHours(17.75).AddDays($(if ($now.Hour -ge 17) { 1 } else { 0 }))),
|
||||
(New-MockJob 'Bluewater Old Mailboxes' $blue $now.AddDays(1) $false) # disabled
|
||||
)
|
||||
$acmeJob, $blueJob, $oldJob = $global:MockVboJobs
|
||||
|
||||
$y = $now.Date.AddDays(-1) # yesterday
|
||||
$blueWarn = New-MockSession $blueJob $y.AddHours(17.75) $y.AddHours(17.75).AddMinutes(3.5) 'Warning' '32.9 MB' 47 @(
|
||||
(New-LogItem 'Success' 'Found 94 objects' $y.AddHours(17.75)),
|
||||
# A success line mentioning "error" must not end up in the message
|
||||
(New-LogItem 'Success' 'Processing site: https://bluewater.sharepoint.com/sites/error-reports' $y.AddHours(17.76)),
|
||||
(New-LogItem 'Warning' 'Processing mailbox [email protected] finished with warning: Item "Q3.pst" exceeds the maximum size' $y.AddHours(17.78))
|
||||
) -WillRetry $true
|
||||
|
||||
$global:MockVboSessions = @(
|
||||
(New-MockSession $acmeJob $y.AddHours(17) $y.AddHours(17).AddMinutes(4.8) 'Success' '187.2 MB' 568 @(
|
||||
(New-LogItem 'Success' 'Found 49 objects' $y.AddHours(17)), (New-LogItem 'Success' 'Connected to organization' $y.AddHours(17)))),
|
||||
(New-MockSession $acmeJob $now.AddMinutes(-20) $null 'Running' '0 B' 12),
|
||||
$blueWarn,
|
||||
# Its retry 13 minutes later, which succeeded
|
||||
(New-MockSession $blueJob $y.AddHours(17.75).AddMinutes(13.5) $y.AddHours(17.75).AddMinutes(16.8) 'Success' '0 B' 0 -MainSessionId $blueWarn.Id),
|
||||
(New-MockSession $blueJob $now.AddHours(-30) $now.AddHours(-29.9) 'Failed' '0 B' 0 @(
|
||||
(New-LogItem 'Failed' 'Failed to connect to organization: The remote server returned an error: (401) Unauthorized.' $now.AddHours(-30)))),
|
||||
(New-MockSession $oldJob $now.AddDays(-5) $now.AddDays(-5).AddHours(1) 'Success') # outside 48h lookback
|
||||
)
|
||||
|
||||
function Get-VBOJob { $global:MockVboJobs }
|
||||
function Get-VBOJobSession { param($Job) $global:MockVboSessions | Where-Object { -not $Job -or $_.JobId -eq $Job.Id } }
|
||||
function Get-VBOOrganization { $acme, $blue }
|
||||
function Get-VBOCopyJob { } # none, as on the real server
|
||||
function Get-VBOCopyJobSession { }
|
||||
Reference in New Issue
Block a user