Merging

backend/app/settings.py

@@ -5,6 +5,12 @@ from pathlib import Path
 # The modal shows this many days of run history; retention can never be shorter.
 HISTORY_DAYS = 30
 
+<<<<<<< HEAD
+=======
+# Shared by the systemd service (EnvironmentFile=) and manage.py, so both always use the same DB.
+DEFAULT_ENV_FILE = "/etc/backup-dashboard/backup-dashboard.env"
+
+>>>>>>> add-backend-api
 
 @dataclass(frozen=True)
 class Settings:
@@ -14,9 +20,35 @@ class Settings:
     serve_static: bool = False
 
 
+<<<<<<< HEAD
 def load_settings() -> Settings:
     return Settings(
         db_path=Path(os.environ.get("BACKUP_DASH_DB", "backup-dashboard.db")),
         retention_days=max(HISTORY_DAYS, int(os.environ.get("BACKUP_DASH_RETENTION_DAYS", "90"))),
         serve_static=os.environ.get("BACKUP_DASH_SERVE_STATIC") == "1",
+=======
+def read_env_file(path: Path) -> dict[str, str]:
+    """Parse simple KEY=VALUE lines (the subset systemd's EnvironmentFile accepts)."""
+    values = {}
+    try:
+        lines = path.read_text().splitlines()
+    except FileNotFoundError:
+        return values
+    for line in lines:
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        values[key.strip()] = value.strip().strip("\"'")
+    return values
+
+
+def load_settings() -> Settings:
+    # Real environment variables win over the env file; the env file wins over dev defaults.
+    env = read_env_file(Path(os.environ.get("BACKUP_DASH_ENV_FILE", DEFAULT_ENV_FILE))) | os.environ
+    return Settings(
+        db_path=Path(env.get("BACKUP_DASH_DB", "backup-dashboard.db")),
+        retention_days=max(HISTORY_DAYS, int(env.get("BACKUP_DASH_RETENTION_DAYS", "90"))),
+        serve_static=env.get("BACKUP_DASH_SERVE_STATIC") == "1",
+>>>>>>> add-backend-api
     )

backend/tests/test_manage.py

@@ -0,0 +1,52 @@
+import subprocess
+import sys
+from pathlib import Path
+
+from app.settings import load_settings
+
+BACKEND = Path(__file__).resolve().parents[1]
+
+
+def run_manage(*args, env, cwd):
+    return subprocess.run([sys.executable, str(BACKEND / "manage.py"), *args],
+                          capture_output=True, text=True, env=env, cwd=cwd)
+
+
+def base_env(tmp_path, monkeypatch):
+    env_file = tmp_path / "backup-dashboard.env"
+    monkeypatch.setenv("BACKUP_DASH_ENV_FILE", str(env_file))
+    monkeypatch.delenv("BACKUP_DASH_DB", raising=False)
+    return env_file, {"BACKUP_DASH_ENV_FILE": str(env_file), "PYTHONPATH": str(BACKEND), "PATH": "/usr/bin:/bin"}
+
+
+def test_env_file_supplies_db_path_and_real_env_wins(tmp_path, monkeypatch):
+    env_file, _ = base_env(tmp_path, monkeypatch)
+    env_file.write_text('# comment\nBACKUP_DASH_DB="/srv/from-file.db"\nBACKUP_DASH_RETENTION_DAYS=120\n')
+    assert load_settings().db_path == Path("/srv/from-file.db")
+    assert load_settings().retention_days == 120
+    monkeypatch.setenv("BACKUP_DASH_DB", "/srv/from-env.db")
+    assert load_settings().db_path == Path("/srv/from-env.db")
+
+
+def test_manage_refuses_to_create_db_implicitly(tmp_path, monkeypatch):
+    _, env = base_env(tmp_path, monkeypatch)
+    workdir = tmp_path / "somewhere-else"
+    workdir.mkdir()
+    r = run_manage("create-key", "k", env=env, cwd=workdir)
+    assert r.returncode != 0
+    assert "Database not found" in r.stderr
+    assert not (workdir / "backup-dashboard.db").exists()
+
+
+def test_manage_uses_db_from_env_file(tmp_path, monkeypatch):
+    env_file, env = base_env(tmp_path, monkeypatch)
+    service_db = tmp_path / "service.db"
+    env_file.write_text(f"BACKUP_DASH_DB={service_db}\n")
+    assert run_manage("--create", "list-keys", env=env, cwd=tmp_path).returncode == 0  # service's first start
+    workdir = tmp_path / "somewhere-else"
+    workdir.mkdir()
+    r = run_manage("create-key", "veeam-acme", env=env, cwd=workdir)
+    assert r.returncode == 0, r.stderr
+    assert str(service_db) in r.stderr
+    assert "veeam-acme" in run_manage("list-keys", env=env, cwd=tmp_path).stdout
+    assert not (workdir / "backup-dashboard.db").exists()

collectors/veeam/README.md

@@ -5,19 +5,37 @@ It covers jobs returned by `Get-VBRJob`: VM backup, replication and backup copy
 
 ## Install (on the VBR server, as an administrator)
 
+<<<<<<< HEAD
+=======
+Requires PowerShell 7 installed machine-wide (`winget install Microsoft.PowerShell` or the MSI),
+which VBR 13's Veeam module expects. The Microsoft Store version won't work, because the SYSTEM
+account can't run it. The installer checks for this before changing anything.
+
+>>>>>>> add-backend-api
 1. On the dashboard host, create a key for this server:
    ```bash
    python manage.py create-key veeam-acme
    ```
+<<<<<<< HEAD
 2. Copy this folder to the VBR server and run:
+=======
+2. Copy this folder to the VBR server and run, from PowerShell 7 (`pwsh`):
+>>>>>>> add-backend-api
    ```powershell
    .\Install-VeeamCollector.ps1 -ApiUrl https://backups.example.com -Tenant "Acme Corp"
    ```
    It prompts for the key, then:
    - writes `C:\ProgramData\BackupDashboard\veeam-collector.json` (readable only by SYSTEM and Administrators)
    - copies the script to `C:\Program Files\BackupDashboard\`
+<<<<<<< HEAD
    - registers the scheduled task **Backup Dashboard - Veeam collector** (SYSTEM, every 15 minutes)
 3. Preview the payloads, then backfill 30 days so the history view is populated straight away:
+=======
+   - registers the scheduled task **Backup Dashboard - Veeam collector** (SYSTEM, every 15 minutes,
+     run by `C:\Program Files\PowerShell\7\pwsh.exe`; use `-PwshPath` if it's installed elsewhere)
+3. Preview the payloads, then backfill 30 days so the history view is populated straight away
+   (from `pwsh`):
+>>>>>>> add-backend-api
    ```powershell
    & 'C:\Program Files\BackupDashboard\Send-VeeamBackupStatus.ps1' -DryRun
    & 'C:\Program Files\BackupDashboard\Send-VeeamBackupStatus.ps1' -LookbackHours 720
@@ -38,6 +56,13 @@ delete the scheduled task and both `BackupDashboard` folders.
 | Message | Session reason, each VM or object that didn't succeed, and warning/error lines from the session log |
 | Size | `BackupStats.BackupSize` |
 
+<<<<<<< HEAD
+=======
+**Backup copy jobs (VBR 12+):** Veeam records their sessions under hidden per-source "worker" jobs.
+The collector files each worker session under its copy job (via the worker's `PolicyJobId`), so a
+copy job gets one run per source job it copies, with the source in `copied_from`.
+
+>>>>>>> add-backend-api
 Each run covers the last 48 hours (`LookbackHours`), so a few missed runs (server rebooting, API
 down) fill in automatically. Sessions already delivered are recorded in
 `veeam-collector.state.json`. Running sessions are re-sent until they finish.
@@ -48,6 +73,12 @@ down) fill in automatically. Sessions already delivered are recorded in
 - Exit codes: `0` OK, `1` some jobs failed to send (they're retried next run), `2` setup failed
   (config, Veeam module, or Veeam connection).
 - `API key rejected`: the key was revoked or mistyped. Re-run the installer with a new key.
+<<<<<<< HEAD
+=======
+- A job shows **Unknown** on the dashboard: the collector sent the job but matched no sessions to it.
+  Run `.\Get-VeeamCollectorDiagnostics.ps1` (read-only). It lists sessions whose job ID matches no
+  job, plus the type and link properties of the jobs left without sessions.
+>>>>>>> add-backend-api
 - `Get-VBRBackupSession` loads every session in the VBR database. On a server with years of history,
   this can make each run take a minute or more. The task's 30-minute limit and "don't start if
   already running" setting keep that safe.

collectors/veeam/tests/MockVeeam.ps1

@@ -0,0 +1,83 @@
+# Stand-ins for the Veeam cmdlets, shaped like the real objects' properties the collector reads.
+# Dot-source this before running the collector to try it without a VBR server:
+#
+#   . ./tests/MockVeeam.ps1
+#   ./Send-VeeamBackupStatus.ps1 -ConfigPath ./tests/config.json -DryRun
+
+# Fixed to the top of the hour, and IDs derived from names/times, so repeated runs see the same
+# jobs and sessions, like a real server
+$now = Get-Date -Minute 0 -Second 0 -Millisecond 0
+function New-StableGuid([string]$Seed) {
+    [guid]::new([Security.Cryptography.MD5]::Create().ComputeHash([Text.Encoding]::UTF8.GetBytes($Seed)))
+}
+$notSet = Get-Date '1900-01-01'
+
+function New-MockJob($Name, $Type, $NextRun, [bool]$Enabled = $true) {
+    [pscustomobject]@{
+        Id = New-StableGuid $Name; Name = $Name; JobType = $Type
+        IsScheduleEnabled = $Enabled
+        ScheduleOptions = [pscustomobject]@{ NextRun = $NextRun }
+    }
+}
+
+function New-MockSession($Job, $Start, $End, $State, $Result, $Reason = '', $Tasks = @(), $LogRecords = @(), $Size = 0) {
+    $s = [pscustomobject]@{
+        Id = New-StableGuid "$($Job.Name)|$($Start.Ticks)"; JobId = $Job.Id; JobName = $Job.Name
+        CreationTime = $Start; EndTime = $End; State = $State; Result = $Result
+        IsRetryMode = $false
+        Info = [pscustomobject]@{ Reason = $Reason }
+        BackupStats = [pscustomobject]@{ BackupSize = $Size; DataSize = $Size * 3 }
+        Logger = [pscustomobject]@{ Records = $LogRecords }
+        Tasks = $Tasks
+    }
+    $s.Logger | Add-Member ScriptMethod GetLog { [pscustomobject]@{ UpdatedRecords = $this.Records } }
+    $s | Add-Member ScriptMethod GetTaskSessions { $this.Tasks }
+    $s
+}
+
+function New-MockTask($Name, $Status, $Reason = '') {
+    [pscustomobject]@{ Name = $Name; Status = $Status; Info = [pscustomobject]@{ Reason = $Reason } }
+}
+
+$global:MockJobs = @(
+    (New-MockJob 'File Server – Nightly' 'Backup' $now.Date.AddDays(1).AddHours(22)),
+    # Older VBR versions return NextRun as a culture-formatted string
+    (New-MockJob 'SQL01 – Logs' 'Backup' $now.AddMinutes(40).ToString()),
+    (New-MockJob 'Offsite Copy' 'BackupSync' $null),
+    (New-MockJob 'Old Exchange' 'Backup' $now.AddDays(1) $false),
+    # VBR 12+ backup copy job (immediate mode, no schedule): the "policy" job Get-VBRJob returns
+    (New-MockJob 'Cloud Copy' 'SimpleBackupCopyPolicy' $null)
+)
+$fs, $sql, $copy, $old, $cloud = $global:MockJobs
+
+# Its hidden worker job, one per source job. Get-VBRJob doesn't return it, and Get-VBRBackupSession
+# returns only the worker's sessions. Shape taken from a real VBR 13 server.
+$cloudWorker = [pscustomobject]@{
+    Id = New-StableGuid 'Cloud Copy\File Server – Nightly'; Name = 'Cloud Copy\File Server – Nightly'
+    JobType = 'SimpleBackupCopyWorker'; PolicyJobId = $cloud.Id
+}
+function New-MockWorkerSession($Start, $End, $Result) {
+    $s = New-MockSession $cloudWorker $Start $End 'Stopped' $Result -Size 40GB
+    $s | Add-Member ScriptMethod GetJob { $cloudWorker }
+    $s
+}
+
+$global:MockSessions = @(
+    (New-MockSession $fs $now.AddDays(-1).AddHours(-2) $now.AddDays(-1).AddHours(-1) 'Stopped' 'Success' -Size 128GB -Tasks @(
+        (New-MockTask 'FS01' 'Success'), (New-MockTask 'FS02' 'Success'))),
+    (New-MockSession $fs $now.AddMinutes(-20) $notSet 'Working' 'None' -Tasks @((New-MockTask 'FS01' 'InProgress'))),
+    (New-MockSession $sql $now.AddHours(-1) $now.AddMinutes(-55) 'Stopped' 'Warning' -Size 2GB `
+        -Tasks @((New-MockTask 'SQL01' 'Warning' 'Failed to truncate transaction logs for database "ERP_PROD"'))),
+    (New-MockSession $copy $now.AddHours(-5) $now.AddHours(-4) 'Stopped' 'Failed' 'Error: Unable to connect to repository NAS02' `
+        -LogRecords @(
+            [pscustomobject]@{ Title = 'Preparing backup chain'; Status = 'ESucceeded' },
+            [pscustomobject]@{ Title = 'Network path not found (0x80070035)'; Status = 'EFailed' })),
+    (New-MockSession $old $now.AddDays(-10) $now.AddDays(-10).AddHours(1) 'Stopped' 'Success'),  # outside 48h lookback
+    (New-MockWorkerSession $now.AddDays(-1).AddMinutes(-55) $now.AddDays(-1).AddMinutes(-30) 'Success'),
+    (New-MockWorkerSession $now.AddMinutes(-55) $now.AddMinutes(-35) 'Warning'),
+    # A session from a job the collector doesn't know (e.g. deleted): must be ignored, not crash
+    (New-MockSession ([pscustomobject]@{ Id = New-StableGuid 'gone'; Name = 'Deleted job' }) $now.AddHours(-2) $now.AddHours(-1) 'Stopped' 'Success')
+)
+
+function Get-VBRJob { [CmdletBinding()] param() $global:MockJobs }
+function Get-VBRBackupSession { [CmdletBinding()] param() $global:MockSessions }

deploy/backup-dashboard-api.service

@@ -6,8 +6,13 @@ After=network.target
 User=backupdash
 Group=backupdash
 WorkingDirectory=/opt/backup-dashboard/backend
+<<<<<<< HEAD
 Environment=BACKUP_DASH_DB=/var/lib/backup-dashboard/backup-dashboard.db
 Environment=BACKUP_DASH_RETENTION_DAYS=90
+=======
+# Settings live in this file so manage.py reads exactly the same ones (see deploy/backup-dashboard.env)
+EnvironmentFile=/etc/backup-dashboard/backup-dashboard.env
+>>>>>>> add-backend-api
 # Bound to localhost: only nginx can reach it, so nginx's auth can't be bypassed from outside.
 # One worker is plenty at this scale and keeps SQLite writes simple.
 ExecStart=/opt/backup-dashboard/venv/bin/uvicorn --factory app.main:create_app --host 127.0.0.1 --port 8000 --proxy-headers

deploy/backup-dashboard.env

@@ -0,0 +1,4 @@
+# Install as /etc/backup-dashboard/backup-dashboard.env
+# Read by both the API service and manage.py, so they always use the same database.
+BACKUP_DASH_DB=/var/lib/backup-dashboard/backup-dashboard.db
+BACKUP_DASH_RETENTION_DAYS=90