Make manage.py use the service's database

manage.py only found the database via BACKUP_DASH_DB, which is set in
the systemd unit and not in an admin's shell, so it silently created a
new backup-dashboard.db in the current directory. API keys created
there were never seen by the service.

- Settings now come from /etc/backup-dashboard/backup-dashboard.env,
  which both the systemd unit (EnvironmentFile=) and manage.py read;
  real environment variables still take precedence
- manage.py prints the database it uses and refuses to create a new
  one unless --create is passed; --db overrides the path

Co-Authored-By: Claude Opus 5.5 <[email protected]>

README.md

@@ -85,7 +85,7 @@ cd backend
 python -m venv .venv && . .venv/bin/activate
 pip install -r requirements.txt -r requirements-dev.txt
 pytest
-python manage.py seed-demo                     # 30 days of fake history
+python manage.py --create seed-demo            # 30 days of fake history in ./backup-dashboard.db
 BACKUP_DASH_SERVE_STATIC=1 uvicorn --factory app.main:create_app --reload
 # → http://localhost:8000
 ```
@@ -93,9 +93,15 @@ BACKUP_DASH_SERVE_STATIC=1 uvicorn --factory app.main:create_app --reload
 ## Deployment (Linux, nginx)
 
 1. Copy the repo to `/opt/backup-dashboard`, create a venv at `/opt/backup-dashboard/venv`, and run `pip install -r backend/requirements.txt`.
-2. Create a `backupdash` system user, then install `deploy/backup-dashboard-api.service` into `/etc/systemd/system/` and enable it.
+2. Copy `deploy/backup-dashboard.env` to `/etc/backup-dashboard/backup-dashboard.env`. Both the service and `manage.py` read their settings from it.
-3. Copy `public/` to `/var/www/backup-dashboard/public`, install `deploy/nginx.conf`, get a certificate with certbot, and create users with `htpasswd -B`.
+3. Create a `backupdash` system user, then install `deploy/backup-dashboard-api.service` into `/etc/systemd/system/` and enable it. The database is created on first start.
-4. Run the `manage.py` commands as the service user with `BACKUP_DASH_DB=/var/lib/backup-dashboard/backup-dashboard.db`.
+4. Copy `public/` to `/var/www/backup-dashboard/public`, install `deploy/nginx.conf`, get a certificate with certbot, and create users with `htpasswd -B`.
+5. Run `manage.py` as the service user. It finds the database through the env file:
+   ```bash
+   cd /opt/backup-dashboard/backend
+   sudo -u backupdash /opt/backup-dashboard/venv/bin/python manage.py create-key veeam-acme
+   ```
+   It prints the database path it's using and won't create a new database unless you pass `--create`.
 
 Back up the SQLite file with `sqlite3 backup-dashboard.db ".backup copy.db"`. Don't use a plain `cp`
 while the service is running.

backend/app/settings.py

@@ -5,6 +5,9 @@ from pathlib import Path
 # The modal shows this many days of run history; retention can never be shorter.
 HISTORY_DAYS = 30
 
+# Shared by the systemd service (EnvironmentFile=) and manage.py, so both always use the same DB.
+DEFAULT_ENV_FILE = "/etc/backup-dashboard/backup-dashboard.env"
+
 
 @dataclass(frozen=True)
 class Settings:
@@ -14,9 +17,27 @@ class Settings:
     serve_static: bool = False
 
 
+def read_env_file(path: Path) -> dict[str, str]:
+    """Parse simple KEY=VALUE lines (the subset systemd's EnvironmentFile accepts)."""
+    values = {}
+    try:
+        lines = path.read_text().splitlines()
+    except FileNotFoundError:
+        return values
+    for line in lines:
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        values[key.strip()] = value.strip().strip("\"'")
+    return values
+
+
 def load_settings() -> Settings:
+    # Real environment variables win over the env file; the env file wins over dev defaults.
+    env = read_env_file(Path(os.environ.get("BACKUP_DASH_ENV_FILE", DEFAULT_ENV_FILE))) | os.environ
     return Settings(
-        db_path=Path(os.environ.get("BACKUP_DASH_DB", "backup-dashboard.db")),
+        db_path=Path(env.get("BACKUP_DASH_DB", "backup-dashboard.db")),
-        retention_days=max(HISTORY_DAYS, int(os.environ.get("BACKUP_DASH_RETENTION_DAYS", "90"))),
+        retention_days=max(HISTORY_DAYS, int(env.get("BACKUP_DASH_RETENTION_DAYS", "90"))),
-        serve_static=os.environ.get("BACKUP_DASH_SERVE_STATIC") == "1",
+        serve_static=env.get("BACKUP_DASH_SERVE_STATIC") == "1",
     )

backend/manage.py

@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
-"""Admin CLI. Uses the same BACKUP_DASH_DB env var as the API.
+"""Admin CLI. Finds the database the same way the API does: BACKUP_DASH_DB from the environment,
+else from /etc/backup-dashboard/backup-dashboard.env, else ./backup-dashboard.db (dev).
+Run it as the service user so files it creates stay writable by the API:
+
+    sudo -u backupdash /opt/backup-dashboard/venv/bin/python manage.py create-key veeam-acme
 
     python manage.py create-key veeam-acme      # prints the key once
     python manage.py list-keys
@@ -7,17 +11,19 @@
     python manage.py list-jobs
     python manage.py delete-job 12              # e.g. a decommissioned job stuck as Overdue
     python manage.py seed-demo                  # 30 days of fake history for trying the UI
+    python manage.py --db ./dev.db --create seed-demo
 """
 
 import argparse
 import random
 import sys
 from datetime import timedelta
+from pathlib import Path
 
 from app import db, repo
 from app.auth import generate_key, hash_key
 from app.models import IngestIn
-from app.settings import load_settings
+from app.settings import DEFAULT_ENV_FILE, load_settings
 from app.timeutil import to_db, utcnow
 
 
@@ -94,6 +100,8 @@ def seed_demo(conn, args):
 
 def main():
     p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--db", type=Path, help="database file (overrides BACKUP_DASH_DB and the env file)")
+    p.add_argument("--create", action="store_true", help="create the database if it doesn't exist")
     sub = p.add_subparsers(required=True)
     s = sub.add_parser("create-key"); s.add_argument("name"); s.set_defaults(fn=create_key)
     s = sub.add_parser("list-keys"); s.set_defaults(fn=list_keys)
@@ -103,9 +111,18 @@ def main():
     s = sub.add_parser("seed-demo"); s.set_defaults(fn=seed_demo)
     args = p.parse_args()
 
-    settings = load_settings()
+    db_path = args.db or load_settings().db_path
-    db.init_db(settings.db_path)
+    # Refuse to silently create a fresh DB: that almost always means we're looking in the wrong
-    conn = db.connect(settings.db_path)
+    # place, and anything written there (like a new API key) would be invisible to the service.
+    if not db_path.exists() and not args.create:
+        sys.exit(
+            f"Database not found: {db_path.resolve()}\n"
+            f"Set BACKUP_DASH_DB in {DEFAULT_ENV_FILE} (or pass --db), "
+            "or add --create to start a new database there."
+        )
+    print(f"Using database {db_path.resolve()}", file=sys.stderr)
+    db.init_db(db_path)
+    conn = db.connect(db_path)
     try:
         args.fn(conn, args)
     finally:

backend/tests/test_manage.py

@@ -0,0 +1,52 @@
+import subprocess
+import sys
+from pathlib import Path
+
+from app.settings import load_settings
+
+BACKEND = Path(__file__).resolve().parents[1]
+
+
+def run_manage(*args, env, cwd):
+    return subprocess.run([sys.executable, str(BACKEND / "manage.py"), *args],
+                          capture_output=True, text=True, env=env, cwd=cwd)
+
+
+def base_env(tmp_path, monkeypatch):
+    env_file = tmp_path / "backup-dashboard.env"
+    monkeypatch.setenv("BACKUP_DASH_ENV_FILE", str(env_file))
+    monkeypatch.delenv("BACKUP_DASH_DB", raising=False)
+    return env_file, {"BACKUP_DASH_ENV_FILE": str(env_file), "PYTHONPATH": str(BACKEND), "PATH": "/usr/bin:/bin"}
+
+
+def test_env_file_supplies_db_path_and_real_env_wins(tmp_path, monkeypatch):
+    env_file, _ = base_env(tmp_path, monkeypatch)
+    env_file.write_text('# comment\nBACKUP_DASH_DB="/srv/from-file.db"\nBACKUP_DASH_RETENTION_DAYS=120\n')
+    assert load_settings().db_path == Path("/srv/from-file.db")
+    assert load_settings().retention_days == 120
+    monkeypatch.setenv("BACKUP_DASH_DB", "/srv/from-env.db")
+    assert load_settings().db_path == Path("/srv/from-env.db")
+
+
+def test_manage_refuses_to_create_db_implicitly(tmp_path, monkeypatch):
+    _, env = base_env(tmp_path, monkeypatch)
+    workdir = tmp_path / "somewhere-else"
+    workdir.mkdir()
+    r = run_manage("create-key", "k", env=env, cwd=workdir)
+    assert r.returncode != 0
+    assert "Database not found" in r.stderr
+    assert not (workdir / "backup-dashboard.db").exists()
+
+
+def test_manage_uses_db_from_env_file(tmp_path, monkeypatch):
+    env_file, env = base_env(tmp_path, monkeypatch)
+    service_db = tmp_path / "service.db"
+    env_file.write_text(f"BACKUP_DASH_DB={service_db}\n")
+    assert run_manage("--create", "list-keys", env=env, cwd=tmp_path).returncode == 0  # service's first start
+    workdir = tmp_path / "somewhere-else"
+    workdir.mkdir()
+    r = run_manage("create-key", "veeam-acme", env=env, cwd=workdir)
+    assert r.returncode == 0, r.stderr
+    assert str(service_db) in r.stderr
+    assert "veeam-acme" in run_manage("list-keys", env=env, cwd=tmp_path).stdout
+    assert not (workdir / "backup-dashboard.db").exists()

deploy/backup-dashboard-api.service

@@ -6,8 +6,8 @@ After=network.target
 User=backupdash
 Group=backupdash
 WorkingDirectory=/opt/backup-dashboard/backend
-Environment=BACKUP_DASH_DB=/var/lib/backup-dashboard/backup-dashboard.db
+# Settings live in this file so manage.py reads exactly the same ones (see deploy/backup-dashboard.env)
-Environment=BACKUP_DASH_RETENTION_DAYS=90
+EnvironmentFile=/etc/backup-dashboard/backup-dashboard.env
 # Bound to localhost: only nginx can reach it, so nginx's auth can't be bypassed from outside.
 # One worker is plenty at this scale and keeps SQLite writes simple.
 ExecStart=/opt/backup-dashboard/venv/bin/uvicorn --factory app.main:create_app --host 127.0.0.1 --port 8000 --proxy-headers

deploy/backup-dashboard.env

@@ -0,0 +1,4 @@
+# Install as /etc/backup-dashboard/backup-dashboard.env
+# Read by both the API service and manage.py, so they always use the same database.
+BACKUP_DASH_DB=/var/lib/backup-dashboard/backup-dashboard.db
+BACKUP_DASH_RETENTION_DAYS=90