diff --git a/AGENTS.md b/AGENTS.md index afc7a7f..b522565 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,7 +34,7 @@ The web app listens on `0.0.0.0:8080`. - delete files with `deletefile`; - remove empty per-backup folders with `rmdir`. - Never delete a local archive unless upload completed and the remote object was verified. -- Mock `pvesh` and `rclone` in automated tests. Manual integration testing is limited to `scripts/integration-test-adguard.sh`. +- Mock `pvesh` and `rclone` in automated tests. Manual integration testing is limited to `scripts/integration-test.sh` and must only run when the user explicitly asks for a real backup test. ## Runtime data @@ -44,17 +44,20 @@ SQLite database: /opt/pve-cloud-backup/data/app.db ``` -Current important settings live in SQLite, not files. Common live values on this node have included: +Important runtime settings live in SQLite, not files: -- Proxmox node: `astrid` -- Proxmox storage: `hitachi` -- local dump path: `/mnt/pve/hitachi/dump` -- rclone remote: `onedrive` -- remote path: `pve-cloud-backup` -- timezone: `Europe/Berlin` -- manual test guest: `adguard`, VMID `110`, type `lxc` +- Proxmox node +- Proxmox backup storage +- local dump directory +- rclone executable path +- rclone remote name +- remote path inside that rclone remote +- timezone +- retention defaults +- Discord webhook URL +- CORS origins -Do not hardcode those into normal app behavior. They are local operational context. +Treat all node names, storage names, guest names, VMIDs, rclone remotes, and filesystem paths as operator-specific runtime data. Do not hardcode deployment-specific values in application code or publish-facing docs. ## Backend map @@ -187,25 +190,21 @@ curl -fsS http://127.0.0.1:8080/api/backups | python3 -m json.tool Manual integration test: ```bash -API_URL=http://127.0.0.1:8080 /opt/pve-cloud-backup/scripts/integration-test-adguard.sh +INTEGRATION_VMID= API_URL=http://127.0.0.1:8080 /opt/pve-cloud-backup/scripts/integration-test.sh ``` Only run the manual integration test when the user explicitly wants a real Proxmox/rclone backup test. ## Git workflow -Remote: - -```text -origin ssh://gitea@gitea.reversed.dev:2222/space/pve-cloud-backups.git -``` - Default branch: ```text main ``` +Do not document private deployment remotes, hostnames, or organization names in committed files. Local git remotes belong in `.git/config`, not project documentation. + Before committing, check: ```bash diff --git a/README.md b/README.md index a5a060e..69d4a3c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The application is intentionally small: - Vue 3 + TypeScript + Tailwind frontend, compiled once into `static/` - SQLite for all runtime configuration and state - `pvesh` for Proxmox API access -- `rclone copyto`, `rclone lsjson`, and `rclone deletefile` for exact remote objects +- `rclone copyto`, `rclone lsjson`, `rclone deletefile`, and `rclone rmdir` for exact remote objects and empty per-backup folders - systemd services, no Docker ## Layout @@ -103,7 +103,7 @@ Retention cleanup runs in two places: - immediately after a backup completes for that job; - periodically from the worker service, once per hour, so changed rules and day-based expiry are enforced even if no new backup runs. -Remote deletion uses exact-object `rclone deletefile` calls for backup records already known in SQLite. +Remote deletion uses exact-object `rclone deletefile` calls for backup records already known in SQLite, then best-effort `rclone rmdir` cleanup for the empty per-backup folder. ## Testing @@ -121,13 +121,39 @@ cd /opt/pve-cloud-backup/frontend npm run build ``` -Manual integration test for the provided Proxmox test target: +Manual integration test against a real Proxmox guest: ```bash -API_URL=http://127.0.0.1:8080 /opt/pve-cloud-backup/scripts/integration-test-adguard.sh +INTEGRATION_VMID= API_URL=http://127.0.0.1:8080 /opt/pve-cloud-backup/scripts/integration-test.sh ``` -This queues a backup for guest `adguard` / VMID `110` using storage `hitachi` and dump path `/mnt/pve/hitachi/dump`. Expected archive size is approximately 250 MB. The script is explicitly for manual integration testing and does not hardcode these values into normal app behavior. +Optional environment variables: + +- `INTEGRATION_GUEST_NAME` to assert the discovered guest name before queueing. +- `INTEGRATION_STORAGE` to override the configured Proxmox backup storage. +- `INTEGRATION_LOCAL_BACKUP_DIR` to assert the configured local dump directory. +- `INTEGRATION_CRON` to choose the created job schedule. Default: `0 3 * * *`. +- `INTEGRATION_RUN_NOW=0` to create the job without immediately queueing it. + +The script creates a disabled backup job and, by default, queues it once. Only run it when you intentionally want a real Proxmox backup and rclone upload. + +## Updates + +After pulling new code, apply it with: + +```bash +cd /opt/pve-cloud-backup +git pull --ff-only +./scripts/apply-update.sh +``` + +For a stricter update that runs backend tests before restarting services: + +```bash +RUN_TESTS=1 ./scripts/apply-update.sh +``` + +The update script backs up `data/app.db`, rebuilds dependencies/assets, runs migrations, installs service units, restarts services, and checks API health. ## Uninstall diff --git a/backend/tests/test_backup_service.py b/backend/tests/test_backup_service.py index 6a54d77..6bd2e08 100644 --- a/backend/tests/test_backup_service.py +++ b/backend/tests/test_backup_service.py @@ -12,7 +12,7 @@ def _settings(tmp_path): return { "setup_complete": False, "proxmox_node": "pve", - "proxmox_storage": "hitachi", + "proxmox_storage": "backup-store", "local_backup_dir": str(tmp_path), "rclone_path": "/usr/bin/rclone", "rclone_remote": "onedrive", @@ -31,12 +31,12 @@ def _job(): return create_job( { "guest_vmid": 110, - "guest_name": "adguard", + "guest_name": "test-guest", "guest_type": "vm", "node": "pve", "enabled": True, "cron_schedule": "0 2 * * *", - "proxmox_storage": "hitachi", + "proxmox_storage": "backup-store", "backup_mode": "snapshot", "compression": "zstd", "retention_type": "latest", @@ -97,7 +97,7 @@ def test_pve_running_without_upid_fails_without_polling_none(isolated_db, no_not conn.execute( """ INSERT INTO backups(id, guest_vmid, guest_name, guest_type, node, state, started_at, updated_at) - VALUES ('missing-upid', 110, 'adguard', 'lxc', 'astrid', 'pve_running', ?, ?) + VALUES ('missing-upid', 110, 'test-guest', 'lxc', 'pve', 'pve_running', ?, ?) """, (utc_now(), utc_now()), ) @@ -156,7 +156,7 @@ def test_retention_latest_selects_expired_remote_backups( id, job_id, guest_vmid, guest_name, guest_type, node, state, remote_path, started_at, completed_at, updated_at ) - VALUES (?, ?, 110, 'adguard', 'vm', 'pve', 'completed', ?, ?, ?, ?) + VALUES (?, ?, 110, 'test-guest', 'vm', 'pve', 'completed', ?, ?, ?, ?) """, ( backup_id, @@ -196,7 +196,7 @@ def test_startup_recovery_resumes_known_states(isolated_db, no_notifications, mo conn.execute( """ INSERT INTO backups(id, guest_vmid, guest_name, guest_type, node, state, started_at, updated_at) - VALUES (?, 110, 'adguard', 'vm', 'pve', ?, ?, ?) + VALUES (?, 110, 'test-guest', 'vm', 'pve', ?, ?, ?) """, (backup_id, state, utc_now(), utc_now()), ) @@ -227,7 +227,7 @@ def test_retention_sweep_applies_all_jobs_with_completed_remote_backups( id, job_id, guest_vmid, guest_name, guest_type, node, state, remote_path, started_at, completed_at, updated_at ) - VALUES ('sweep-test', ?, 110, 'adguard', 'vm', 'pve', 'completed', + VALUES ('sweep-test', ?, 110, 'test-guest', 'vm', 'pve', 'completed', 'onedrive:pve-cloud-backup/sweep-test/archive.tar.zst', ?, ?, ?) """, (job["id"], utc_now(), utc_now(), utc_now()), @@ -260,7 +260,7 @@ def test_manual_backup_delete_removes_remote_and_known_local_file( id, guest_vmid, guest_name, guest_type, node, state, local_path, remote_path, started_at, completed_at, updated_at ) - VALUES ('manual-delete', 110, 'adguard', 'vm', 'pve', 'completed', + VALUES ('manual-delete', 110, 'test-guest', 'vm', 'pve', 'completed', ?, 'onedrive:pve-cloud-backup/manual-delete/archive.tar.zst', ?, ?, ?) """, (str(local_file), utc_now(), utc_now(), utc_now()), @@ -303,7 +303,7 @@ def test_manual_backup_delete_removes_empty_remote_folder_when_file_is_already_m id, guest_vmid, guest_name, guest_type, node, state, remote_path, started_at, completed_at, updated_at ) - VALUES ('missing-remote-file', 110, 'adguard', 'vm', 'pve', 'completed', + VALUES ('missing-remote-file', 110, 'test-guest', 'vm', 'pve', 'completed', 'onedrive:pve-cloud-backup/missing-remote-file/archive.tar.zst', ?, ?, ?) """, (utc_now(), utc_now(), utc_now()), @@ -327,7 +327,7 @@ def test_manual_backup_delete_refuses_active_backup(isolated_db, no_notification conn.execute( """ INSERT INTO backups(id, guest_vmid, guest_name, guest_type, node, state, started_at, updated_at) - VALUES ('active-delete', 110, 'adguard', 'vm', 'pve', 'uploading', ?, ?) + VALUES ('active-delete', 110, 'test-guest', 'vm', 'pve', 'uploading', ?, ?) """, (utc_now(), utc_now()), ) @@ -346,7 +346,7 @@ def test_deleting_already_deleted_backup_purges_metadata_row(isolated_db, no_not id, guest_vmid, guest_name, guest_type, node, state, remote_path, started_at, deleted_at, updated_at ) - VALUES ('purge-deleted', 110, 'adguard', 'vm', 'pve', 'deleted', + VALUES ('purge-deleted', 110, 'test-guest', 'vm', 'pve', 'deleted', 'onedrive:pve-cloud-backup/purge-deleted/archive.tar.zst', ?, ?, ?) """, (utc_now(), utc_now(), utc_now()), @@ -380,7 +380,7 @@ def test_delete_job_deletes_backups_before_removing_job( conn.execute( """ INSERT INTO backups(id, job_id, guest_vmid, guest_name, guest_type, node, state, remote_path, started_at, completed_at, updated_at) - VALUES ('job-delete-backup', ?, 110, 'adguard', 'vm', 'pve', 'completed', + VALUES ('job-delete-backup', ?, 110, 'test-guest', 'vm', 'pve', 'completed', 'onedrive:pve-cloud-backup/job-delete-backup/archive.tar.zst', ?, ?, ?) """, (job["id"], utc_now(), utc_now(), utc_now()), diff --git a/backend/tests/test_commands.py b/backend/tests/test_commands.py index 59e40cf..a4c5106 100644 --- a/backend/tests/test_commands.py +++ b/backend/tests/test_commands.py @@ -19,7 +19,7 @@ def test_start_proxmox_backup_parses_upid(monkeypatch): assert commands.start_proxmox_backup( node="pve", vmid=110, - storage="hitachi", + storage="backup-store", mode="snapshot", compression="zstd", ) == expected @@ -33,7 +33,7 @@ def test_start_proxmox_backup_parses_plain_upid(monkeypatch): assert commands.start_proxmox_backup( node="pve", vmid=110, - storage="hitachi", + storage="backup-store", mode="snapshot", compression="zstd", ) == expected @@ -48,7 +48,7 @@ def test_start_proxmox_backup_parses_upid_after_info_lines(monkeypatch): assert commands.start_proxmox_backup( node="pve", vmid=110, - storage="hitachi", + storage="backup-store", mode="snapshot", compression="zstd", ) == expected @@ -56,7 +56,7 @@ def test_start_proxmox_backup_parses_upid_after_info_lines(monkeypatch): def test_list_guests_normalizes_vm_and_lxc(monkeypatch): payload = [ - {"vmid": "110", "name": "adguard", "type": "qemu", "node": "pve", "status": "running"}, + {"vmid": "110", "name": "test-guest", "type": "qemu", "node": "pve", "status": "running"}, {"vmid": "111", "name": "ct", "type": "lxc", "node": "pve", "status": "stopped"}, ] @@ -65,7 +65,7 @@ def test_list_guests_normalizes_vm_and_lxc(monkeypatch): guests = commands.list_guests() assert guests == [ - {"vmid": 110, "name": "adguard", "type": "vm", "node": "pve", "status": "running"}, + {"vmid": 110, "name": "test-guest", "type": "vm", "node": "pve", "status": "running"}, {"vmid": 111, "name": "ct", "type": "lxc", "node": "pve", "status": "stopped"}, ] diff --git a/backend/tests/test_settings_api.py b/backend/tests/test_settings_api.py index dccd92b..0e69b53 100644 --- a/backend/tests/test_settings_api.py +++ b/backend/tests/test_settings_api.py @@ -9,8 +9,8 @@ def test_plain_onedrive_remote_is_accepted(monkeypatch): { "setup_complete": True, "proxmox_node": "pve", - "proxmox_storage": "hitachi", - "local_backup_dir": "/mnt/pve/hitachi/dump", + "proxmox_storage": "backup-store", + "local_backup_dir": "/mnt/pve/backups/dump", "rclone_path": "/usr/bin/rclone", "rclone_remote": "onedrive", "rclone_remote_path": "Backups/Proxmox", @@ -31,8 +31,8 @@ def test_invalid_timezone_is_rejected(monkeypatch): { "setup_complete": True, "proxmox_node": "pve", - "proxmox_storage": "hitachi", - "local_backup_dir": "/mnt/pve/hitachi/dump", + "proxmox_storage": "backup-store", + "local_backup_dir": "/mnt/pve/backups/dump", "rclone_path": "/usr/bin/rclone", "rclone_remote": "onedrive", "rclone_remote_path": "Backups/Proxmox", diff --git a/scripts/integration-test-adguard.sh b/scripts/integration-test-adguard.sh deleted file mode 100755 index 8151617..0000000 --- a/scripts/integration-test-adguard.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -API_URL="${API_URL:-http://127.0.0.1:8080}" - -python3 - "$API_URL" <<'PY' -import json -import sys -import urllib.error -import urllib.request - -api = sys.argv[1].rstrip("/") - -def request(method, path, body=None): - data = None if body is None else json.dumps(body).encode() - req = urllib.request.Request( - api + path, - data=data, - method=method, - headers={"Content-Type": "application/json"}, - ) - try: - with urllib.request.urlopen(req, timeout=30) as res: - return json.loads(res.read().decode()) - except urllib.error.HTTPError as exc: - raise SystemExit(f"{method} {path} failed: {exc.code} {exc.read().decode()}") from exc - -settings = request("GET", "/api/settings") -if settings.get("proxmox_storage") != "hitachi": - raise SystemExit("Expected configured Proxmox storage to be 'hitachi'") -if settings.get("local_backup_dir") != "/mnt/pve/hitachi/dump": - raise SystemExit("Expected configured local backup directory to be '/mnt/pve/hitachi/dump'") - -guests = request("GET", "/api/guests") -guest = next((g for g in guests if int(g["vmid"]) == 110 and g["name"] == "adguard"), None) -if not guest: - raise SystemExit("Expected Proxmox guest adguard with VMID 110") - -job = request("POST", "/api/jobs", { - "guest_vmid": 110, - "guest_name": "adguard", - "guest_type": guest["type"], - "node": guest["node"], - "enabled": False, - "cron_schedule": "0 3 * * *", - "proxmox_storage": "hitachi", - "backup_mode": settings["default_backup_mode"], - "compression": settings["default_compression"], - "retention_type": "days", - "retention_value": 1, -}) -backup = request("POST", f"/api/jobs/{job['id']}/run") -print(json.dumps({"job": job, "queued_backup": backup}, indent=2)) -print("Queued integration backup for adguard VMID 110 on storage hitachi (/mnt/pve/hitachi/dump). Expected archive size is approximately 250 MB.") -PY diff --git a/scripts/integration-test.sh b/scripts/integration-test.sh new file mode 100755 index 0000000..5987571 --- /dev/null +++ b/scripts/integration-test.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +API_URL="${API_URL:-http://127.0.0.1:8080}" +INTEGRATION_VMID="${INTEGRATION_VMID:-}" +INTEGRATION_GUEST_NAME="${INTEGRATION_GUEST_NAME:-}" +INTEGRATION_STORAGE="${INTEGRATION_STORAGE:-}" +INTEGRATION_LOCAL_BACKUP_DIR="${INTEGRATION_LOCAL_BACKUP_DIR:-}" +INTEGRATION_CRON="${INTEGRATION_CRON:-0 3 * * *}" +INTEGRATION_RETENTION_TYPE="${INTEGRATION_RETENTION_TYPE:-latest}" +INTEGRATION_RETENTION_VALUE="${INTEGRATION_RETENTION_VALUE:-1}" +INTEGRATION_RUN_NOW="${INTEGRATION_RUN_NOW:-1}" + +if [[ -z "$INTEGRATION_VMID" ]]; then + echo "Set INTEGRATION_VMID to the Proxmox VMID/LXC ID to test." >&2 + echo "Example:" >&2 + echo " INTEGRATION_VMID=110 INTEGRATION_STORAGE=local API_URL=http://127.0.0.1:8080 $0" >&2 + exit 1 +fi + +python3 - "$API_URL" \ + "$INTEGRATION_VMID" \ + "$INTEGRATION_GUEST_NAME" \ + "$INTEGRATION_STORAGE" \ + "$INTEGRATION_LOCAL_BACKUP_DIR" \ + "$INTEGRATION_CRON" \ + "$INTEGRATION_RETENTION_TYPE" \ + "$INTEGRATION_RETENTION_VALUE" \ + "$INTEGRATION_RUN_NOW" <<'PY' +import json +import sys +import urllib.error +import urllib.request + +api = sys.argv[1].rstrip("/") +vmid = int(sys.argv[2]) +expected_name = sys.argv[3] or None +storage_override = sys.argv[4] or None +expected_local_dir = sys.argv[5] or None +cron = sys.argv[6] +retention_type = sys.argv[7] +retention_value = int(sys.argv[8]) +run_now = sys.argv[9] == "1" + + +def request(method, path, body=None): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request( + api + path, + data=data, + method=method, + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=30) as res: + return json.loads(res.read().decode()) + except urllib.error.HTTPError as exc: + raise SystemExit(f"{method} {path} failed: {exc.code} {exc.read().decode()}") from exc + + +settings = request("GET", "/api/settings") +if not settings.get("setup_complete"): + raise SystemExit("Setup is not complete. Configure the app before running integration tests.") + +if expected_local_dir and settings.get("local_backup_dir") != expected_local_dir: + raise SystemExit( + f"Expected configured local backup directory to be {expected_local_dir!r}, " + f"found {settings.get('local_backup_dir')!r}" + ) + +guests = request("GET", "/api/guests") +guest = next((g for g in guests if int(g["vmid"]) == vmid), None) +if not guest: + raise SystemExit(f"Expected Proxmox guest with VMID {vmid}") +if expected_name and guest.get("name") != expected_name: + raise SystemExit(f"Expected guest {vmid} to be named {expected_name!r}, found {guest.get('name')!r}") + +storage = storage_override or settings["proxmox_storage"] +job = request("POST", "/api/jobs", { + "guest_vmid": vmid, + "guest_name": guest["name"], + "guest_type": guest["type"], + "node": guest["node"], + "enabled": False, + "cron_schedule": cron, + "proxmox_storage": storage, + "backup_mode": settings["default_backup_mode"], + "compression": settings["default_compression"], + "retention_type": retention_type, + "retention_value": retention_value, +}) + +result = {"job": job} +if run_now: + result["queued_backup"] = request("POST", f"/api/jobs/{job['id']}/run") + +print(json.dumps(result, indent=2)) +print(f"Created integration job for VMID {vmid} using storage {storage!r}.") +if run_now: + print("Queued the job immediately. Watch Backup history or worker logs for completion.") +else: + print("Did not queue the job because INTEGRATION_RUN_NOW is not 1.") +PY