diff --git a/CLAUDE.md b/CLAUDE.md index 2ace3c4..2f06376 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,6 +54,7 @@ Important runtime settings live in SQLite, not files: - remote path inside that rclone remote - timezone - retention defaults +- keep local backups after upload (global toggle) - Discord webhook URL - CORS origins @@ -102,6 +103,7 @@ Expected states: - `completed` - `failed` - `deleting` +- `local_downloading` - `deleted` Important behavior: @@ -115,10 +117,14 @@ Important behavior: :// ``` +- `local_deleting` only unlinks the local archive when the `keep_local_backups` setting is off (the default). When it's on, the archive stays in `local_backup_dir` after `completed` and `local_path` stays populated. +- Deleting a backup (manual delete, job-cascade delete, retention, or crash-recovery resume of a `deleting` backup) always removes both the remote object and any locally-retained archive, regardless of `keep_local_backups` — that setting only affects the immediate post-upload step, not deletion. - Deleting a backup removes the exact remote file and then attempts to remove the empty `` folder. - Deleting a backup already in `deleted` state purges the SQLite metadata row. - Deleting a job deletes all known non-active backups for that job first, then removes the job. - Retention runs after backup completion and hourly from the worker. +- A completed backup whose local archive is missing (never kept, or deleted via "Delete local copy") can be re-fetched from the remote into `local_backup_dir` via `POST /api/backups/{id}/download`; this is a transient `local_downloading` state that resumes on worker restart. Since `local_backup_dir` is the actual Proxmox storage dump directory, a downloaded archive shows up in the Proxmox UI for the operator to restore from directly — this app never runs the restore itself. +- A locally-retained (or downloaded) copy can be removed independently of the remote object via `DELETE /api/backups/{id}/local`, without affecting `state` or `remote_path`. ## Applying updates after pulling changes diff --git a/backend/app/backup_service.py b/backend/app/backup_service.py index 8c9c666..0d901cd 100644 --- a/backend/app/backup_service.py +++ b/backend/app/backup_service.py @@ -22,14 +22,7 @@ ACTIVE_STATES = { "deleting", } RUNNING_STATES = ACTIVE_STATES - {"queued"} -RECOVERABLE_STATES = { - "pve_running", - "local_ready", - "uploading", - "remote_ready", - "local_deleting", - "deleting", -} +RECOVERABLE_STATES = (ACTIVE_STATES - {"queued"}) | {"local_downloading"} TERMINAL_STATES = {"completed", "failed", "deleted"} @@ -298,10 +291,11 @@ async def process_backup(backup_id: str) -> None: backup = update_backup(backup_id, state="local_deleting") if backup["state"] == "local_deleting": - if backup.get("local_path"): + if not settings.get("keep_local_backups") and backup.get("local_path"): local_path = Path(backup["local_path"]) if local_path.exists(): local_path.unlink() + backup = update_backup(backup_id, local_path=None) backup = update_backup(backup_id, state="completed", completed_at=utc_now()) await notify("backup_completed", backup=backup) await apply_retention_for_job(backup["job_id"]) @@ -310,15 +304,26 @@ async def process_backup(backup_id: str) -> None: await notify("backup_failed", backup=failed) -async def delete_remote_backup(backup: dict, reason: str) -> None: - if not backup.get("remote_path"): - update_backup(backup["id"], state="deleted", deleted_at=utc_now()) - return +async def delete_remote_backup(backup: dict, reason: str) -> dict: settings = get_settings() update_backup(backup["id"], state="deleting") - await _delete_remote_file_and_empty_parent(settings, backup["remote_path"]) - deleted = update_backup(backup["id"], state="deleted", deleted_at=utc_now()) + if backup.get("remote_path"): + exists = await asyncio.to_thread( + commands.rclone_object_exists, + settings["rclone_path"], + backup["remote_path"], + ) + if exists: + await _delete_remote_file_and_empty_parent(settings, backup["remote_path"]) + else: + await _remove_empty_remote_parent(settings, backup["remote_path"]) + if backup.get("local_path"): + local_path = Path(backup["local_path"]) + if local_path.exists() and local_path.is_file(): + local_path.unlink() + deleted = update_backup(backup["id"], state="deleted", deleted_at=utc_now(), local_path=None) await notify("remote_deleted", backup=deleted, message=reason) + return deleted async def delete_backup_manually(backup_id: str, reason: str = "manual deletion") -> dict: @@ -327,28 +332,61 @@ async def delete_backup_manually(backup_id: str, reason: str = "manual deletion" return purge_deleted_backup(backup_id) if backup["state"] in ACTIVE_STATES: raise RuntimeError(f"Backup {backup_id} is active in state {backup['state']} and cannot be deleted") + return await delete_remote_backup(backup, reason) + + +def _archive_name_from_remote(remote_path: str) -> str: + return Path(remote_path.split(":", 1)[1]).name + + +async def download_backup_locally(backup_id: str) -> dict: + backup = _backup_row(backup_id) + if backup["state"] != "completed" or not backup.get("remote_path"): + raise RuntimeError(f"Backup {backup_id} has no completed remote archive to download") + if backup.get("local_path") and Path(backup["local_path"]).exists(): + return backup + + now = utc_now() + with db() as conn: + cur = conn.execute( + "UPDATE backups SET state = 'local_downloading', updated_at = ? WHERE id = ? AND state = 'completed'", + (now, backup_id), + ) + if cur.rowcount == 0: + raise RuntimeError(f"Backup {backup_id} is busy and cannot be downloaded right now") settings = get_settings() - if backup.get("remote_path") and backup["state"] != "deleted": - exists = await asyncio.to_thread( - commands.rclone_object_exists, + target = Path(settings["local_backup_dir"]) / _archive_name_from_remote(backup["remote_path"]) + try: + await asyncio.to_thread( + commands.rclone_download, settings["rclone_path"], backup["remote_path"], + str(target), ) - if exists: - update_backup(backup_id, state="deleting") - await _delete_remote_file_and_empty_parent(settings, backup["remote_path"]) - else: - await _remove_empty_remote_parent(settings, backup["remote_path"]) + if not target.exists(): + raise RuntimeError("Download finished but local archive was not found") + except Exception: + update_backup(backup_id, state="completed") + raise - if backup.get("local_path"): - local_path = Path(backup["local_path"]) - if local_path.exists() and local_path.is_file(): - local_path.unlink() + downloaded = update_backup(backup_id, state="completed", local_path=str(target)) + await notify("local_downloaded", backup=downloaded) + return downloaded - deleted = update_backup(backup_id, state="deleted", deleted_at=utc_now()) - await notify("remote_deleted", backup=deleted, message=reason) - return deleted + +async def delete_local_copy(backup_id: str) -> dict: + backup = _backup_row(backup_id) + if backup["state"] != "completed": + raise RuntimeError(f"Backup {backup_id} is in state {backup['state']} and has no local copy to delete") + if not backup.get("local_path"): + return backup + local_path = Path(backup["local_path"]) + if local_path.exists() and local_path.is_file(): + local_path.unlink() + updated = update_backup(backup_id, local_path=None) + await notify("local_deleted", backup=updated) + return updated async def delete_job_with_backups(job_id: int) -> dict: @@ -440,6 +478,12 @@ async def recover_unfinished() -> None: await notify("recovery_action", backup=backup, message=f"resuming state {backup['state']}") if backup["state"] == "deleting": await delete_remote_backup(backup, "recovery delete resume") + elif backup["state"] == "local_downloading": + update_backup(backup["id"], state="completed") + try: + await download_backup_locally(backup["id"]) + except Exception as exc: + await notify("recovery_action", backup=backup, message=f"resume download failed: {exc}") else: await process_backup(backup["id"]) diff --git a/backend/app/commands.py b/backend/app/commands.py index 58d8405..f65be63 100644 --- a/backend/app/commands.py +++ b/backend/app/commands.py @@ -193,6 +193,10 @@ def rclone_copyto(rclone_path: str, local_path: str, remote_object: str) -> None run([rclone_path, "copyto", local_path, remote_object], timeout=None) +def rclone_download(rclone_path: str, remote_object: str, local_path: str) -> None: + run([rclone_path, "copyto", remote_object, local_path], timeout=None) + + def rclone_object_exists(rclone_path: str, remote_object: str) -> bool: try: run([rclone_path, "lsjson", remote_object], timeout=120) diff --git a/backend/app/main.py b/backend/app/main.py index 66ec8bb..7ead4ac 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -15,6 +15,8 @@ from .backup_service import ( dashboard, delete_backup_manually, delete_job_with_backups, + delete_local_copy, + download_backup_locally, get_backup, list_backups, process_backup, @@ -40,6 +42,7 @@ class SettingsPayload(BaseModel): default_retention_value: int = 30 max_concurrent_backups: int = 1 timezone: str = "UTC" + keep_local_backups: bool = False setup_complete: bool = False @@ -252,6 +255,28 @@ async def delete_backup(backup_id: str): raise HTTPException(409, str(exc)) from exc +@app.post("/api/backups/{backup_id}/download") +async def download_backup(backup_id: str): + try: + return await download_backup_locally(backup_id) + except KeyError as exc: + raise HTTPException(404, str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(409, str(exc)) from exc + except Exception as exc: + raise HTTPException(502, f"Download failed: {exc}") from exc + + +@app.delete("/api/backups/{backup_id}/local") +async def delete_backup_local_copy(backup_id: str): + try: + return await delete_local_copy(backup_id) + except KeyError as exc: + raise HTTPException(404, str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(409, str(exc)) from exc + + if (STATIC_DIR / "assets").exists(): app.mount("/assets", StaticFiles(directory=STATIC_DIR / "assets"), name="assets") diff --git a/backend/app/migrations.py b/backend/app/migrations.py index 0e5b32b..8815112 100644 --- a/backend/app/migrations.py +++ b/backend/app/migrations.py @@ -46,6 +46,7 @@ DEFAULT_SETTINGS = { "default_retention_value": "30", "max_concurrent_backups": "1", "timezone": _system_timezone(), + "keep_local_backups": "false", } @@ -186,3 +187,17 @@ def run_migrations() -> None: "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)", (5, now), ) + if 6 not in applied: + now = utc_now() + conn.execute( + """ + INSERT INTO settings(key, value, updated_at) + VALUES ('keep_local_backups', 'false', ?) + ON CONFLICT(key) DO NOTHING + """, + (now,), + ) + conn.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)", + (6, now), + ) diff --git a/backend/app/notifications.py b/backend/app/notifications.py index a47910a..57b784f 100644 --- a/backend/app/notifications.py +++ b/backend/app/notifications.py @@ -18,6 +18,8 @@ COLORS = { "backup_failed": 0xE74C3C, "remote_deleted": 0xF39C12, "recovery_action": 0x95A5A6, + "local_downloaded": 0x1ABC9C, + "local_deleted": 0x95A5A6, } diff --git a/backend/app/settings_store.py b/backend/app/settings_store.py index 41b709c..f584f21 100644 --- a/backend/app/settings_store.py +++ b/backend/app/settings_store.py @@ -7,7 +7,7 @@ from .migrations import DEFAULT_SETTINGS, run_migrations INT_KEYS = {"default_retention_value", "max_concurrent_backups"} -BOOL_KEYS = {"setup_complete"} +BOOL_KEYS = {"setup_complete", "keep_local_backups"} def parse_value(key: str, value: str): diff --git a/backend/tests/test_backup_service.py b/backend/tests/test_backup_service.py index cae1e83..92dd917 100644 --- a/backend/tests/test_backup_service.py +++ b/backend/tests/test_backup_service.py @@ -71,6 +71,34 @@ def test_backup_state_transitions_complete_and_delete_local_safely( assert not archive.exists() +def test_keep_local_backups_setting_preserves_archive_after_completion( + isolated_db, no_notifications, tmp_path, monkeypatch +): + from app import backup_service, commands + + settings = _settings(tmp_path) + settings["keep_local_backups"] = True + set_settings(settings) + job = _job() + backup = backup_service.queue_backup_for_job(job) + archive = tmp_path / "vzdump-qemu-110-2026_01_01-00_00_00.vma.zst" + archive.write_bytes(b"backup") + + monkeypatch.setattr(commands, "start_proxmox_backup", lambda **kwargs: "UPID:pve:test") + monkeypatch.setattr(commands, "get_task_status", lambda node, upid: {"status": "stopped", "exitstatus": "OK"}) + monkeypatch.setattr(commands, "discover_archive", lambda *args: archive) + monkeypatch.setattr(commands, "rclone_copyto", lambda *args: None) + monkeypatch.setattr(commands, "rclone_object_exists", lambda *args: True) + monkeypatch.setattr(backup_service, "apply_retention_for_job", lambda job_id: asyncio.sleep(0)) + + asyncio.run(backup_service.process_backup(backup["id"])) + final = backup_service.get_backup(backup["id"]) + + assert final["state"] == "completed" + assert final["local_path"] == str(archive) + assert archive.exists() + + def test_only_one_worker_can_claim_queued_backup(isolated_db, no_notifications, tmp_path): from app import backup_service @@ -391,3 +419,180 @@ def test_delete_job_deletes_backups_before_removing_job( assert deleted == [("job-delete-backup", f"job {job['id']} deleted")] with db() as conn: assert conn.execute("SELECT COUNT(*) AS count FROM backup_jobs WHERE id = ?", (job["id"],)).fetchone()["count"] == 0 + + +def test_download_backup_locally_refetches_missing_archive( + isolated_db, no_notifications, tmp_path, monkeypatch +): + from app import backup_service, commands + + set_settings(_settings(tmp_path)) + downloads = [] + + def fake_rclone_download(rclone_path, remote_object, local_path): + downloads.append((rclone_path, remote_object, local_path)) + Path(local_path).write_bytes(b"restored") + + monkeypatch.setattr(commands, "rclone_download", fake_rclone_download) + with db() as conn: + conn.execute( + """ + INSERT INTO backups( + id, guest_vmid, guest_name, guest_type, node, state, + remote_path, started_at, completed_at, updated_at + ) + VALUES ('download-test', 110, 'test-guest', 'vm', 'pve', 'completed', + 'onedrive:pve-cloud-backup/download-test/archive.tar.zst', ?, ?, ?) + """, + (utc_now(), utc_now(), utc_now()), + ) + + result = asyncio.run(backup_service.download_backup_locally("download-test")) + + expected_path = tmp_path / "archive.tar.zst" + assert result["state"] == "completed" + assert result["local_path"] == str(expected_path) + assert expected_path.read_bytes() == b"restored" + assert downloads == [ + ("/usr/bin/rclone", "onedrive:pve-cloud-backup/download-test/archive.tar.zst", str(expected_path)) + ] + + +def test_download_backup_locally_is_noop_when_local_file_present( + isolated_db, no_notifications, tmp_path, monkeypatch +): + from app import backup_service, commands + + set_settings(_settings(tmp_path)) + local_file = tmp_path / "already-here.tar.zst" + local_file.write_bytes(b"existing") + monkeypatch.setattr(commands, "rclone_download", lambda *args: pytest.fail("rclone_download should not run")) + + with db() as conn: + conn.execute( + """ + INSERT INTO backups( + id, guest_vmid, guest_name, guest_type, node, state, + local_path, remote_path, started_at, completed_at, updated_at + ) + VALUES ('download-noop', 110, 'test-guest', 'vm', 'pve', 'completed', + ?, 'onedrive:pve-cloud-backup/download-noop/archive.tar.zst', ?, ?, ?) + """, + (str(local_file), utc_now(), utc_now(), utc_now()), + ) + + result = asyncio.run(backup_service.download_backup_locally("download-noop")) + + assert result["local_path"] == str(local_file) + assert result["state"] == "completed" + + +def test_delete_local_copy_removes_only_local_file(isolated_db, no_notifications, tmp_path): + from app import backup_service + + local_file = tmp_path / "keep-local.tar.zst" + local_file.write_bytes(b"local copy") + + with db() as conn: + conn.execute( + """ + INSERT INTO backups( + id, guest_vmid, guest_name, guest_type, node, state, + local_path, remote_path, started_at, completed_at, updated_at + ) + VALUES ('delete-local', 110, 'test-guest', 'vm', 'pve', 'completed', + ?, 'onedrive:pve-cloud-backup/delete-local/archive.tar.zst', ?, ?, ?) + """, + (str(local_file), utc_now(), utc_now(), utc_now()), + ) + + result = asyncio.run(backup_service.delete_local_copy("delete-local")) + + assert not local_file.exists() + assert result["local_path"] is None + assert result["state"] == "completed" + assert result["remote_path"] == "onedrive:pve-cloud-backup/delete-local/archive.tar.zst" + + +def test_delete_local_copy_noop_without_local_path(isolated_db, no_notifications): + from app import backup_service + + with db() as conn: + conn.execute( + """ + INSERT INTO backups( + id, guest_vmid, guest_name, guest_type, node, state, + remote_path, started_at, completed_at, updated_at + ) + VALUES ('no-local', 110, 'test-guest', 'vm', 'pve', 'completed', + 'onedrive:pve-cloud-backup/no-local/archive.tar.zst', ?, ?, ?) + """, + (utc_now(), utc_now(), utc_now()), + ) + + result = asyncio.run(backup_service.delete_local_copy("no-local")) + + assert result["local_path"] is None + assert result["state"] == "completed" + + +def test_retention_delete_also_removes_local_copy( + isolated_db, no_notifications, tmp_path, monkeypatch +): + from app import backup_service, commands + + set_settings(_settings(tmp_path)) + job = _job() + with db() as conn: + conn.execute("UPDATE backup_jobs SET retention_value = 0 WHERE id = ?", (job["id"],)) + local_file = tmp_path / "retention-local.tar.zst" + local_file.write_bytes(b"local copy") + + monkeypatch.setattr(commands, "rclone_object_exists", lambda *args: True) + monkeypatch.setattr(commands, "rclone_delete_file", lambda *args: None) + monkeypatch.setattr(commands, "rclone_rmdir", lambda *args: None) + + with db() as conn: + conn.execute( + """ + INSERT INTO backups( + id, job_id, guest_vmid, guest_name, guest_type, node, state, + local_path, remote_path, started_at, completed_at, updated_at + ) + VALUES ('retention-local', ?, 110, 'test-guest', 'vm', 'pve', 'completed', + ?, 'onedrive:pve-cloud-backup/retention-local/archive.tar.zst', ?, ?, ?) + """, + (job["id"], str(local_file), utc_now(), utc_now(), utc_now()), + ) + + asyncio.run(backup_service.apply_retention_for_job(job["id"])) + + assert not local_file.exists() + final = backup_service.get_backup("retention-local") + assert final["state"] == "deleted" + assert final["local_path"] is None + + +def test_recover_unfinished_resumes_local_downloading(isolated_db, no_notifications, monkeypatch): + from app import backup_service + + resumed = [] + + async def fake_download_backup_locally(backup_id): + resumed.append(backup_id) + + monkeypatch.setattr(backup_service, "download_backup_locally", fake_download_backup_locally) + with db() as conn: + conn.execute( + """ + INSERT INTO backups(id, guest_vmid, guest_name, guest_type, node, state, started_at, updated_at) + VALUES ('resume-download', 110, 'test-guest', 'vm', 'pve', 'local_downloading', ?, ?) + """, + (utc_now(), utc_now()), + ) + + asyncio.run(backup_service.recover_unfinished()) + + assert resumed == ["resume-download"] + final = backup_service.get_backup("resume-download") + assert final["state"] == "completed" diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 6184594..9ad9918 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -17,6 +17,7 @@ type Settings = { default_retention_value: number max_concurrent_backups: number timezone: string + keep_local_backups: boolean } type Guest = { vmid: number; name: string; type: string; node: string; status: string } @@ -71,6 +72,7 @@ const emptySettings: Settings = { default_retention_value: 30, max_concurrent_backups: 1, timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC', + keep_local_backups: false, } const page = ref('Dashboard') @@ -87,6 +89,8 @@ const dashboard = ref({}) const selectedBackup = ref(null) const deletingJobId = ref(null) const deletingBackupId = ref(null) +const downloadingBackupId = ref(null) +const deletingLocalBackupId = ref(null) const nowMs = ref(Date.now()) let clockTimer: number | undefined const filters = reactive({ guest_vmid: '', state: '' }) @@ -348,6 +352,46 @@ async function deleteBackup(backup: Backup) { } } +async function downloadBackup(backup: Backup) { + error.value = '' + actionMessage.value = '' + downloadingBackupId.value = backup.id + loading.value = true + actionMessage.value = `Downloading backup ${backup.id} from the remote. This can take a while for large archives...` + try { + const result = await api(`/api/backups/${backup.id}/download`, { method: 'POST' }) + selectedBackup.value = result + actionMessage.value = `Downloaded to ${result.local_path}. In the Proxmox UI, open storage "${settings.proxmox_storage}" on node "${settings.proxmox_node}", find this archive under Backups, and click Restore. This app does not perform restores itself.` + await loadBackups() + } catch (e: any) { + error.value = e.message + } finally { + loading.value = false + downloadingBackupId.value = null + } +} + +async function deleteLocalCopy(backup: Backup) { + error.value = '' + actionMessage.value = '' + const ok = window.confirm(`Delete the local copy of backup ${backup.id}? The remote copy is left untouched.`) + if (!ok) return + deletingLocalBackupId.value = backup.id + loading.value = true + actionMessage.value = `Deleting local copy of backup ${backup.id}...` + try { + const result = await api(`/api/backups/${backup.id}/local`, { method: 'DELETE' }) + selectedBackup.value = result + actionMessage.value = `Deleted local copy of backup ${backup.id}.` + await loadBackups() + } catch (e: any) { + error.value = e.message + } finally { + loading.value = false + deletingLocalBackupId.value = null + } +} + function fmtBytes(value?: number) { if (!value) return '0 B' const units = ['B', 'KB', 'MB', 'GB', 'TB'] @@ -451,6 +495,7 @@ onUnmounted(() => { +
@@ -595,13 +640,31 @@ onUnmounted(() => {

Backup details

- +
+ + + +

Select a backup from Backup history.

-
+
State
{{ selectedBackup.state }} @@ -619,6 +682,10 @@ onUnmounted(() => {
Job
#{{ valueOrDash(selectedBackup.job_id) }}
+
+
Local copy
+
{{ selectedBackup.local_path ? 'Kept' : 'Not kept' }}
+
@@ -674,6 +741,7 @@ onUnmounted(() => {
+