Initial pve cloud backup app
This commit is contained in:
Codex
@@ -0,0 +1,489 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from . import commands
|
||||
from .db import db, row_to_dict, utc_now
|
||||
from .jobs import advance_job, due_jobs
|
||||
from .notifications import notify
|
||||
from .settings_store import get_settings
|
||||
|
||||
|
||||
ACTIVE_STATES = {
|
||||
"queued",
|
||||
"pve_running",
|
||||
"local_ready",
|
||||
"uploading",
|
||||
"remote_ready",
|
||||
"local_deleting",
|
||||
"deleting",
|
||||
}
|
||||
RUNNING_STATES = ACTIVE_STATES - {"queued"}
|
||||
RECOVERABLE_STATES = {
|
||||
"pve_running",
|
||||
"local_ready",
|
||||
"uploading",
|
||||
"remote_ready",
|
||||
"local_deleting",
|
||||
"deleting",
|
||||
}
|
||||
TERMINAL_STATES = {"completed", "failed", "deleted"}
|
||||
|
||||
|
||||
def _active_for_guest(conn, vmid: int) -> bool:
|
||||
row = conn.execute(
|
||||
f"SELECT 1 FROM backups WHERE guest_vmid = ? AND state IN ({','.join('?' for _ in ACTIVE_STATES)}) LIMIT 1",
|
||||
(vmid, *sorted(ACTIVE_STATES)),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _backup_row(backup_id: str) -> dict:
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
if not row:
|
||||
raise KeyError(f"Backup {backup_id} not found")
|
||||
return dict(row)
|
||||
|
||||
|
||||
def _job_row(job_id: int | None) -> dict | None:
|
||||
if job_id is None:
|
||||
return None
|
||||
with db() as conn:
|
||||
row = conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
return row_to_dict(row)
|
||||
|
||||
|
||||
def update_backup(backup_id: str, **values) -> dict:
|
||||
values["updated_at"] = utc_now()
|
||||
assignments = ", ".join(f"{key} = ?" for key in values)
|
||||
with db() as conn:
|
||||
conn.execute(
|
||||
f"UPDATE backups SET {assignments} WHERE id = ?",
|
||||
(*values.values(), backup_id),
|
||||
)
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def fail_backup(backup_id: str, error: str) -> dict:
|
||||
return update_backup(backup_id, state="failed", error_message=error, completed_at=utc_now())
|
||||
|
||||
|
||||
def _claim_queued_backup(backup_id: str) -> dict | None:
|
||||
now = utc_now()
|
||||
with db() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE backups SET state = 'pve_running', updated_at = ? WHERE id = ? AND state = 'queued'",
|
||||
(now, backup_id),
|
||||
)
|
||||
if cur.rowcount == 0:
|
||||
return None
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def queue_backup_for_job(job: dict) -> dict | None:
|
||||
now = utc_now()
|
||||
backup_id = str(uuid.uuid4())
|
||||
settings = get_settings()
|
||||
max_active = max(1, int(settings["max_concurrent_backups"]))
|
||||
with db() as conn:
|
||||
active_count = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE state IN ({','.join('?' for _ in ACTIVE_STATES)})",
|
||||
tuple(sorted(ACTIVE_STATES)),
|
||||
).fetchone()["count"]
|
||||
if active_count >= max_active:
|
||||
return None
|
||||
if _active_for_guest(conn, int(job["guest_vmid"])):
|
||||
return None
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO backups(
|
||||
id, job_id, guest_vmid, guest_name, guest_type, node, state,
|
||||
started_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'queued', ?, ?)
|
||||
""",
|
||||
(
|
||||
backup_id,
|
||||
job["id"],
|
||||
job["guest_vmid"],
|
||||
job["guest_name"],
|
||||
job["guest_type"],
|
||||
job["node"],
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
row = conn.execute("SELECT * FROM backups WHERE id = ?", (backup_id,)).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def list_backups(guest_vmid: int | None = None, state: str | None = None) -> list[dict]:
|
||||
query = "SELECT * FROM backups"
|
||||
args: list = []
|
||||
where = []
|
||||
if guest_vmid is not None:
|
||||
where.append("guest_vmid = ?")
|
||||
args.append(guest_vmid)
|
||||
if state:
|
||||
where.append("state = ?")
|
||||
args.append(state)
|
||||
if where:
|
||||
query += " WHERE " + " AND ".join(where)
|
||||
query += " ORDER BY started_at DESC LIMIT 500"
|
||||
with db() as conn:
|
||||
return [dict(row) for row in conn.execute(query, args).fetchall()]
|
||||
|
||||
|
||||
def get_backup(backup_id: str) -> dict:
|
||||
return _backup_row(backup_id)
|
||||
|
||||
|
||||
def purge_deleted_backup(backup_id: str) -> dict:
|
||||
backup = _backup_row(backup_id)
|
||||
if backup["state"] != "deleted":
|
||||
raise RuntimeError(f"Backup {backup_id} is in state {backup['state']} and is not ready to purge")
|
||||
with db() as conn:
|
||||
conn.execute("DELETE FROM backups WHERE id = ?", (backup_id,))
|
||||
return {"id": backup_id, "purged": True}
|
||||
|
||||
|
||||
def dashboard() -> dict:
|
||||
with db() as conn:
|
||||
latest = row_to_dict(conn.execute("SELECT * FROM backups ORDER BY started_at DESC LIMIT 1").fetchone())
|
||||
active = [dict(row) for row in conn.execute(
|
||||
f"SELECT * FROM backups WHERE state IN ({','.join('?' for _ in ACTIVE_STATES)}) ORDER BY started_at DESC",
|
||||
tuple(sorted(ACTIVE_STATES)),
|
||||
).fetchall()]
|
||||
failed = conn.execute("SELECT COUNT(*) AS count FROM backups WHERE state = 'failed'").fetchone()["count"]
|
||||
remote = conn.execute("SELECT COUNT(*) AS count FROM backups WHERE state = 'completed' AND remote_path IS NOT NULL").fetchone()["count"]
|
||||
next_jobs = [dict(row) for row in conn.execute(
|
||||
"SELECT * FROM backup_jobs WHERE enabled = 1 ORDER BY next_run_at ASC LIMIT 10"
|
||||
).fetchall()]
|
||||
settings = get_settings()
|
||||
local_dir = Path(settings["local_backup_dir"])
|
||||
usage = {"path": str(local_dir), "bytes": 0}
|
||||
if local_dir.exists():
|
||||
usage["bytes"] = sum(p.stat().st_size for p in local_dir.glob("*") if p.is_file())
|
||||
return {
|
||||
"latest_backup": latest,
|
||||
"active_tasks": active,
|
||||
"failed_backups": failed,
|
||||
"remote_backup_count": remote,
|
||||
"local_staging_usage": usage,
|
||||
"next_scheduled_jobs": next_jobs,
|
||||
}
|
||||
|
||||
|
||||
async def _wait_for_pve_task(backup: dict) -> dict:
|
||||
while True:
|
||||
status = await asyncio.to_thread(commands.get_task_status, backup["node"], backup["upid"])
|
||||
if status.get("status") == "stopped":
|
||||
exitstatus = status.get("exitstatus")
|
||||
if exitstatus == "OK":
|
||||
return status
|
||||
raise RuntimeError(f"Proxmox task failed with exitstatus={exitstatus}")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
def _remote_object(settings: dict, backup: dict, archive: Path) -> str:
|
||||
remote = str(settings["rclone_remote"]).rstrip(":")
|
||||
base_path = str(settings.get("rclone_remote_path") or "pve-cloud-backup").strip("/")
|
||||
object_path = f"{backup['id']}/{archive.name}"
|
||||
if base_path:
|
||||
object_path = f"{base_path}/{object_path}"
|
||||
return f"{remote}:{object_path}"
|
||||
|
||||
|
||||
def _remote_parent_dir(remote_object: str) -> str | None:
|
||||
if ":" not in remote_object:
|
||||
return None
|
||||
remote, object_path = remote_object.split(":", 1)
|
||||
object_path = object_path.strip("/")
|
||||
if "/" not in object_path:
|
||||
return None
|
||||
parent_path = object_path.rsplit("/", 1)[0]
|
||||
if not parent_path:
|
||||
return None
|
||||
return f"{remote}:{parent_path}"
|
||||
|
||||
|
||||
async def _remove_empty_remote_parent(settings: dict, remote_object: str) -> None:
|
||||
parent = _remote_parent_dir(remote_object)
|
||||
if not parent:
|
||||
return
|
||||
try:
|
||||
await asyncio.to_thread(commands.rclone_rmdir, settings["rclone_path"], parent)
|
||||
except commands.CommandError:
|
||||
# Non-empty folders and backend-specific "directory not found" behavior
|
||||
# should not turn an already successful exact file delete into a failed
|
||||
# backup deletion.
|
||||
return
|
||||
|
||||
|
||||
async def _delete_remote_file_and_empty_parent(settings: dict, remote_object: str) -> None:
|
||||
await asyncio.to_thread(commands.rclone_delete_file, settings["rclone_path"], remote_object)
|
||||
await _remove_empty_remote_parent(settings, remote_object)
|
||||
|
||||
|
||||
async def process_backup(backup_id: str) -> None:
|
||||
backup = _backup_row(backup_id)
|
||||
settings = get_settings()
|
||||
job = _job_row(backup.get("job_id"))
|
||||
try:
|
||||
if backup["state"] == "queued":
|
||||
await notify("backup_started", backup=backup)
|
||||
backup = _claim_queued_backup(backup_id)
|
||||
if backup is None:
|
||||
return
|
||||
upid = await asyncio.to_thread(
|
||||
commands.start_proxmox_backup,
|
||||
node=backup["node"],
|
||||
vmid=int(backup["guest_vmid"]),
|
||||
storage=(job or {}).get("proxmox_storage") or settings["proxmox_storage"],
|
||||
mode=(job or {}).get("backup_mode") or settings["default_backup_mode"],
|
||||
compression=(job or {}).get("compression") or settings["default_compression"],
|
||||
)
|
||||
backup = update_backup(backup_id, upid=upid)
|
||||
|
||||
if backup["state"] == "pve_running":
|
||||
if not backup.get("upid"):
|
||||
raise RuntimeError("Backup is pve_running but has no Proxmox UPID recorded; refusing to poll task status")
|
||||
await _wait_for_pve_task(backup)
|
||||
backup = update_backup(backup_id, state="local_ready", pve_completed_at=utc_now())
|
||||
await notify("pve_completed", backup=backup)
|
||||
|
||||
if backup["state"] == "local_ready":
|
||||
archive = await asyncio.to_thread(
|
||||
commands.discover_archive,
|
||||
settings["local_backup_dir"],
|
||||
int(backup["guest_vmid"]),
|
||||
backup["guest_type"],
|
||||
backup["started_at"],
|
||||
)
|
||||
size = archive.stat().st_size
|
||||
remote_object = _remote_object(settings, backup, archive)
|
||||
backup = update_backup(
|
||||
backup_id,
|
||||
local_path=str(archive),
|
||||
remote_path=remote_object,
|
||||
size_bytes=size,
|
||||
state="uploading",
|
||||
upload_started_at=utc_now(),
|
||||
)
|
||||
await notify("upload_started", backup=backup)
|
||||
|
||||
if backup["state"] == "uploading":
|
||||
await asyncio.to_thread(
|
||||
commands.rclone_copyto,
|
||||
settings["rclone_path"],
|
||||
backup["local_path"],
|
||||
backup["remote_path"],
|
||||
)
|
||||
exists = await asyncio.to_thread(
|
||||
commands.rclone_object_exists,
|
||||
settings["rclone_path"],
|
||||
backup["remote_path"],
|
||||
)
|
||||
if not exists:
|
||||
raise RuntimeError("Upload finished but remote object was not found")
|
||||
backup = update_backup(backup_id, state="remote_ready")
|
||||
|
||||
if backup["state"] == "remote_ready":
|
||||
backup = update_backup(backup_id, state="local_deleting")
|
||||
|
||||
if backup["state"] == "local_deleting":
|
||||
if backup.get("local_path"):
|
||||
local_path = Path(backup["local_path"])
|
||||
if local_path.exists():
|
||||
local_path.unlink()
|
||||
backup = update_backup(backup_id, state="completed", completed_at=utc_now())
|
||||
await notify("backup_completed", backup=backup)
|
||||
await apply_retention_for_job(backup["job_id"])
|
||||
except Exception as exc:
|
||||
failed = fail_backup(backup_id, str(exc))
|
||||
await notify("backup_failed", backup=failed)
|
||||
|
||||
|
||||
async def delete_remote_backup(backup: dict, reason: str) -> None:
|
||||
if not backup.get("remote_path"):
|
||||
update_backup(backup["id"], state="deleted", deleted_at=utc_now())
|
||||
return
|
||||
settings = get_settings()
|
||||
update_backup(backup["id"], state="deleting")
|
||||
await _delete_remote_file_and_empty_parent(settings, backup["remote_path"])
|
||||
deleted = update_backup(backup["id"], state="deleted", deleted_at=utc_now())
|
||||
await notify("remote_deleted", backup=deleted, message=reason)
|
||||
|
||||
|
||||
async def delete_backup_manually(backup_id: str, reason: str = "manual deletion") -> dict:
|
||||
backup = _backup_row(backup_id)
|
||||
if backup["state"] == "deleted":
|
||||
return purge_deleted_backup(backup_id)
|
||||
if backup["state"] in ACTIVE_STATES:
|
||||
raise RuntimeError(f"Backup {backup_id} is active in state {backup['state']} and cannot be deleted")
|
||||
|
||||
settings = get_settings()
|
||||
if backup.get("remote_path") and backup["state"] != "deleted":
|
||||
exists = await asyncio.to_thread(
|
||||
commands.rclone_object_exists,
|
||||
settings["rclone_path"],
|
||||
backup["remote_path"],
|
||||
)
|
||||
if exists:
|
||||
update_backup(backup_id, state="deleting")
|
||||
await _delete_remote_file_and_empty_parent(settings, backup["remote_path"])
|
||||
else:
|
||||
await _remove_empty_remote_parent(settings, backup["remote_path"])
|
||||
|
||||
if backup.get("local_path"):
|
||||
local_path = Path(backup["local_path"])
|
||||
if local_path.exists() and local_path.is_file():
|
||||
local_path.unlink()
|
||||
|
||||
deleted = update_backup(backup_id, state="deleted", deleted_at=utc_now())
|
||||
await notify("remote_deleted", backup=deleted, message=reason)
|
||||
return deleted
|
||||
|
||||
|
||||
async def delete_job_with_backups(job_id: int) -> dict:
|
||||
with db() as conn:
|
||||
job = row_to_dict(conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone())
|
||||
if not job:
|
||||
raise KeyError(f"Backup job {job_id} not found")
|
||||
active = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE job_id = ? AND state IN ({','.join('?' for _ in ACTIVE_STATES)})",
|
||||
(job_id, *sorted(ACTIVE_STATES)),
|
||||
).fetchone()["count"]
|
||||
if active:
|
||||
raise RuntimeError(f"Backup job {job_id} has {active} active backup(s) and cannot be deleted")
|
||||
backups = [
|
||||
dict(row)
|
||||
for row in conn.execute(
|
||||
"SELECT * FROM backups WHERE job_id = ? AND state != 'deleted' ORDER BY started_at ASC",
|
||||
(job_id,),
|
||||
).fetchall()
|
||||
]
|
||||
|
||||
deleted_backup_ids: list[str] = []
|
||||
for backup in backups:
|
||||
deleted = await delete_backup_manually(backup["id"], reason=f"job {job_id} deleted")
|
||||
deleted_backup_ids.append(deleted["id"])
|
||||
|
||||
with db() as conn:
|
||||
conn.execute("DELETE FROM backup_jobs WHERE id = ?", (job_id,))
|
||||
return {"job_id": job_id, "deleted_backups": deleted_backup_ids}
|
||||
|
||||
|
||||
async def apply_retention_for_job(job_id: int | None) -> None:
|
||||
if job_id is None:
|
||||
return
|
||||
with db() as conn:
|
||||
job = row_to_dict(conn.execute("SELECT * FROM backup_jobs WHERE id = ?", (job_id,)).fetchone())
|
||||
backups = [dict(row) for row in conn.execute(
|
||||
"""
|
||||
SELECT * FROM backups
|
||||
WHERE job_id = ? AND state = 'completed' AND remote_path IS NOT NULL
|
||||
ORDER BY completed_at DESC
|
||||
""",
|
||||
(job_id,),
|
||||
).fetchall()]
|
||||
if not job:
|
||||
return
|
||||
delete: list[dict] = []
|
||||
if job["retention_type"] == "latest":
|
||||
delete = backups[int(job["retention_value"]):]
|
||||
else:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=int(job["retention_value"]))
|
||||
for backup in backups:
|
||||
completed_at = backup.get("completed_at")
|
||||
if completed_at and datetime.fromisoformat(completed_at) < cutoff:
|
||||
delete.append(backup)
|
||||
for backup in delete:
|
||||
try:
|
||||
await delete_remote_backup(backup, f"retention {job['retention_type']}={job['retention_value']}")
|
||||
except Exception as exc:
|
||||
failed = fail_backup(backup["id"], f"Retention delete failed: {exc}")
|
||||
await notify("backup_failed", backup=failed)
|
||||
|
||||
|
||||
async def retention_sweep() -> None:
|
||||
with db() as conn:
|
||||
job_ids = [
|
||||
row["id"]
|
||||
for row in conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT backup_jobs.id
|
||||
FROM backup_jobs
|
||||
JOIN backups ON backups.job_id = backup_jobs.id
|
||||
WHERE backups.state = 'completed' AND backups.remote_path IS NOT NULL
|
||||
"""
|
||||
).fetchall()
|
||||
]
|
||||
for job_id in job_ids:
|
||||
await apply_retention_for_job(job_id)
|
||||
|
||||
|
||||
async def recover_unfinished() -> None:
|
||||
with db() as conn:
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM backups WHERE state IN ({','.join('?' for _ in RECOVERABLE_STATES)})",
|
||||
tuple(sorted(RECOVERABLE_STATES)),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
backup = dict(row)
|
||||
await notify("recovery_action", backup=backup, message=f"resuming state {backup['state']}")
|
||||
if backup["state"] == "deleting":
|
||||
await delete_remote_backup(backup, "recovery delete resume")
|
||||
else:
|
||||
await process_backup(backup["id"])
|
||||
|
||||
|
||||
async def scheduler_tick() -> None:
|
||||
settings = get_settings()
|
||||
limit = max(1, int(settings["max_concurrent_backups"]))
|
||||
with db() as conn:
|
||||
active_count = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE state IN ({','.join('?' for _ in ACTIVE_STATES)})",
|
||||
tuple(sorted(ACTIVE_STATES)),
|
||||
).fetchone()["count"]
|
||||
free_slots = max(0, limit - active_count)
|
||||
if free_slots <= 0:
|
||||
return
|
||||
for job in due_jobs(free_slots):
|
||||
queued = queue_backup_for_job(job)
|
||||
advance_job(job)
|
||||
if queued:
|
||||
asyncio.create_task(process_backup(queued["id"]))
|
||||
|
||||
|
||||
async def worker_loop() -> None:
|
||||
await recover_unfinished()
|
||||
retention_sweep_interval_seconds = 3600
|
||||
last_retention_sweep = 0.0
|
||||
while True:
|
||||
await scheduler_tick()
|
||||
now = asyncio.get_running_loop().time()
|
||||
if now - last_retention_sweep >= retention_sweep_interval_seconds:
|
||||
await retention_sweep()
|
||||
last_retention_sweep = now
|
||||
settings = get_settings()
|
||||
max_active = max(1, int(settings["max_concurrent_backups"]))
|
||||
with db() as conn:
|
||||
active_count = conn.execute(
|
||||
f"SELECT COUNT(*) AS count FROM backups WHERE state IN ({','.join('?' for _ in RUNNING_STATES)})",
|
||||
tuple(sorted(RUNNING_STATES)),
|
||||
).fetchone()["count"]
|
||||
free_slots = max(0, max_active - active_count)
|
||||
queued = [dict(row) for row in conn.execute(
|
||||
"SELECT * FROM backups WHERE state = 'queued' ORDER BY started_at ASC LIMIT ?",
|
||||
(free_slots,),
|
||||
).fetchall()]
|
||||
for backup in queued:
|
||||
asyncio.create_task(process_backup(backup["id"]))
|
||||
await asyncio.sleep(30)
|
||||
Reference in New Issue
Block a user